Generate And Incrementally Stream A ZIP Archive To Amazon S3 Using Multipart Uploads In Lucee CFML 5.3.7.47

<cfscript>

	// NOTE: We'll be using Lucee CFML's ability to create Java classes from a set of JAR
	// files. In this case, we have the JAR files for AWS SDK 1.11.870 locally.
	s3Client = new S3Client(
		awsAccessID = server.aws.accessID,
		awsSecretKey = server.aws.secretKey,
		awsRegion = server.aws.region,
		awsBucket = server.aws.bucket,
		jarPaths = directoryList(
			path = expandPath( "../aws-s3-sdk/" ),
			recurse = true,
			listInfo = "path",
			type = "file",
			filter = "*.jar"
		)
	);

	// NOTE: Defines an array, "imageUrls", that contains remote files to download and
	// then zip / archive together.
	include "./image_urls.cfm";

	// ------------------------------------------------------------------------------- //
	// ------------------------------------------------------------------------------- //

	// While we can't "stream" the ZIP archive to Amazon S3 in the purest sense, we can
	// "chunk" the ZIP archive using a Multipart Upload wherein we slice-off parts of the
	// Output Stream and upload them when they are large enough (5mb minimum). Every
	// upload request requires us to provide the S3 resource path (key).
	path = "multipart-stream/images-#createUniqueId()#.zip";
	contentType = "application/zip";

	// When we finalize the multipart upload, we have to provide all of the ETags
	// returned by the individual parts so that Amazon can ensure nothing was corrupted.
	// And, since we have to keep track of these, we'll use the LENGTH of this array to
	// determine the Part Number of each upload.
	partFutures = [];

	uploadID = s3Client
		.createMultipartUpload(
			resourcePath = path,
			resourceContentType = contentType
		)
		.id
	;

	try {

		ZipEntryClass = createObject( "java", "java.util.zip.ZipEntry" );

		// In order to chunk the ZIP archive, we need to write the output to an in-memory
		// stream so that we can read from the stream as it is being generated. This way,
		// as we reach our minimum-part-size, we can stop and upload it.
		binaryOutputStream = javaNew( "java.io.ByteArrayOutputStream" )
			.init()
		;
		zipOutputStream = javaNew( "java.util.zip.ZipOutputStream" )
			.init( binaryOutputStream )
		;

		// In a multipart upload, Amazon requires that all parts be AT LEAST 5mb in size.
		// Except for the last part, which has no limit. As such, we can only initial an
		// upload when our ByteArrayOutputStream exceeds this limit.
		minimumPartSize = ( 1024 * 1024 * 5 );

		imageUrls.each(
			( imageUrl ) => {

				info( "Downloading: #imageUrl#" );

				// Download the remote URL.
				// --
				// NOTE: How awesome is it that we can download the remote URL with a
				// simple file-read operation. Yo, come on. Lucee, you complete me!
				var imageFilename = getFileFromPath( imageUrl );
				var imageBinary = fileReadBinary( imageUrl );

				// Write the image binary to the Zip output stream. Notice that we are
				// using the STORED method which means that we're not actually
				// compressing the entries - we're just archiving them. Since image files
				// already have compression, this should result in faster processing /
				// less CPU overhead.
				var zipEntry = javaNew( "java.util.zip.ZipEntry" )
					.init( "streaming-zip/images/#imageFilename#" )
				;
				zipEntry.setMethod( ZipEntryClass.STORED );
				zipEntry.setSize( arrayLen( imageBinary ) );
				zipEntry.setCrc( crc32( imageBinary ) );

				zipOutputStream.putNextEntry( zipEntry );
				zipOutputStream.write( imageBinary );
				zipOutputStream.closeEntry();
				zipOutputStream.flush();

				info( "Buffer size: #numberFormat( binaryOutputStream.size() )#" );

				// Each upload in the multipart upload (less the last part) has to be at
				// least 5mb in size. As such, let's check our output stream after each
				// Zip entry to see if we've reached the upload size threshold.
				if ( binaryOutputStream.size() < minimumPartSize ) {

					return;

				}

				// We are OVER THE MINIMUM PART SIZE! At this point, we can strip-off
				// whatever ZIP archive data we have flushed to our output stream and
				// push it up to Amazon S3.
				info( "**** ByteArrayOutputStream size has passed minimum size requirement ****" );
				info( "**** Uploading part to S3 ****" );

				var partNumber = ( partFutures.len() + 1 );
				var partContent = binaryOutputStream.toByteArray();

				// To try and keep things moving along quickly, we're going to try
				// uploading the part IN PARALLEL with the rest of the operations using a
				// Future.
				partFutures.append(
					runAsync(
						() => {

							var apiResponse = s3Client.uploadPart(
								resourcePath = path,
								multipartUploadID = uploadID,
								partNumber = partNumber,
								content = partContent,
								isLastPart = false
							);

							info( "**** Completed part #partNumber# upload ****" );
							return( apiResponse );

						}
					)
				);

				// Reset the output stream so all new ZIP archive data will be pushed to
				// the next "part".
				binaryOutputStream.reset();

			}
		); // END: each().

		// Finalize the Zip content.
		zipOutputStream.flush();
		zipOutputStream.close();
		binaryOutputStream.close();

		info( "Uploading LAST part to S3" );

		// TODO: What if output stream is empty at this point? Is that something we have
		// to account for with the ZipOutputStream class?

		partNumber = ( partFutures.len() + 1 );
		partContent = binaryOutputStream.toByteArray();
		partFutures.append(
			runAsync(
				() => {

					var apiResponse = s3Client.uploadPart(
						resourcePath = path,
						multipartUploadID = uploadID,
						partNumber = partNumber,
						content = partContent,
						isLastPart = true
					);

					info( "**** Completed LAST PART upload ****" );
					return( apiResponse );

				}
			)
		);

		info( "Waiting for parts to finish uploading" );

		// At this point, we've INITIATED all the part uploads; but, they may not have
		// finished. By mapping the Futures onto the Responses we will block-and-wait
		// for each part to complete uploading.
		etags = partFutures.map(
			( future ) => {

				return( future.get().awsResponse.getPartETag() );

			}
		);

		apiResponse = s3Client.completeMultipartUpload(
			resourcePath = path,
			multipartUploadID = uploadID,
			multipartETags = etags
		);

		info( "*******************************" )
		info( "-- Multipart upload complete --" );
		info( "*******************************" )
		echo( "Zip archive complete: #path#" );

	// If any errors occur during the processing, let's abort the multi-part request so
	// that we don't keep getting charged for the parts that have been uploaded.
	} catch ( any error ) {

		s3Client.abortMultipartUpload(
			resourcePath = path,
			multipartUploadID = uploadID
		);

		dump( error );

	}

	// ------------------------------------------------------------------------------- //
	// ------------------------------------------------------------------------------- //

	/**
	* I compute the CRC-32 checksum for the byte array.
	*
	* @input I am the input being checked.
	*/
	public numeric function crc32( required binary input ) {

		var checksum = createObject( "java", "java.util.zip.CRC32" ).init();
		checksum.update( input );

		return( checksum.getValue() );
	}


	/**
	* I log the given message to the standard output on its own line.
	* 
	* @message I am the message being logged.
	*/
	public void function info( required string message ) {

		systemOutput( message, true );

	}


	/**
	* I create a Java class instance with the given class name. This is just a short-hand
	* method for the createObject() call.
	* 
	* @className I am the Java class being created.
	*/
	public any function javaNew( required string className ) {

		return( createObject( "java", className ) );

	}

</cfscript>

component
	output = false
	hint = "I provide a simple wrapper for the S3 Client JDK classes."
	{

	/**
	* I initialize the S3 Client wrapper with the given configuration.
	* 
	* JavaDocs: https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/
	*/
	public void function init(
		required string awsAccessID,
		required string awsSecretKey,
		required string awsRegion,
		required string awsBucket,
		required array jarPaths
		) {

		variables.awsAccessID = arguments.awsAccessID;
		variables.awsSecretKey = arguments.awsSecretKey;
		variables.awsRegion = arguments.awsRegion;
		variables.awsBucket = arguments.awsBucket;
		variables.jarPaths = arguments.jarPaths;

		variables.s3Client = createS3Client();

	}

	// ---
	// PUBLIC METHODS.
	// ---

	/**
	* I abort the multipart upload with the given ID.
	* 
	* @resourcePath I am the key at which the final upload was going to be stored.
	* @multipartUploadID I am the ID for the multipart upload workflow.
	*/
	public void function abortMultipartUpload(
		required string resourcePath,
		required string multipartUploadID
		) {

		var abortRequest = loadClass( "com.amazonaws.services.s3.model.AbortMultipartUploadRequest" )
			.init( awsBucket, resourcePath, multipartUploadID )
		;

		s3Client.abortMultipartUpload( abortRequest );

	}


	/**
	* I complete the multipart upload with the given ID.
	* 
	* @resourcePath I am the key at which the final upload is going to be stored.
	* @multipartUploadID I am the ID for the multipart upload workflow.
	* @multipartETags I am the collection of ETags returned from the part-uploads.
	*/
	public struct function completeMultipartUpload(
		required string resourcePath,
		required string multipartUploadID,
		required array multipartETags
		) {

		var completeRequest = loadClass( "com.amazonaws.services.s3.model.CompleteMultipartUploadRequest" )
			.init( awsBucket, resourcePath, multipartUploadID, multipartETags )
		;

		var awsResponse = s3Client.completeMultipartUpload( completeRequest );

		return({
			awsResponse: awsResponse
		});

	}


	/**
	* I initialize a multipart upload for the given path.
	* 
	* @resourcePath I am the key at which the final upload is going to be stored.
	* @resourceContentType I am the mime-type of the final upload.
	*/
	public struct function createMultipartUpload(
		required string resourcePath,
		required string resourceContentType
		) {

		var uploadMetadata = loadClass( "com.amazonaws.services.s3.model.ObjectMetadata" ).init();
		uploadMetadata.setContentType( resourceContentType );

		var uploadRequest = loadClass( "com.amazonaws.services.s3.model.InitiateMultipartUploadRequest" )
			.init( awsBucket, resourcePath, uploadMetadata )
		;

		var awsResponse = s3Client.initiateMultipartUpload( uploadRequest );

		return({
			id: awsResponse.getUploadId(),
			awsResponse: awsResponse
		});

	}


	/**
	* I upload the given binary content as a unique part in a multipart upload.
	* 
	* @resourcePath I am the key at which the final upload is going to be stored.
	* @multipartUploadID I am the ID for the multipart upload workflow.
	* @partNumber I am the unique part number (1 - 10,000).
	* @content I am the binary / byte array data for the part.
	* @isLastPart I determine if this is the last part in the workflow.
	*/
	public struct function uploadPart(
		required string resourcePath,
		required string multipartUploadID,
		required numeric partNumber,
		required binary content,
		required boolean isLastPart
		) {

		var uploadRequest = loadClass( "com.amazonaws.services.s3.model.UploadPartRequest" )
			.init()
			.withBucketName( awsBucket )
			.withKey( resourcePath )
			.withUploadId( multipartUploadID )
			.withPartNumber( partNumber )
			.withLastPart( isLastPart )
			.withInputStream( asByteArrayInputStream( content ) )
			.withPartSize( arrayLen( content ) )
		;

		var awsResponse = s3Client.uploadPart( uploadRequest );

		return({
			awsResponse: awsResponse
		});

	}

	// ---
	// PRIVATE METHODS.
	// ---

	/**
	* I create a byte-array input stream from the given binary value.
	* 
	* @value I am the binary payload being wrapped.
	*/
	private any function asByteArrayInputStream( required binary value ) {

		return( javaNew( "java.io.ByteArrayInputStream" ).init( value ) );

	}


	/**
	* I create the S3 Client instance that this component is wrapping.
	*/
	private any function createS3Client() {

		// Load some class that we'll use for static references.
		var ProtocolClass = loadClass( "com.amazonaws.Protocol" );
		var RegionClass = loadClass( "com.amazonaws.regions.Region" );
		var RegionsClass = loadClass( "com.amazonaws.regions.Regions" );

		var clientConfig = loadClass( "com.amazonaws.ClientConfiguration" ).init();
		clientConfig.setProtocol( ProtocolClass.HTTPS );
		clientConfig.setMaxErrorRetry( 4 );

		var clientCredentials = loadClass( "com.amazonaws.auth.BasicAWSCredentials" )
			.init( awsAccessID, awsSecretKey )
		;

		var clientOptions = loadClass( "com.amazonaws.services.s3.S3ClientOptions" )
			.init()
			.withPathStyleAccess( true )
		;

		var clientRegion = RegionClass.getRegion( RegionsClass.fromName( awsRegion ) );

		var s3Client = loadClass( "com.amazonaws.services.s3.AmazonS3Client" )
			.init( clientCredentials, clientConfig )
		;
		s3Client.setS3ClientOptions( clientOptions );
		s3Client.setRegion( clientRegion );

		return( s3Client );

	}


	/**
	* I load the Java class with the given name using the base class loader.
	* 
	* @className I am the class to load.
	*/
	private any function javaNew( required string className ) {

		return( createObject( "java", className ) );

	}


	/**
	* I load the Java class with the given name (using the AWS SDK JAR files).
	* 
	* @className I am the class to load.
	*/
	private any function loadClass( required string className ) {

		return( createObject( "java", className, jarPaths ) );

	}

}