Paginating The Postmark Bounces API In ColdFusion

<cfscript>

	// Move Postmark credentials to another file for demo.
	include "./credentials.cfm";

	config = {
		type: "SoftBounce",
		pattern: "(out of storage space|mailbox full|mailbox is full)",
		filename: "soft-bounce-mailbox-full.txt"
	};

	// ------------------------------------------------------------------------------- //
	// ------------------------------------------------------------------------------- //

	// Allocate enough time to make a lot of API calls.
	setting
		requestTimeout = ( 60 * 10 )
	;

	// The last mass emailing job was carried-out between the following dates.
	minFromDate = createDateTime( 2024, 3, 15, 0, 0, 0 );
	maxToDate = createDateTime( 2024, 3, 21, 0, 0, 0 );

	// The /bounces API only allows for the first 10,000 results in a given FROM/TO range
	// even if we're paginating over the results. And, due to the high-volume of bad
	// emails (collected over a decade-and-a-half), we hit this limit with some queries.
	// As such, we're going to iterate over the FROM/TO range one HOUR at a time. This
	// will be implemented by adding N-hours to the MIN FROM date.
	offsetInHours = 0;

	// With all DO-WHILE loops, I like to build in some sort of reasonable upper-boundary
	// that will short-circuit the loop if something goes wrong in my logic.
	attemptsRemaining = 150;

	// Gathering up all of these bounces from the Postmark API is going to require many
	// API calls. And, any one of these calls might break or timeout or my server might
	// crash. As such, we want to store "mile markers" of progress such that if we do need
	// to restart the process, we don't have to start from scratch. To keep things simple,
	// each HOUR of bounces will be considered a "unit of work". I'm going to collect
	// these emails in "part files" and store them individually.
	partFiles = [];

	// ------------------------------------------------------------------------------- //
	// ------------------------------------------------------------------------------- //

	do {

		// Iterate over FROM/TO range 1-HOUR at a time.
		fromDate = minFromDate.add( "h", offsetInHours );
		toDate = fromDate.add( "h", 1 );

		// Define the "mile marker" part file.
		partFile = "./parts/#config.filename#.#offsetInHours#";
		partFiles.append( partFile );

		// The part file is only written to disk once all the results within the current
		// HOUR have been collected. As such, we know that we only need to process this
		// hour when the part file hasn't been created yet.
		if ( ! fileExists( partFile ) ) {

			// Each page of results can only contain a maximum of 500 bounces. We'll need
			// collect the emails for this "part" across (potentially) many API calls.
			partEmails = [];

			// The total (limit + offset) can never go higher than 10,000. If we hit this
			// limit, we're just going to eat the fact that we drop some emails.
			for ( apiOffset = 0 ; apiOffset <= 9500 ; apiOffset += 500 ) {

				bounces = getBouncesFromPostmark(
					bounceType = config.type,
					fromDate = fromDate,
					toDate = toDate,
					offset = apiOffset
				);

				if ( ! bounces.len() ) {

					break; // Break out of pagination loop.

				}

				// Since many different bounce variations are categorized under the same
				// bounce type, we need to inspect the details to see if this bounce
				// result is relevant to our current query.
				for ( bounce in bounces ) {

					if ( lcase( bounce.Details ).reFind( config.pattern ) ) {

						partEmails.append( lcase( bounce.Email ) );

					}

				}

			} // END: Pagination loop.

			// Save "mile marker" part file.
			fileWrite( partFile, serializeJson( partEmails ), "utf-8" );

		} // END: If part file exists.

		offsetInHours++;

	} while ( ( toDate < maxToDate ) && ( --attemptsRemaining >= 0 ) );

	// ------------------------------------------------------------------------------- //
	// ------------------------------------------------------------------------------- //

	// At this point, we've iterated over our FROM/TO date range one HOUR at a time, and
	// stored each hour's worth of emails into a PART file. Now, it's time to coalesce all
	// of the part files into a single results file. However, since the FROM/TO filters
	// are INCLUSIVE on both ends, we end up with some overlapping emails within our part
	// files. As such, I'm going to use a STRUCT to coalesce the values such that
	// duplicate emails are automatically de-duplicated.
	emailsIndex = {};

	for ( partFile in partFiles ) {

		for ( email in deserializeJson( fileRead( partFile, "utf-8" ) ) ) {

			emailsIndex[ email ] = true;

		}

	}

	// Get the list of emails from the STRUCT that we used for de-duplication.
	emails = emailsIndex.keyArray();
	emails.sort( "text" );

	writeResultsFile( "./data/#config.filename#", emails );

	// Now that we've coalesced all the parts into a final results file, we can clean-up
	// the part files - we no longer need them.
	for ( partFile in partFiles ) {

		fileDelete( partFile );

	}

	echo( "Emails have been compiled. Rock on with your bad self!" );

	// ------------------------------------------------------------------------------- //
	// -- Utility Functions ---------------------------------------------------------- //
	// ------------------------------------------------------------------------------- //

	/**
	* I get a limited set of bounce results from the Postmark API.
	*/
	public struct function getBouncesFromPostmark(
		required string bounceType,
		required date fromDate,
		required date toDate,
		required numeric offset,
		numeric limit = 500,
		numeric timeout = 10
		) {

		// Since our FROM/TO range will be hourly, we need to format the EST dates (which
		// is what the Postmark API uses for filtering) to include the TIME portion.
		var filterMask = "yyyy-mm-dd'T'HH:nn:ss";
		var fromDateFilter = fromDate.dateTimeFormat( filterMask );
		var toDateFilter = toDate.dateTimeFormat( filterMask );

		// For my own debugging.
		systemOutput( "Bounces API: #bounceType# : #fromDateFilter# : #offset#", true );

		http
			result = "local.httpResponse"
			method = "GET"
			url = "https://api.postmarkapp.com/bounces"
			timeout = timeout
			{

			httpparam
				type = "header"
				name = "Accept"
				value = "application/json"
			;
			httpparam
				type = "header"
				name = "X-Postmark-Server-Token"
				value = credentials.serverToken
			;
			httpparam
				type = "url"
				name = "fromdate"
				value = fromDateFilter
			;
			httpparam
				type = "url"
				name = "todate"
				value = toDateFilter
			;
			httpparam
				type = "url"
				name = "messagestream"
				value = credentials.messageStream
			;
			httpparam
				type = "url"
				name = "type"
				value = bounceType
			;
			httpparam
				type = "url"
				name = "inactive"
				value = "false"
			;
			httpparam
				type = "url"
				name = "count"
				value = limit
			;
			httpparam
				type = "url"
				name = "offset"
				value = offset
			;
		}

		var apiResponse = deserializeJson( httpResponse.fileContent );

		return ( apiResponse.Bounces ?: [] );

	}

	/**
	* I write the emails to a formatted results file (for internal usage).
	*/
	public void function writeResultsFile(
		required string resultsFile,
		required array emails
		) {

		var suppressions = [ "Result,Email" ];

		for ( var email in emails ) {

			suppressions.append( "suppressed,#email#" );

		}

		fileWrite(
			resultsFile,
			suppressions.toList( chr( 10 ) ),
			"utf-8"
		);

	}

</cfscript>