From 3544408e842442e4fed150589cbc89c7b6ca40b7 Mon Sep 17 00:00:00 2001
From: Dorian Zaccaria <dorian.zaccaria@flamefy.com>
Date: Wed, 27 Jun 2018 14:50:56 +0000
Subject: [PATCH] [aws] First pass at AWS S3 cp multiparts

erlcloud_s3:put_object can upload a file smaller than 5GB according
to https://docs.aws.amazon.com/AmazonS3/latest/dev/UploadingObjects.html.
In order to upload bigger files we need to use multiparts upload, erlcloud
provides the needed functions to do that. This commit is a first try at
using those functions.
---
 src/rfile_aws.erl | 55 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 55 insertions(+)

diff --git a/src/rfile_aws.erl b/src/rfile_aws.erl
index 52ef52a..0235c0a 100644
--- a/src/rfile_aws.erl
+++ b/src/rfile_aws.erl
@@ -282,6 +282,61 @@ copy_file_s3_to_fs(SourceBucket, SourceKey, Destination, _Options, AwsConfig) ->
 copy_file_fs_to_s3(Source, DestinationBucket, DestinationKey, Options, AwsConfig) ->
   lager:info("COPY ~ts TO s3://~ts/~ts", [Source, DestinationBucket, DestinationKey]),
   AwsOptions = aws_options(Options, [acl]),
+  MaxFileSize = 5000000000,
+  FileSize = filelib:file_size(Source),
+  if
+    FileSize > MaxFileSize ->
+      lager:info("Starting multipart upload"),
+      do_copy_file_fs_to_s3_multiparts(Source, DestinationBucket, DestinationKey, AwsConfig, AwsOptions);
+    true ->
+      lager:info("Starting single upload"),
+      do_copy_file_fs_to_s3(Source, DestinationBucket, DestinationKey, AwsConfig, AwsOptions)
+  end.
+
+do_copy_file_fs_to_s3_multiparts(Source, DestinationBucket, DestinationKey, AwsConfig, AwsOptions) ->
+  case erlcloud_s3:start_multipart(DestinationBucket, DestinationKey, AwsOptions, [], AwsConfig) of
+    {ok, PropsList} ->
+      UploadId = proplists:get_value(uploadId, PropsList),
+      case file:open(Source, [read]) of
+        {ok, IoDevice} ->
+          lager:info("File is open"),
+          upload_part(DestinationBucket, DestinationKey, UploadId, 1, AwsOptions, AwsConfig, IoDevice);
+        {error, Reason} ->
+          erlcloud_s3:abort_multipart(DestinationBucket, DestinationKey, UploadId, AwsOptions, [], AwsConfig),
+          {error, Reason}
+      end;
+    {error, Reason} ->
+      {error, Reason}
+  end.
+
+upload_part(DestinationBucket, DestinationKey, UploadId, PartNumber, AwsOptions, AwsConfig, IoDevice) ->
+  case file:read(IoDevice, 5000000) of
+    {ok, Data} ->
+      lager:info("Uploading part ~w", [PartNumber]),
+      case erlcloud_s3:upload_part(DestinationBucket, DestinationKey, UploadId, PartNumber, Data, [], AwsConfig) of
+        {ok, _PropsList} ->
+          lager:info("Uploaded part ~w", [PartNumber]),
+          upload_part(DestinationBucket, DestinationKey, UploadId, PartNumber + 1, AwsOptions, AwsConfig, IoDevice);
+        {error, Reason} ->
+          erlcloud_s3:abort_multipart(DestinationBucket, DestinationKey, UploadId, AwsOptions, [], AwsConfig),
+          {error, Reason}
+      end;
+    eof ->
+      case file:close(IoDevice) of
+        ok ->
+          lager:info("File is closed"),
+          erlcloud_s3:complete_multipart(DestinationBucket, DestinationKey, UploadId, [], [], AwsConfig);
+        {error, Reason} ->
+          lager:info("Failed to close the file"),
+          erlcloud_s3:abort_multipart(DestinationBucket, DestinationKey, UploadId, AwsOptions, [], AwsConfig),
+          {error, Reason}
+      end;
+    {error, Reason} ->
+      erlcloud_s3:abort_multipart(DestinationBucket, DestinationKey, UploadId, AwsOptions, [], AwsConfig),
+      {error, Reason}
+  end.
+
+do_copy_file_fs_to_s3(Source, DestinationBucket, DestinationKey, AwsConfig, AwsOptions) ->
   case file:read_file(Source) of
     {ok, Data} ->
       try