Merge branch 'mocchira-develop' into develop

apps/leo_gateway/include/leo_gateway.hrl

@@ -166,6 +166,12 @@
             _ -> []
         end).
 
+-define(env_multipart_properties(),
+        case application:get_env(leo_gateway, multipart) of
+            {ok, EnvMultiObject} -> EnvMultiObject;
+            _ -> []
+        end).
+
 -define(env_recover_properties(),
         case application:get_env(leo_gateway, recover) of
             {ok, EnvRecover} -> EnvRecover;

apps/leo_gateway/include/leo_http.hrl

@@ -147,6 +147,7 @@
 -define(DEF_LOBJ_THRESHOLD_OF_CHUNK_LEN, 5767168).
 -define(DEF_S3API_MAX_KEYS, 1000).
 -define(DEF_MAX_NUM_OF_METADATAS, 50).
+-define(DEF_DONT_ABORT_CLEANUP, false).
 
 %% error codes used in a error response
 -define(XML_ERROR_CODE_EntityTooLarge, "EntityTooLarge").
@@ -507,6 +508,8 @@
           cachable_content_type = []   :: list(),         %% cachable content types
           cachable_path_pattern = []   :: list(),         %% cachable path patterns
           has_disk_cache = false       :: boolean(),      %% Has Disk Cache?
+          %% for multipart upload
+          dont_abort_cleanup = false   :: boolean(),      %% whether removing related objects when handling abort MU request
           %% for large-object
           max_chunked_objs = 0         :: pos_integer(),  %% max chunked objects
           max_len_of_obj = 0           :: pos_integer(),  %% max length a object (byte)
@@ -540,6 +543,8 @@
           is_cached = false          :: boolean(),              %% is cached?
           is_dir = false             :: boolean(),              %% is directory?
           is_multi_delete = false    :: boolean(),              %% is multi delete request?
+          %% for multipart upload
+          dont_abort_cleanup = false :: boolean(),              %% whether removing related objects when handling abort MU request
           %% for large-object
           is_upload = false            :: boolean(),            %% is upload operation? (for multipart upload)
           is_aws_chunked = false       :: boolean(),            %% is AWS Chunked? (Signature V4)

apps/leo_gateway/priv/leo_gateway.schema

@@ -335,6 +335,18 @@
   {default, 32}
  ]}.
 
+%% --------------------------------------------------------------------
+%% GATEWAY - Multipart
+%% --------------------------------------------------------------------
+%% @doc Whether removing related objects when handling abort MU request
+{mapping,
+ "multipart.dont_abort_cleanup",
+ "leo_gateway.multipart.dont_abort_cleanup",
+ [
+  {datatype, {enum, [true, false]}},
+  {default, false}
+ ]}.
+
 
 %% --------------------------------------------------------------------
 %% GATEWAY - CACHE

apps/leo_gateway/src/leo_gateway_app.erl

@@ -624,6 +624,10 @@ get_options() ->
                            false
                    end,
 
+    %% Retrieve multipart-upload-related properties
+    MultipartProp = ?env_multipart_properties(),
+    DontAbortCleanup = leo_misc:get_value('dont_abort_cleanup', MultipartProp, ?DEF_DONT_ABORT_CLEANUP),
+
     %% Retrieve large-object-related properties:
     LargeObjectProp = ?env_large_object_properties(),
     MaxChunkedObjs = leo_misc:get_value('max_chunked_objs', LargeObjectProp, ?DEF_LOBJ_MAX_CHUNKED_OBJS),
@@ -667,6 +671,7 @@ get_options() ->
                                 cachable_content_type = CachableContentTypes1,
                                 cachable_path_pattern = CachablePathPatterns1,
                                 has_disk_cache = HasDiskCache,
+                                dont_abort_cleanup = DontAbortCleanup,
                                 max_chunked_objs = MaxChunkedObjs,
                                 max_len_of_obj = MaxObjLen,
                                 chunked_obj_len = ChunkedObjLen,
@@ -694,6 +699,7 @@ get_options() ->
     ?info("start/3", "cache_max_content_len: ~p", [CacheMaxContentLen]),
     ?info("start/3", "cacheable_content_types: ~p", [CachableContentTypes]),
     ?info("start/3", "cacheable_path_patterns: ~p", [CachablePathPatterns]),
+    ?info("start/3", "dont_abort_cleanup: ~p", [DontAbortCleanup]),
     ?info("start/3", "max_chunked_obj: ~p", [MaxChunkedObjs]),
     ?info("start/3", "max_len_of_obj: ~p", [MaxObjLen]),
     ?info("start/3", "chunked_obj_len: ~p", [ChunkedObjLen]),

apps/leo_gateway/src/leo_gateway_s3_api.erl

@@ -814,6 +814,7 @@ handle_1(Req, [{NumOfMinLayers, NumOfMaxLayers},
                                   sending_chunked_obj_len = Props#http_options.sending_chunked_obj_len,
                                   reading_chunked_obj_len = Props#http_options.reading_chunked_obj_len,
                                   threshold_of_chunk_len = Props#http_options.threshold_of_chunk_len,
+                                  dont_abort_cleanup = Props#http_options.dont_abort_cleanup,
                                   begin_time = BeginTime}),
     case ReqParams of
         {error, metadata_too_large} ->
@@ -970,9 +971,24 @@ handle_2({ok,_AccessKeyId}, Req, ?HTTP_PUT,_Key,
         end,
     {ok, Req_2, State};
 
+%% For Multipart upload - Abort
+%% Nothing but removing a temporary object to deal with https://github.com/leo-project/leofs/issues/903
 handle_2({ok,_AccessKeyId}, Req, ?HTTP_DELETE,_Key,
          #req_params{path = Path,
+                     dont_abort_cleanup = true,
                      upload_id = UploadId}, State) when UploadId /= <<>> ->
+    TemporaryKey = << Path/binary, ?STR_NEWLINE, UploadId/binary >>,
+    _ = leo_gateway_rpc_handler:delete(TemporaryKey),
+    {ok, Req_2} = ?reply_no_content([?SERVER_HEADER], Req),
+    {ok, Req_2, State};
+%% For Multipart upload - Abort
+%% Removing all related objects: default behavior
+handle_2({ok,_AccessKeyId}, Req, ?HTTP_DELETE,_Key,
+         #req_params{path = Path,
+                     dont_abort_cleanup = false,
+                     upload_id = UploadId}, State) when UploadId /= <<>> ->
+
+
     {ok, Req_2} =
         case abort_multipart_upload(Path) of
             ok ->