From d9aefc32dc746608daf689d3b7d1005e189daea6 Mon Sep 17 00:00:00 2001 From: Wilson Li Date: Mon, 8 Jun 2015 10:38:27 +0900 Subject: [PATCH 01/10] Changed to reply 200 for range_get --- src/leo_gateway_http_commons.erl | 72 ++++++++++++++++++-------------- 1 file changed, 41 insertions(+), 31 deletions(-) diff --git a/src/leo_gateway_http_commons.erl b/src/leo_gateway_http_commons.erl index a467c8b..8844ac9 100644 --- a/src/leo_gateway_http_commons.erl +++ b/src/leo_gateway_http_commons.erl @@ -740,28 +740,34 @@ get_range_object(Req, Bucket, Key, {_Unit, Range}) when is_list(Range) -> Mime = leo_mime:guess_mime(Key), Header = [?SERVER_HEADER, {?HTTP_HEAD_RESP_CONTENT_TYPE, Mime}], - {ok, Req2} = cowboy_req:chunked_reply(?HTTP_ST_PARTIAL_CONTENT, Header, Req), - get_range_object_1(Req2, Bucket, Key, Range, undefined). + Req2 = cowboy_req:set_resp_body_fun( + fun(Socket, Transport) -> + get_range_object_1(Req, Bucket, Key, Range, undefined, Socket, Transport) + end, Req), + ?reply_ok(Header, Req2). +% {ok, Req2} = cowboy_req:chunked_reply(?HTTP_ST_PARTIAL_CONTENT, Header, Req), +% get_range_object_1(Req2, Bucket, Key, Range, undefined). -get_range_object_1(Req,_Bucket,_Key, [], _) -> + +get_range_object_1(Req,_Bucket,_Key, [], _, _Socket, _Transport) -> {ok, Req}; -get_range_object_1(Req,_Bucket,_Key, _, {error, _}) -> +get_range_object_1(Req,_Bucket,_Key, _, {error, _}, _Socket, _Transport) -> {ok, Req}; -get_range_object_1(Req, Bucket, Key, [{Start, infinity}|Rest], _) -> - Ret = get_range_object_2(Req, Bucket, Key, Start, 0), - get_range_object_1(Req, Bucket, Key, Rest, Ret); -get_range_object_1(Req, Bucket, Key, [{Start, End}|Rest], _) -> - Ret = get_range_object_2(Req, Bucket, Key, Start, End), - get_range_object_1(Req, Bucket, Key, Rest, Ret); -get_range_object_1(Req, Bucket, Key, [End|Rest], _) -> - Ret = get_range_object_2(Req, Bucket, Key, 0, End), - get_range_object_1(Req, Bucket, Key, Rest, Ret). - -get_range_object_2(Req, Bucket, Key, Start, End) -> +get_range_object_1(Req, Bucket, Key, [{Start, infinity}|Rest], _, Socket, Transport) -> + Ret = get_range_object_2(Req, Bucket, Key, Start, 0, Socket, Transport), + get_range_object_1(Req, Bucket, Key, Rest, Ret, Socket, Transport); +get_range_object_1(Req, Bucket, Key, [{Start, End}|Rest], _, Socket, Transport) -> + Ret = get_range_object_2(Req, Bucket, Key, Start, End, Socket, Transport), + get_range_object_1(Req, Bucket, Key, Rest, Ret, Socket, Transport); +get_range_object_1(Req, Bucket, Key, [End|Rest], _, Socket, Transport) -> + Ret = get_range_object_2(Req, Bucket, Key, 0, End, Socket, Transport), + get_range_object_1(Req, Bucket, Key, Rest, Ret, Socket, Transport). + +get_range_object_2(Req, Bucket, Key, Start, End, Socket, Transport) -> case leo_gateway_rpc_handler:head(Key) of {ok, #?METADATA{del = 0, cnumber = 0}} -> - get_range_object_small(Req, Bucket, Key, Start, End); + get_range_object_small(Req, Bucket, Key, Start, End, Socket, Transport); {ok, #?METADATA{del = 0, cnumber = N, dsize = ObjectSize, @@ -784,7 +790,8 @@ get_range_object_2(Req, Bucket, Key, Start, End) -> {CurPos, Index} end, get_range_object_large(Req, Bucket, Key, - NewStartPos, NewEndPos, N, Index_1, CurPos_1); + NewStartPos, NewEndPos, N, Index_1, CurPos_1, + Socket, Transport); {error, unavailable} -> ?reply_service_unavailable_error([?SERVER_HEADER], Key, <<>>, Req); _ -> @@ -794,14 +801,15 @@ get_range_object_2(Req, Bucket, Key, Start, End) -> %% @doc Retrieve the small object %% @private -get_range_object_small(Req, Bucket, Key, Start, End) -> +get_range_object_small(Req, Bucket, Key, Start, End, Socket, Transport) -> case leo_gateway_rpc_handler:get(Key, Start, End) of {ok, _Meta, <<>>} -> ?access_log_get(Bucket, Key, 0, ?HTTP_ST_OK), ok; {ok, _Meta, Bin} -> ?access_log_get(Bucket, Key, byte_size(Bin), ?HTTP_ST_OK), - cowboy_req:chunk(Bin, Req); + %cowboy_req:chunk(Bin, Req); + Transport:send(Socket, Bin); {error, unavailable} -> ?reply_service_unavailable_error([?SERVER_HEADER], Key, <<>>, Req); {error, Cause} -> @@ -830,11 +838,11 @@ calc_pos(StartPos, EndPos, _ObjectSize) -> %% @doc Retrieve the large object %% @private -get_range_object_large(_Req,_Bucket,_Key,_Start,_End, Total, Total, CurPos) -> +get_range_object_large(_Req,_Bucket,_Key,_Start,_End, Total, Total, CurPos, _Socket, _Transport) -> {ok, CurPos}; -get_range_object_large(_Req,_Bucket,_Key,_Start, End,_Total,_Index, CurPos) when CurPos > End -> +get_range_object_large(_Req,_Bucket,_Key,_Start, End,_Total,_Index, CurPos, _Socket, _Transport) when CurPos > End -> {ok, CurPos}; -get_range_object_large( Req, Bucket, Key, Start, End, Total, Index, CurPos) -> +get_range_object_large( Req, Bucket, Key, Start, End, Total, Index, CurPos, Socket, Transport) -> IndexBin = list_to_binary(integer_to_list(Index + 1)), Key2 = << Key/binary, ?DEF_SEPARATOR/binary, IndexBin/binary >>, @@ -842,13 +850,13 @@ get_range_object_large( Req, Bucket, Key, Start, End, Total, Index, CurPos) -> {ok, #?METADATA{cnumber = 0, dsize = CS}} -> %% get and chunk an object - NewPos = send_chunk(Req, Bucket, Key2, Start, End, CurPos, CS), - get_range_object_large(Req, Bucket, Key, Start, End, Total, Index + 1, NewPos); + NewPos = send_chunk(Req, Bucket, Key2, Start, End, CurPos, CS, Socket, Transport), + get_range_object_large(Req, Bucket, Key, Start, End, Total, Index + 1, NewPos, Socket, Transport); {ok, #?METADATA{cnumber = GrandChildNum}} -> - case get_range_object_large(Req, Bucket, Key2, Start, End, GrandChildNum, 0, CurPos) of + case get_range_object_large(Req, Bucket, Key2, Start, End, GrandChildNum, 0, CurPos, Socket, Transport) of {ok, NewPos} -> - get_range_object_large(Req, Bucket, Key, Start, End, Total, Index + 1, NewPos); + get_range_object_large(Req, Bucket, Key, Start, End, Total, Index + 1, NewPos, Socket, Transport); {error, Cause} -> {error, Cause} end; @@ -860,22 +868,23 @@ get_range_object_large( Req, Bucket, Key, Start, End, Total, Index, CurPos) -> %% @doc %% @private -send_chunk(_Req,_,_Key, Start,_End, CurPos, ChunkSize) when (CurPos + ChunkSize - 1) < Start -> +send_chunk(_Req,_,_Key, Start,_End, CurPos, ChunkSize, _Socket, _Transport) when (CurPos + ChunkSize - 1) < Start -> %% skip proc CurPos + ChunkSize; -send_chunk(Req,_Bucket, Key, Start, End, CurPos, ChunkSize) when CurPos >= Start andalso +send_chunk(_Req,_Bucket, Key, Start, End, CurPos, ChunkSize, Socket, Transport) when CurPos >= Start andalso (CurPos + ChunkSize - 1) =< End -> %% whole get case leo_gateway_rpc_handler:get(Key) of {ok, _Meta, Bin} -> %% @FIXME current impl can't handle a file which consist of grand children %% ?access_log_get(Bucket, Key, ChunkSize, ?HTTP_ST_OK), - cowboy_req:chunk(Bin, Req), + %cowboy_req:chunk(Bin, Req), + Transport:send(Socket, Bin), CurPos + ChunkSize; Error -> Error end; -send_chunk(Req, _Bucket, Key, Start, End, CurPos, ChunkSize) -> +send_chunk(_Req, _Bucket, Key, Start, End, CurPos, ChunkSize, Socket, Transport) -> %% partial get StartPos = case Start =< CurPos of true -> 0; @@ -891,7 +900,8 @@ send_chunk(Req, _Bucket, Key, Start, End, CurPos, ChunkSize) -> {ok, _Meta, Bin} -> %% @FIXME current impl can't handle a file which consist of grand childs %% ?access_log_get(Bucket, Key, ChunkSize, ?HTTP_ST_OK), - cowboy_req:chunk(Bin, Req), + %cowboy_req:chunk(Bin, Req), + Transport:send(Socket, Bin), CurPos + ChunkSize; {error, Cause} -> {error, Cause} From 22f1c2b21f8b3c17aaa319b4c3af063461857b0b Mon Sep 17 00:00:00 2001 From: Wilson Li Date: Mon, 8 Jun 2015 12:17:26 +0900 Subject: [PATCH 02/10] Added Content-Length --- src/leo_gateway_http_commons.erl | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/src/leo_gateway_http_commons.erl b/src/leo_gateway_http_commons.erl index 8844ac9..b8e4377 100644 --- a/src/leo_gateway_http_commons.erl +++ b/src/leo_gateway_http_commons.erl @@ -740,14 +740,34 @@ get_range_object(Req, Bucket, Key, {_Unit, Range}) when is_list(Range) -> Mime = leo_mime:guess_mime(Key), Header = [?SERVER_HEADER, {?HTTP_HEAD_RESP_CONTENT_TYPE, Mime}], + Length = get_body_length(Req, Key, Range), Req2 = cowboy_req:set_resp_body_fun( + Length, fun(Socket, Transport) -> get_range_object_1(Req, Bucket, Key, Range, undefined, Socket, Transport) - end, Req), + end, + Req), ?reply_ok(Header, Req2). % {ok, Req2} = cowboy_req:chunked_reply(?HTTP_ST_PARTIAL_CONTENT, Header, Req), % get_range_object_1(Req2, Bucket, Key, Range, undefined). +get_body_length(Req, Key, Range) -> + get_body_length(Req, Key, Range, 0). + +get_body_length(_Req, _Key, [], Acc) -> + Acc; +get_body_length(Req, Key, [{Start, infinity}|Rest], Acc) -> + case leo_gateway_rpc_handler:head(Key) of + {ok, #?METADATA{dsize = ObjectSize}} -> + get_body_length(Req, Key, Rest, Acc + ObjectSize - Start); + {_, _} -> + ?reply_service_unavailable_error([?SERVER_HEADER], Key, <<>>, Req), + error + end; +get_body_length(Req, Key, [{Start, End}|Rest], Acc) -> + get_body_length(Req, Key, Rest, Acc + End - Start + 1); +get_body_length(Req, Key, [End|Rest], Acc) -> + get_body_length(Req, Key, Rest, Acc + End + 1). get_range_object_1(Req,_Bucket,_Key, [], _, _Socket, _Transport) -> {ok, Req}; From b3adfacc0f3b527b745bb04eea56156df63e0905 Mon Sep 17 00:00:00 2001 From: Wilson Li Date: Mon, 8 Jun 2015 15:27:24 +0900 Subject: [PATCH 03/10] Code Refactoring --- src/leo_gateway_http_commons.erl | 56 +++++++++++++++----------------- 1 file changed, 27 insertions(+), 29 deletions(-) diff --git a/src/leo_gateway_http_commons.erl b/src/leo_gateway_http_commons.erl index b8e4377..055b3e8 100644 --- a/src/leo_gateway_http_commons.erl +++ b/src/leo_gateway_http_commons.erl @@ -738,36 +738,37 @@ get_range_object(Req, Bucket, Key, {error, badarg}) -> ?reply_bad_range([?SERVER_HEADER], Key, <<>>, Req); get_range_object(Req, Bucket, Key, {_Unit, Range}) when is_list(Range) -> Mime = leo_mime:guess_mime(Key), - Header = [?SERVER_HEADER, - {?HTTP_HEAD_RESP_CONTENT_TYPE, Mime}], - Length = get_body_length(Req, Key, Range), - Req2 = cowboy_req:set_resp_body_fun( - Length, - fun(Socket, Transport) -> - get_range_object_1(Req, Bucket, Key, Range, undefined, Socket, Transport) - end, - Req), - ?reply_ok(Header, Req2). -% {ok, Req2} = cowboy_req:chunked_reply(?HTTP_ST_PARTIAL_CONTENT, Header, Req), -% get_range_object_1(Req2, Bucket, Key, Range, undefined). - -get_body_length(Req, Key, Range) -> - get_body_length(Req, Key, Range, 0). - -get_body_length(_Req, _Key, [], Acc) -> - Acc; -get_body_length(Req, Key, [{Start, infinity}|Rest], Acc) -> + case get_body_length_1(Key, Range) of + error -> + ?reply_service_unavailable_error([?SERVER_HEADER], Key, <<>>, Req); + Length -> + Header = [?SERVER_HEADER, + {?HTTP_HEAD_RESP_CONTENT_TYPE, Mime}, + {?HTTP_HEAD_RESP_CONTENT_LENGTH,integer_to_list(Length)}], + Req2 = cowboy_req:set_resp_body_fun( + fun(Socket, Transport) -> + get_range_object_1(Req, Bucket, Key, Range, undefined, Socket, Transport) + end, + Req), + ?reply_ok(Header, Req2) + end. + +get_body_length_1(Key, Range) -> case leo_gateway_rpc_handler:head(Key) of {ok, #?METADATA{dsize = ObjectSize}} -> - get_body_length(Req, Key, Rest, Acc + ObjectSize - Start); + get_body_length(Range, ObjectSize, 0); {_, _} -> - ?reply_service_unavailable_error([?SERVER_HEADER], Key, <<>>, Req), error - end; -get_body_length(Req, Key, [{Start, End}|Rest], Acc) -> - get_body_length(Req, Key, Rest, Acc + End - Start + 1); -get_body_length(Req, Key, [End|Rest], Acc) -> - get_body_length(Req, Key, Rest, Acc + End + 1). + end. + +get_body_length([], _ObjectSize, Acc) -> + Acc; +get_body_length([{Start, infinity}|Rest], ObjectSize, Acc) -> + get_body_length(Rest, ObjectSize, Acc + ObjectSize - Start); +get_body_length([{Start, End}|Rest], ObjectSize, Acc) when End < ObjectSize -> + get_body_length(Rest, ObjectSize, Acc + End - Start + 1); +get_body_length([End|Rest], ObjectSize, Acc) when End < ObjectSize-> + get_body_length(Rest, ObjectSize, Acc + End + 1). get_range_object_1(Req,_Bucket,_Key, [], _, _Socket, _Transport) -> {ok, Req}; @@ -828,7 +829,6 @@ get_range_object_small(Req, Bucket, Key, Start, End, Socket, Transport) -> ok; {ok, _Meta, Bin} -> ?access_log_get(Bucket, Key, byte_size(Bin), ?HTTP_ST_OK), - %cowboy_req:chunk(Bin, Req); Transport:send(Socket, Bin); {error, unavailable} -> ?reply_service_unavailable_error([?SERVER_HEADER], Key, <<>>, Req); @@ -898,7 +898,6 @@ send_chunk(_Req,_Bucket, Key, Start, End, CurPos, ChunkSize, Socket, Transport) {ok, _Meta, Bin} -> %% @FIXME current impl can't handle a file which consist of grand children %% ?access_log_get(Bucket, Key, ChunkSize, ?HTTP_ST_OK), - %cowboy_req:chunk(Bin, Req), Transport:send(Socket, Bin), CurPos + ChunkSize; Error -> @@ -920,7 +919,6 @@ send_chunk(_Req, _Bucket, Key, Start, End, CurPos, ChunkSize, Socket, Transport) {ok, _Meta, Bin} -> %% @FIXME current impl can't handle a file which consist of grand childs %% ?access_log_get(Bucket, Key, ChunkSize, ?HTTP_ST_OK), - %cowboy_req:chunk(Bin, Req), Transport:send(Socket, Bin), CurPos + ChunkSize; {error, Cause} -> From fd5d13f1468c5659cde24e3a12603ab291fbc70b Mon Sep 17 00:00:00 2001 From: Wilson Li Date: Tue, 9 Jun 2015 12:33:36 +0900 Subject: [PATCH 04/10] Bug fix for End < 0 --- src/leo_gateway_http_commons.erl | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/leo_gateway_http_commons.erl b/src/leo_gateway_http_commons.erl index 055b3e8..9db81df 100644 --- a/src/leo_gateway_http_commons.erl +++ b/src/leo_gateway_http_commons.erl @@ -765,10 +765,16 @@ get_body_length([], _ObjectSize, Acc) -> Acc; get_body_length([{Start, infinity}|Rest], ObjectSize, Acc) -> get_body_length(Rest, ObjectSize, Acc + ObjectSize - Start); +get_body_length([{Start, End}|Rest], ObjectSize, Acc) when End < 0 -> + get_body_length(Rest, ObjectSize, Acc + ObjectSize - Start); get_body_length([{Start, End}|Rest], ObjectSize, Acc) when End < ObjectSize -> get_body_length(Rest, ObjectSize, Acc + End - Start + 1); -get_body_length([End|Rest], ObjectSize, Acc) when End < ObjectSize-> - get_body_length(Rest, ObjectSize, Acc + End + 1). +get_body_length([End|Rest], ObjectSize, Acc) when End < 0 -> + get_body_length(Rest, ObjectSize, Acc + ObjectSize); +get_body_length([End|Rest], ObjectSize, Acc) when End < ObjectSize -> + get_body_length(Rest, ObjectSize, Acc + End + 1); +get_body_length(_, _, _) -> + error. get_range_object_1(Req,_Bucket,_Key, [], _, _Socket, _Transport) -> {ok, Req}; From 789478f870a761f300c272d395ef2bab87b71ba4 Mon Sep 17 00:00:00 2001 From: Wilson Li Date: Tue, 9 Jun 2015 12:33:44 +0900 Subject: [PATCH 05/10] Updates Unit Test --- test/leo_gateway_web_tests.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/leo_gateway_web_tests.erl b/test/leo_gateway_web_tests.erl index 70f9fe7..3f64680 100644 --- a/test/leo_gateway_web_tests.erl +++ b/test/leo_gateway_web_tests.erl @@ -660,7 +660,7 @@ range_object_base([_TermFun, _Node0, Node1], RangeValue) -> ?TARGET_HOST, ":8080/a/b.png"]), [{"connection", "close"},{"range", RangeValue}]}, [], []), - ?assertEqual(206, SC), + ?assertEqual(200, SC), ?assertEqual("od", Body) catch throw:Reason -> From d52c9b250356a0307d4c21c2ca3d8806ea32d44d Mon Sep 17 00:00:00 2001 From: Wilson Li Date: Wed, 10 Jun 2015 14:39:02 +0900 Subject: [PATCH 06/10] Fixed Reply HTTP 206, categorized error types --- src/leo_gateway_http_commons.erl | 20 ++++++++++++-------- test/leo_gateway_web_tests.erl | 2 +- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/leo_gateway_http_commons.erl b/src/leo_gateway_http_commons.erl index 9db81df..745bada 100644 --- a/src/leo_gateway_http_commons.erl +++ b/src/leo_gateway_http_commons.erl @@ -739,9 +739,7 @@ get_range_object(Req, Bucket, Key, {error, badarg}) -> get_range_object(Req, Bucket, Key, {_Unit, Range}) when is_list(Range) -> Mime = leo_mime:guess_mime(Key), case get_body_length_1(Key, Range) of - error -> - ?reply_service_unavailable_error([?SERVER_HEADER], Key, <<>>, Req); - Length -> + {ok, Length} -> Header = [?SERVER_HEADER, {?HTTP_HEAD_RESP_CONTENT_TYPE, Mime}, {?HTTP_HEAD_RESP_CONTENT_LENGTH,integer_to_list(Length)}], @@ -750,19 +748,25 @@ get_range_object(Req, Bucket, Key, {_Unit, Range}) when is_list(Range) -> get_range_object_1(Req, Bucket, Key, Range, undefined, Socket, Transport) end, Req), - ?reply_ok(Header, Req2) + ?reply_partial_content(Header, Req2); + {error, bad_range} -> + ?reply_bad_range([?SERVER_HEADER], Key, <<>>, Req); + {error, unavailable} -> + ?reply_service_unavailable_error([?SERVER_HEADER], Key, <<>>, Req); + _ -> + ?reply_not_found([?SERVER_HEADER], Key, <<>>, Req) end. get_body_length_1(Key, Range) -> case leo_gateway_rpc_handler:head(Key) of {ok, #?METADATA{dsize = ObjectSize}} -> get_body_length(Range, ObjectSize, 0); - {_, _} -> - error + {error, Reason} -> + {error, Reason} end. get_body_length([], _ObjectSize, Acc) -> - Acc; + {ok, Acc}; get_body_length([{Start, infinity}|Rest], ObjectSize, Acc) -> get_body_length(Rest, ObjectSize, Acc + ObjectSize - Start); get_body_length([{Start, End}|Rest], ObjectSize, Acc) when End < 0 -> @@ -774,7 +778,7 @@ get_body_length([End|Rest], ObjectSize, Acc) when End < 0 -> get_body_length([End|Rest], ObjectSize, Acc) when End < ObjectSize -> get_body_length(Rest, ObjectSize, Acc + End + 1); get_body_length(_, _, _) -> - error. + {error, bad_range}. get_range_object_1(Req,_Bucket,_Key, [], _, _Socket, _Transport) -> {ok, Req}; diff --git a/test/leo_gateway_web_tests.erl b/test/leo_gateway_web_tests.erl index 3f64680..70f9fe7 100644 --- a/test/leo_gateway_web_tests.erl +++ b/test/leo_gateway_web_tests.erl @@ -660,7 +660,7 @@ range_object_base([_TermFun, _Node0, Node1], RangeValue) -> ?TARGET_HOST, ":8080/a/b.png"]), [{"connection", "close"},{"range", RangeValue}]}, [], []), - ?assertEqual(200, SC), + ?assertEqual(206, SC), ?assertEqual("od", Body) catch throw:Reason -> From 6d85afd244baa5d734e18a14a700bc182704720d Mon Sep 17 00:00:00 2001 From: Wilson Li Date: Wed, 10 Jun 2015 15:14:00 +0900 Subject: [PATCH 07/10] Added Error Handling to Range Get --- src/leo_gateway_http_commons.erl | 56 ++++++++++++++++++++++---------- 1 file changed, 39 insertions(+), 17 deletions(-) diff --git a/src/leo_gateway_http_commons.erl b/src/leo_gateway_http_commons.erl index 745bada..70bb3f9 100644 --- a/src/leo_gateway_http_commons.erl +++ b/src/leo_gateway_http_commons.erl @@ -753,8 +753,12 @@ get_range_object(Req, Bucket, Key, {_Unit, Range}) when is_list(Range) -> ?reply_bad_range([?SERVER_HEADER], Key, <<>>, Req); {error, unavailable} -> ?reply_service_unavailable_error([?SERVER_HEADER], Key, <<>>, Req); + {error, timeout} -> + ?reply_timeout_without_body([?SERVER_HEADER], Req); + {error, ?ERR_TYPE_INTERNAL_ERROR} -> + ?reply_internal_error_without_body([?SERVER_HEADER], Req); _ -> - ?reply_not_found([?SERVER_HEADER], Key, <<>>, Req) + ?reply_not_found_without_body([?SERVER_HEADER], Req) end. get_body_length_1(Key, Range) -> @@ -780,10 +784,19 @@ get_body_length([End|Rest], ObjectSize, Acc) when End < ObjectSize -> get_body_length(_, _, _) -> {error, bad_range}. +get_range_object_1(_Req, _Bucket, _Key, _, {error, _Reason}, Socket, Transport) -> + Transport:close(Socket); +% Transport:close(Socket), +% case Reason of +% unavailable -> +% ?reply_service_unavailable_error([?SERVER_HEADER], Key, <<>>, Req); +% not_found -> +% ?reply_not_found([?SERVER_HEADER], Key, <<>>, Req); +% _ -> +% ?reply_internal_error_without_body([?SERVER_HEADER], Req) +% end; get_range_object_1(Req,_Bucket,_Key, [], _, _Socket, _Transport) -> {ok, Req}; -get_range_object_1(Req,_Bucket,_Key, _, {error, _}, _Socket, _Transport) -> - {ok, Req}; get_range_object_1(Req, Bucket, Key, [{Start, infinity}|Rest], _, Socket, Transport) -> Ret = get_range_object_2(Req, Bucket, Key, Start, 0, Socket, Transport), get_range_object_1(Req, Bucket, Key, Rest, Ret, Socket, Transport); @@ -823,25 +836,26 @@ get_range_object_2(Req, Bucket, Key, Start, End, Socket, Transport) -> get_range_object_large(Req, Bucket, Key, NewStartPos, NewEndPos, N, Index_1, CurPos_1, Socket, Transport); - {error, unavailable} -> - ?reply_service_unavailable_error([?SERVER_HEADER], Key, <<>>, Req); - _ -> - {error, not_found} + Error -> + Error end. %% @doc Retrieve the small object %% @private -get_range_object_small(Req, Bucket, Key, Start, End, Socket, Transport) -> +get_range_object_small(_Req, Bucket, Key, Start, End, Socket, Transport) -> case leo_gateway_rpc_handler:get(Key, Start, End) of {ok, _Meta, <<>>} -> ?access_log_get(Bucket, Key, 0, ?HTTP_ST_OK), ok; {ok, _Meta, Bin} -> ?access_log_get(Bucket, Key, byte_size(Bin), ?HTTP_ST_OK), - Transport:send(Socket, Bin); - {error, unavailable} -> - ?reply_service_unavailable_error([?SERVER_HEADER], Key, <<>>, Req); + case Transport:send(Socket, Bin) of + ok -> + ok; + {error, Cause} -> + {error, Cause} + end; {error, Cause} -> {error, Cause} end. @@ -868,6 +882,8 @@ calc_pos(StartPos, EndPos, _ObjectSize) -> %% @doc Retrieve the large object %% @private +get_range_object_large(_Req,_Bucket,_Key,_Start,_End, _Total, _Index, {error, _} = Error, _Socket, _Transport) -> + Error; get_range_object_large(_Req,_Bucket,_Key,_Start,_End, Total, Total, CurPos, _Socket, _Transport) -> {ok, CurPos}; get_range_object_large(_Req,_Bucket,_Key,_Start, End,_Total,_Index, CurPos, _Socket, _Transport) when CurPos > End -> @@ -890,8 +906,6 @@ get_range_object_large( Req, Bucket, Key, Start, End, Total, Index, CurPos, Sock {error, Cause} -> {error, Cause} end; - {error, unavailable} -> - ?reply_service_unavailable_error([?SERVER_HEADER], Key, <<>>, Req); {error, Cause} -> {error, Cause} end. @@ -908,8 +922,12 @@ send_chunk(_Req,_Bucket, Key, Start, End, CurPos, ChunkSize, Socket, Transport) {ok, _Meta, Bin} -> %% @FIXME current impl can't handle a file which consist of grand children %% ?access_log_get(Bucket, Key, ChunkSize, ?HTTP_ST_OK), - Transport:send(Socket, Bin), - CurPos + ChunkSize; + case Transport:send(Socket, Bin) of + ok -> + CurPos + ChunkSize; + {error, Cause} -> + {error, Cause} + end; Error -> Error end; @@ -929,8 +947,12 @@ send_chunk(_Req, _Bucket, Key, Start, End, CurPos, ChunkSize, Socket, Transport) {ok, _Meta, Bin} -> %% @FIXME current impl can't handle a file which consist of grand childs %% ?access_log_get(Bucket, Key, ChunkSize, ?HTTP_ST_OK), - Transport:send(Socket, Bin), - CurPos + ChunkSize; + case Transport:send(Socket, Bin) of + ok -> + CurPos + ChunkSize; + {error, Cause} -> + {error, Cause} + end; {error, Cause} -> {error, Cause} end. From 33a3407dce8dafa9069b2002b0dcc16b2406abb5 Mon Sep 17 00:00:00 2001 From: Wilson Li Date: Wed, 10 Jun 2015 16:53:31 +0900 Subject: [PATCH 08/10] Fix for R16B03 --- src/leo_gateway_http_commons.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/leo_gateway_http_commons.erl b/src/leo_gateway_http_commons.erl index 70bb3f9..43c3a2c 100644 --- a/src/leo_gateway_http_commons.erl +++ b/src/leo_gateway_http_commons.erl @@ -744,6 +744,7 @@ get_range_object(Req, Bucket, Key, {_Unit, Range}) when is_list(Range) -> {?HTTP_HEAD_RESP_CONTENT_TYPE, Mime}, {?HTTP_HEAD_RESP_CONTENT_LENGTH,integer_to_list(Length)}], Req2 = cowboy_req:set_resp_body_fun( + Length, fun(Socket, Transport) -> get_range_object_1(Req, Bucket, Key, Range, undefined, Socket, Transport) end, From c6aaa663f5ca791c4552a25fb3e2f13f16a286f2 Mon Sep 17 00:00:00 2001 From: Wilson Li Date: Mon, 15 Jun 2015 12:42:34 +0900 Subject: [PATCH 09/10] Use set_resp_body_fun/3 for Content Length, Require Patch to cowboy --- src/leo_gateway_http_commons.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/leo_gateway_http_commons.erl b/src/leo_gateway_http_commons.erl index 43c3a2c..15c8d16 100644 --- a/src/leo_gateway_http_commons.erl +++ b/src/leo_gateway_http_commons.erl @@ -741,8 +741,8 @@ get_range_object(Req, Bucket, Key, {_Unit, Range}) when is_list(Range) -> case get_body_length_1(Key, Range) of {ok, Length} -> Header = [?SERVER_HEADER, - {?HTTP_HEAD_RESP_CONTENT_TYPE, Mime}, - {?HTTP_HEAD_RESP_CONTENT_LENGTH,integer_to_list(Length)}], + {?HTTP_HEAD_RESP_CONTENT_TYPE, Mime} + ], Req2 = cowboy_req:set_resp_body_fun( Length, fun(Socket, Transport) -> From 794856ca6d68eb27ddd765fc1744bee2cbb88664 Mon Sep 17 00:00:00 2001 From: Wilson Li Date: Mon, 15 Jun 2015 17:47:45 +0900 Subject: [PATCH 10/10] Switched to use forked cowboy --- rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config b/rebar.config index 2c8800f..565c1f7 100644 --- a/rebar.config +++ b/rebar.config @@ -34,7 +34,7 @@ {savanna_agent, ".*", {git, "https://github.com/leo-project/savanna_agent.git", {tag, "0.4.8"}}}, {erpcgen, ".*", {git, "https://github.com/leo-project/erpcgen.git", {tag, "0.2.3"}}}, {nfs_rpc_server, ".*", {git, "https://github.com/leo-project/nfs_rpc_server.git", {tag, "0.2.3"}}}, - {cowboy, ".*", {git, "https://github.com/ninenines/cowboy.git", {tag, "1.0.0"}}}, + {cowboy, ".*", {git, "https://github.com/leo-project/cowboy.git", {branch, "feature/fix-issue-376"}}}, {cowlib, ".*", {git, "https://github.com/ninenines/cowlib.git", {tag, "1.0.0"}}}, {meck, ".*", {git, "https://github.com/eproxus/meck.git", {tag, "0.8.2"}}}, {proper, ".*", {git, "https://github.com/manopapad/proper.git", "7d535ac61e9c2a9a7616af4615c39595a3c5739f"}}