Support AWS Signature Version 4 (Refactored) #2
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -34,20 +34,26 @@ | |
| -include("leo_s3_user.hrl"). | ||
| -include_lib("eunit/include/eunit.hrl"). | ||
| -include_lib("stdlib/include/qlc.hrl"). | ||
| -include_lib("leo_logger/include/leo_logger.hrl"). | ||
|
|
||
| -export([start/2, | ||
| create_table/2, put/1, bulk_put/1, | ||
| update_providers/1, | ||
| create_key/1, get_credential/1, has_credential/1, has_credential/2, | ||
| authenticate/3, get_signature/3, | ||
| find_all/0, checksum/0 | ||
| ]). | ||
|
|
||
|
|
||
| -record(sign_v4_params, {credential :: binary(), | ||
| signature :: binary(), | ||
| signed_headers :: binary() | ||
| }). | ||
| -record(auth_params, {access_key_id :: binary(), | ||
| secret_access_key :: binary(), | ||
| signature :: binary(), | ||
| sign_params :: #sign_params{}, | ||
| sign_v4_params :: #sign_v4_params{}, | ||
| auth_info :: #auth_info{} | ||
| }). | ||
|
|
||
|
|
@@ -237,28 +243,50 @@ has_credential(MasterNodes, AccessKey) -> | |
| %% @doc Authenticate | ||
| %% | ||
| -spec(authenticate(Authorization, SignParams, IsCreateBucketOp) -> | ||
| {ok, binary(), binary()} | {error, any()} when Authorization::binary(), | ||
| SignParams::#sign_params{}, | ||
| IsCreateBucketOp::boolean()). | ||
| authenticate(Authorization, #sign_params{sign_ver = SignVer} = SignParams, IsCreateBucketOp) -> | ||
| {AccessKeyId, Signature, SignV4Params} = | ||
| case SignVer of | ||
| v4 -> | ||
| [<<"AWS4", _Method/binary>>, Params] = binary:split(Authorization, <<" ">>), | ||
| ParamList = binary:split(Params, <<",">>, [global]), | ||
| SignV4Params2 = extract_v4_params(ParamList), | ||
| [AccessKeyId2|_] = binary:split(SignV4Params2#sign_v4_params.credential, <<"/">>), | ||
| Signature2 = SignV4Params2#sign_v4_params.signature, | ||
| {AccessKeyId2, Signature2, SignV4Params2}; | ||
| _ -> | ||
| [AccWithAWS,Signature2|_] = binary:split(Authorization, <<":">>), | ||
| <<"AWS ", AccessKeyId2/binary>> = AccWithAWS, | ||
| SignV4Params2 = #sign_v4_params{}, | ||
| {AccessKeyId2, Signature2, SignV4Params2} | ||
| end, | ||
| % ?debug("authenticate/3", "Access Key: ~p, Signature: ~p, Ver: ~p", [AccessKeyId, Signature, SignVer]), | ||
| % ?debug("authenticate/3", "Access Key: ~p, Signature: ~p, SignParams: ~p, SignV4Params: ~p", [AccessKeyId, Signature, SignParams, SignV4Params]), | ||
| authenticate_0(AccessKeyId, Signature, SignParams, SignV4Params, IsCreateBucketOp). | ||
|
|
||
| authenticate_0(AccessKeyId, Signature, #sign_params{bucket = <<>>} = SignParams, SignV4Params, _IsCreateBucketOp) -> | ||
| authenticate_1(#auth_params{access_key_id = AccessKeyId, | ||
| signature = Signature, | ||
| sign_params = SignParams, | ||
| sign_v4_params = SignV4Params | ||
| }); | ||
|
|
||
| authenticate_0(AccessKeyId, Signature, #sign_params{bucket = Bucket} = SignParams, SignV4Params, IsCreateBucketOp) -> | ||
| case {leo_s3_bucket:head(AccessKeyId, Bucket), IsCreateBucketOp} of | ||
| {ok, false} -> | ||
| authenticate_1(#auth_params{access_key_id = AccessKeyId, | ||
| signature = Signature, | ||
| sign_params = SignParams#sign_params{bucket = Bucket}, | ||
| sign_v4_params = SignV4Params | ||
| }); | ||
| {not_found, true} -> | ||
| authenticate_1(#auth_params{access_key_id = AccessKeyId, | ||
| signature = Signature, | ||
| sign_params = SignParams#sign_params{bucket = Bucket}, | ||
| sign_v4_params = SignV4Params | ||
| }); | ||
| _Other -> | ||
| {error, unmatch} | ||
| end. | ||
|
|
@@ -287,11 +315,79 @@ authenticate(Authorization, #sign_params{bucket = Bucket} = SignParams, IsCreate | |
| <<"response-content-disposition">>, | ||
| <<"response-content-encoding">>]). | ||
|
|
||
| -spec(get_signature(SecretAccessKey, SignParams, SignV4Params) -> | ||
| binary() when SecretAccessKey::binary(), | ||
| SignParams::#sign_params{}, | ||
| SignV4Params::#sign_v4_params{}). | ||
| get_signature(SecretAccessKey, SignParams, SignV4Params) -> | ||
| % ?debug("get_signature/3", "Key: ~p, Sign: ~p, SignV4: ~p", [SecretAccessKey, SignParams, SignV4Params]), | ||
| case SignParams#sign_params.sign_ver of | ||
| v4 -> | ||
| get_signature_v4(SecretAccessKey, SignParams, SignV4Params); | ||
| _ -> | ||
| {get_signature_v2(SecretAccessKey, SignParams), <<>>, <<>>} | ||
| end. | ||
|
|
||
|
There was a problem hiding this comment. In this case, function pattern match is more better than case branching. get_signature(SecretAccessKey, #sign_params{sign_ver = v4} = SignParams, SignV4Params) ->
get_signature_v4(SecretAccessKey, SignParams, SignV4Params);
get_signature(SecretAccessKey, #sign_params{sign_ver = v2} = SignParams, SignV4Params) ->
{get_signature_v2(SecretAccessKey, SignParams), <<>>, <<>>}. |
||
| %% @doc Get AWS signature version 4 | ||
| %% @private | ||
| get_signature_v4(SecretAccessKey, SignParams, SignV4Params) -> | ||
| #sign_params{http_verb = HTTPVerb, | ||
| date = Date, | ||
| raw_uri = URI, | ||
| query_str = QueryStr, | ||
| headers = Headers | ||
| } = SignParams, | ||
| #sign_v4_params{credential = Credential, | ||
| signed_headers = SignedHeaders | ||
| } = SignV4Params, | ||
| Header_1 = auth_v4_headers(Headers, SignedHeaders), | ||
| Hash_2 = case lists:keyfind(<<"x-amz-content-sha256">>, 1, Headers) of | ||
| false -> | ||
| <<"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855">>; | ||
| {_, Hash_1} -> | ||
| Hash_1 | ||
| end, | ||
|
|
||
| QueryStr_1 = auth_v4_qs(QueryStr), | ||
|
|
||
| Request_1 = <<HTTPVerb/binary, "\n", | ||
| URI/binary, "\n", | ||
| QueryStr_1/binary, "\n", | ||
| Header_1/binary, "\n", | ||
| SignedHeaders/binary, "\n", | ||
| Hash_2/binary>>, | ||
|
|
||
| % ?debug("get_signature_v4/3", "Request: ~p", [Request_1]), | ||
| RequestHash = crypto:hash(sha256, Request_1), | ||
|
|
||
| Date_1 = auth_v4_date(Date, Headers), | ||
| [_AWSAccessKeyId, Date_2, Region, Service, <<"aws4_request">>] = binary:split(Credential, <<"/">>, [global]), | ||
|
|
||
| Scope = <<Date_2/binary, "/", Region/binary, "/", Service/binary, "/aws4_request">>, | ||
|
|
||
| RequestBin = leo_hex:binary_to_hexbin(RequestHash), | ||
|
|
||
| BinToSignHead = <<Date_1/binary, "\n", | ||
| Scope/binary, "\n">>, | ||
| BinToSign = <<"AWS4-HMAC-SHA256\n", | ||
| BinToSignHead/binary, | ||
| RequestBin/binary>>, | ||
|
|
||
| % ?debug("get_signature_v4/3", "BinToSign: ~p", [BinToSign]), | ||
|
|
||
| DateKey = crypto:hmac(sha256, <<"AWS4", SecretAccessKey/binary>>, Date_2), | ||
| DateRegionKey = crypto:hmac(sha256, DateKey, Region), | ||
| DateRegionServiceKey = crypto:hmac(sha256, DateRegionKey, Service), | ||
| SigningKey = crypto:hmac(sha256, DateRegionServiceKey, <<"aws4_request">>), | ||
|
|
||
| Signature = crypto:hmac(sha256, SigningKey, BinToSign), | ||
| SignatureBin = leo_hex:binary_to_hexbin(Signature), | ||
| % ?debug("get_signature_v4/3", "Signature: ~p", [SignatureBin]), | ||
| {SignatureBin, BinToSignHead, SigningKey}. | ||
|
|
||
| %% @doc Get AWS signature version 2 | ||
| %% @private | ||
| get_signature_v2(SecretAccessKey, SignParams) -> | ||
| #sign_params{http_verb = HTTPVerb, | ||
| content_md5 = ETag, | ||
| content_type = ContentType, | ||
|
|
@@ -359,6 +455,65 @@ setup(DB, Provider) -> | |
| true = ets:insert(?AUTH_INFO, {1, #auth_info{db = DB, | ||
| provider = Provider}}), | ||
| ok. | ||
| %% @doc Trim Space From Binary | ||
| %% @private | ||
| trim(<<>>) -> | ||
| <<>>; | ||
| trim(Bin) -> | ||
| trim(Bin, byte_size(Bin)). | ||
|
|
||
| trim(Bin = <<Byte:1/binary, Tail/binary>>, Size) -> | ||
| case is_space(Byte) of | ||
| true -> | ||
| trim(Tail, Size - 1); | ||
| false -> | ||
| trim_tail(Bin, Size) | ||
| end. | ||
|
|
||
| trim_tail(Bin, Size) -> | ||
| SizeMinus1 = Size - 1, | ||
| <<Rest:SizeMinus1/binary, Byte:1/binary>> = Bin, | ||
| case is_space(Byte) of | ||
| true -> | ||
| trim_tail(Rest, Size - 1); | ||
| false -> | ||
| Bin | ||
| end. | ||
|
|
||
| is_space(<<" ">>) -> | ||
| true; | ||
| is_space(_) -> | ||
| false. | ||
|
There was a problem hiding this comment. There are some implementations to trim binary with Erlang. https://gist.github.com/mocchira/0b637ac0d66389492363 The second one is slightly faster than yours in my env(ec2, 8GB, 4core). There was a problem hiding this comment. and also since this function triming binary is reusable, |
||
|
|
||
| %% @doc Extract Signature V4 Params to Record | ||
| %% @private | ||
| -spec(extract_v4_params(SignV4Params) -> | ||
| {ok, #sign_v4_params{}} | {error, any()} when SignV4Params :: list()). | ||
| extract_v4_params(ParamList) -> | ||
| extract_v4_params(ParamList, #sign_v4_params{}). | ||
|
|
||
| extract_v4_params([], #sign_v4_params{} = SignV4Params) -> | ||
| SignV4Params; | ||
| extract_v4_params([Head|Rest], #sign_v4_params{} = SignV4Params) -> | ||
| [Key2, Val2|_] = binary:split(Head, <<"=">>), | ||
| Key = trim(Key2), | ||
| Val = trim(Val2), | ||
|
|
||
| SignV4Params2 = | ||
| case Key of | ||
| <<"Credential">> -> | ||
| SignV4Params#sign_v4_params{ | ||
| credential = Val}; | ||
| <<"Signature">> -> | ||
| SignV4Params#sign_v4_params{ | ||
| signature = Val}; | ||
| <<"SignedHeaders">> -> | ||
| SignV4Params#sign_v4_params{ | ||
| signed_headers = Val}; | ||
| _ -> | ||
| SignV4Params | ||
| end, | ||
| extract_v4_params(Rest, SignV4Params2). | ||
|
|
||
|
|
||
| %% @doc Authenticate#1 | ||
|
|
@@ -397,16 +552,18 @@ authenticate_2(AuthParams) -> | |
| %% @doc Authenticate#3 | ||
| %% @private | ||
| -spec(authenticate_3(AuthParams) -> | ||
| {ok, binary(), binary()} | {error, any()} when AuthParams::#auth_params{}). | ||
| authenticate_3(#auth_params{secret_access_key = SecretAccessKey, | ||
| access_key_id = AccessKeyId, | ||
| signature = Signature, | ||
| sign_params = SignParams, | ||
| sign_v4_params = SignV4Params | ||
| }) -> | ||
| %% ?debugVal({Signature, SignParams, SignV4Params}), | ||
| case get_signature(SecretAccessKey, SignParams, SignV4Params) of | ||
| {Signature, _SignHead, _SignKey} = Ret -> | ||
| {ok, AccessKeyId, Ret}; | ||
| {WrongSig, _, _} -> | ||
| error_logger:error_msg("~p,~p,~p,~p~n", | ||
| [{module, ?MODULE_STRING}, {function, "authenticate_3/1"}, | ||
| {line, ?LINE}, {body, WrongSig}]), | ||
|
|
@@ -460,6 +617,54 @@ get_auth_info() -> | |
| not_found | ||
| end. | ||
|
|
||
| %% @doc Construct Canonical Headers | ||
| %% @private | ||
| auth_v4_headers(Headers, SignedHeaders) -> | ||
| HeaderList = binary:split(SignedHeaders, <<";">>, [global]), | ||
| auth_v4_headers(Headers, HeaderList, <<>>). | ||
|
|
||
| auth_v4_headers(_Headers, [], Acc) -> | ||
| Acc; | ||
| auth_v4_headers(Headers, [Head|Rest], Acc) -> | ||
| Val = case lists:keyfind(Head, 1, Headers) of | ||
| false -> | ||
| <<>>; | ||
| {_, Bin} -> | ||
| trim(Bin) | ||
| end, | ||
| auth_v4_headers(Headers, Rest, <<Acc/binary, Head/binary, ":", Val/binary, "\n">>). | ||
|
|
||
| %% @doc Consutrct Canonical Query String | ||
| %% @private | ||
| auth_v4_qs(QueryStr) -> | ||
| List = cow_qs:parse_qs(QueryStr), | ||
| lists:foldl(fun({Key, Val}, Acc) -> | ||
| KeyBin = cow_qs:urlencode(Key), | ||
| ValBin = case Val of | ||
| true -> | ||
| <<>>; | ||
| _ -> | ||
| cow_qs:urlencode(Val) | ||
| end, | ||
| case Acc of | ||
| <<>> -> | ||
| <<KeyBin/binary, "=", ValBin/binary>>; | ||
| _ -> | ||
| <<Acc/binary, "&", KeyBin/binary, "=", ValBin/binary>> | ||
| end | ||
| end, <<>>, List). | ||
| %% @doc Retrieve date V4 | ||
| %% | ||
| -spec(auth_v4_date(Date, Headers) -> | ||
| binary() when Date::binary(), | ||
| Headers::list()). | ||
| auth_v4_date(Date, Headers) -> | ||
| case lists:keyfind(<<"x-amz-date">>, 1, Headers) of | ||
| false -> | ||
| Date; | ||
| {<<"x-amz-date">>, Date_2} -> | ||
| Date_2 | ||
| end. | ||
|
|
||
| %% @doc Retrieve date | ||
| %% @private | ||
|
|
||
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
s/binary()/list()/
and basically you need to dialyze for type checking before PL.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
? These parameters look binaries to me.
BTW, I will check how to use dialyzer
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
yep.
you are right.
At first,
issue the below command only once in the repo root. ( basically when you cloned into local )
issue the below command if you want to dialyze. ( basically when you finished to implement before UT )