From 63e63ec524e779dd2ed689a7e95b0d2607645a41 Mon Sep 17 00:00:00 2001 From: Dominik Mostowiec Date: Tue, 11 Mar 2014 22:16:18 +0000 Subject: [PATCH 1/4] first version of sign_encode_uri flag and dynamic aws_access_key, aws_secret_key variables --- ngx_http_aws_auth.c | 130 ++++++++++++++++++++++++++++++++++++++--- tests/nginx_sampleconf | 23 ++++---- tests/test.py | 6 +- 3 files changed, 137 insertions(+), 22 deletions(-) diff --git a/ngx_http_aws_auth.c b/ngx_http_aws_auth.c index d8fcf43..1ded1fa 100644 --- a/ngx_http_aws_auth.c +++ b/ngx_http_aws_auth.c @@ -16,6 +16,10 @@ static void* ngx_http_aws_auth_create_loc_conf(ngx_conf_t *cf); static char* ngx_http_aws_auth_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child); static ngx_int_t register_variable(ngx_conf_t *cf); static char * +ngx_http_aws_auth_set_access_key(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); +static char * +ngx_http_aws_auth_set_secret(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); +static char * ngx_http_aws_auth_set_s3_bucket(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); static char * ngx_http_aws_auth_set_chop_prefix(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); @@ -31,6 +35,9 @@ typedef struct { ngx_str_t s3_bucket; ngx_str_t chop_prefix; ngx_uint_t add_date; + ngx_flag_t encode_uri; + ngx_http_aws_auth_script_t *access_key_script; + ngx_http_aws_auth_script_t *secret_script; ngx_http_aws_auth_script_t *s3_bucket_script; ngx_http_aws_auth_script_t *chop_prefix_script; } ngx_http_aws_auth_conf_t; @@ -65,14 +72,14 @@ static const char *signed_subresources[] = { static ngx_command_t ngx_http_aws_auth_commands[] = { { ngx_string("aws_access_key"), NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, - ngx_conf_set_str_slot, + ngx_http_aws_auth_set_access_key, NGX_HTTP_LOC_CONF_OFFSET, offsetof(ngx_http_aws_auth_conf_t, access_key), NULL }, { ngx_string("aws_secret_key"), NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, - ngx_conf_set_str_slot, + ngx_http_aws_auth_set_secret, NGX_HTTP_LOC_CONF_OFFSET, offsetof(ngx_http_aws_auth_conf_t, secret), NULL }, @@ -91,6 +98,13 @@ static ngx_command_t ngx_http_aws_auth_commands[] = { offsetof(ngx_http_aws_auth_conf_t, chop_prefix), NULL }, + { ngx_string("sign_encode_uri"), + NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG, + ngx_conf_set_flag_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_aws_auth_conf_t, encode_uri), + NULL }, + ngx_null_command }; @@ -124,6 +138,84 @@ ngx_module_t ngx_http_aws_auth_module = { NGX_MODULE_V1_PADDING }; +static char * +ngx_http_aws_auth_set_access_key(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) +{ + ngx_http_aws_auth_conf_t *aws_conf = conf; + ngx_http_script_compile_t sc; + ngx_str_t *value; + ngx_uint_t n; + value = cf->args->elts; + n = ngx_http_script_variables_count(&value[1]); + + if (n == 0) { + // set access_key as string + aws_conf->access_key.data = value[1].data; + aws_conf->access_key.len = value[1].len; + } else { + //add script to compile + aws_conf->access_key_script = ngx_pcalloc(cf->pool, sizeof(ngx_http_aws_auth_script_t)); + if (aws_conf->access_key_script == NULL) { + return NGX_CONF_ERROR; + } + + ngx_memzero(&sc, sizeof(ngx_http_script_compile_t)); + + sc.cf = cf; + sc.source = &value[1]; + sc.lengths = &aws_conf->access_key_script->lengths; + sc.values = &aws_conf->access_key_script->values; + sc.variables = n; + sc.complete_lengths = 1; + sc.complete_values = 1; + + if (ngx_http_script_compile(&sc) != NGX_OK) { + return NGX_CONF_ERROR; + } + } + return NGX_CONF_OK; +} + +static char * +ngx_http_aws_auth_set_secret(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) +{ + ngx_http_aws_auth_conf_t *aws_conf = conf; + ngx_http_script_compile_t sc; + ngx_str_t *value; + ngx_uint_t n; + value = cf->args->elts; + n = ngx_http_script_variables_count(&value[1]); + + if (n == 0) { + // set secret as string + aws_conf->secret.data = value[1].data; + aws_conf->secret.len = value[1].len; + } else { + //add script to compile + aws_conf->secret_script = ngx_pcalloc(cf->pool, sizeof(ngx_http_aws_auth_script_t)); + if (aws_conf->secret_script == NULL) { + return NGX_CONF_ERROR; + } + + ngx_memzero(&sc, sizeof(ngx_http_script_compile_t)); + + sc.cf = cf; + sc.source = &value[1]; + sc.lengths = &aws_conf->secret_script->lengths; + sc.values = &aws_conf->secret_script->values; + sc.variables = n; + sc.complete_lengths = 1; + sc.complete_values = 1; + + if (ngx_http_script_compile(&sc) != NGX_OK) { + return NGX_CONF_ERROR; + } + } + return NGX_CONF_OK; +} + + + static char * ngx_http_aws_auth_set_s3_bucket(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) { @@ -210,7 +302,7 @@ ngx_http_aws_auth_create_loc_conf(ngx_conf_t *cf) if (conf == NULL) { return NGX_CONF_ERROR; } - + conf->encode_uri = NGX_CONF_UNSET; return conf; } @@ -226,6 +318,7 @@ ngx_http_aws_auth_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) ngx_conf_merge_str_value(conf->secret, prev->secret, ""); ngx_conf_merge_str_value(conf->chop_prefix, prev->chop_prefix, ""); ngx_conf_merge_uint_value(conf->add_date, prev->add_date, 0); + ngx_conf_merge_value(conf->encode_uri, prev->encode_uri, 1); return NGX_CONF_OK; } @@ -374,9 +467,18 @@ ngx_http_aws_auth_get_canon_resource(ngx_http_request_t *r, ngx_str_t *retstr) { ngx_http_aws_auth_conf_t *aws_conf; int uri_len; aws_conf = ngx_http_get_module_loc_conf(r, ngx_http_aws_auth_module); - u_char *uri = ngx_palloc(r->pool, r->uri.len * 3 + 1); // allow room for escaping - u_char *uri_end = (u_char*) ngx_escape_uri(uri,r->uri.data, r->uri.len, NGX_ESCAPE_URI); - *uri_end = '\0'; // null terminate + u_char *uri, *uri_end; + if (aws_conf->encode_uri) { + ngx_log_error(NGX_LOG_DEBUG, r->connection->log, 0, "sign_encode_uri on: %d", aws_conf->encode_uri); + uri = ngx_palloc(r->pool, r->uri.len * 3 + 1); // allow room for escaping + uri_end = (u_char*) ngx_escape_uri(uri,r->uri.data, r->uri.len, NGX_ESCAPE_URI); + *uri_end = '\0'; // null terminate + } else { + ngx_log_error(NGX_LOG_DEBUG, r->connection->log, 0, "sign_encode_uri off: %d", aws_conf->encode_uri); + uri = ngx_palloc(r->pool, r->uri.len + 1); + ngx_memcpy(uri, r->uri.data, r->uri.len); + *(uri+r->uri.len) = '\0'; // null terminate + } if (aws_conf->chop_prefix.len > 0) { if (!ngx_strncmp(r->uri.data, aws_conf->chop_prefix.data, aws_conf->chop_prefix.len)) { @@ -440,10 +542,24 @@ ngx_http_aws_auth_get_canon_resource(ngx_http_request_t *r, ngx_str_t *retstr) { static ngx_int_t ngx_http_aws_auth_get_dynamic_variables(ngx_http_request_t *r){ /* - * Get value for s3_bucket and chop_prefix + * Get value for access_key, secret, s3_bucket and chop_prefix */ ngx_http_aws_auth_conf_t *aws_conf; aws_conf = ngx_http_get_module_loc_conf(r, ngx_http_aws_auth_module); + if (aws_conf->access_key_script != NULL){ + if (ngx_http_script_run(r, &aws_conf->access_key, aws_conf->access_key_script->lengths->elts, 1, + aws_conf->access_key_script->values->elts) == NULL) { + return NGX_ERROR; + } + aws_conf->access_key.len = aws_conf->access_key.len -1; + } + if (aws_conf->secret_script != NULL){ + if (ngx_http_script_run(r, &aws_conf->secret, aws_conf->secret_script->lengths->elts, 1, + aws_conf->secret_script->values->elts) == NULL) { + return NGX_ERROR; + } + aws_conf->secret.len = aws_conf->secret.len -1; + } if (aws_conf->s3_bucket_script != NULL){ if (ngx_http_script_run(r, &aws_conf->s3_bucket, aws_conf->s3_bucket_script->lengths->elts, 1, aws_conf->s3_bucket_script->values->elts) == NULL) { diff --git a/tests/nginx_sampleconf b/tests/nginx_sampleconf index 2ea0f68..f3c29d4 100644 --- a/tests/nginx_sampleconf +++ b/tests/nginx_sampleconf @@ -1,22 +1,19 @@ server { listen 8000; + + #location ~ /rewrite/(.*)$ { + # rewrite .* /test1/$1 last; + #} + set $key 4WLAD43EZZ64EPK1CIRO; + set $secret uGA3yy/NJqITgERIVmr9AgUZRBqUjPADvfQoxpKL; location /test1 { - proxy_pass http://precise64; - aws_access_key 4WLAD43EZZ64EPK1CIRO; - aws_secret_key uGA3yy/NJqITgERIVmr9AgUZRBqUjPADvfQoxpKL; - s3_bucket test1; - chop_prefix /test1; - proxy_set_header Authorization $s3_auth_token; - } - location / { - proxy_pass http://precise64/test1/; - aws_access_key 4WLAD43EZZ64EPK1CIRO; - aws_secret_key uGA3yy/NJqITgERIVmr9AgUZRBqUjPADvfQoxpKL; + aws_access_key $key; + aws_secret_key $secret; + #sign_encode_uri off; s3_bucket test1; chop_prefix /test1; proxy_set_header Authorization $s3_auth_token; - proxy_set_header x-amz-date $aws_date; + proxy_pass http://precise64; } - } diff --git a/tests/test.py b/tests/test.py index 6c298a4..2bb20b6 100755 --- a/tests/test.py +++ b/tests/test.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +# -*- coding: utf-8 -*- import unittest from unittest import TestCase @@ -12,6 +12,7 @@ #'port': 80, 'access_key':'4WLAD43EZZ64EPK1CIRO', 'secret_key':'uGA3yy/NJqITgERIVmr9AgUZRBqUjPADvfQoxpKL', + #'bucket': 'rewrite', 'bucket': 'test1', } @@ -157,7 +158,8 @@ def test_multipart_upload(self): class BotoTest(TestCase): def setUp(self): self.boto_tester = Tester(s3_cred['host'], s3_cred['port'], s3_cred['access_key'], - s3_cred['secret_key'], s3_cred['bucket'], 'filename.txt', 'filecontentttttt', 'text/html', U_M_LIMIT + 100) + s3_cred['secret_key'], s3_cred['bucket'], "filenamefile.txt", 'filecontentttttt', 'text/html', U_M_LIMIT + 100) + #s3_cred['secret_key'], s3_cred['bucket'], "filename_ żźćŻŹĆŁÓĄaą.txt", 'filecontentttttt', 'text/html', U_M_LIMIT + 100) #def test_create_bucket(self): # self.assertEquals(self.boto_tester.create_bucket(), True) From d2de6d700ff26beb772811cc5afa6bfecee0fe78 Mon Sep 17 00:00:00 2001 From: Dominik Mostowiec Date: Thu, 13 Mar 2014 11:48:10 +0000 Subject: [PATCH 2/4] refactor dynamic variables functions --- ngx_http_aws_auth.c | 136 ++++++++--------------------------------- tests/nginx_sampleconf | 1 + 2 files changed, 27 insertions(+), 110 deletions(-) diff --git a/ngx_http_aws_auth.c b/ngx_http_aws_auth.c index 1ded1fa..e7b39e7 100644 --- a/ngx_http_aws_auth.c +++ b/ngx_http_aws_auth.c @@ -15,6 +15,14 @@ static const EVP_MD* evp_md = NULL; static void* ngx_http_aws_auth_create_loc_conf(ngx_conf_t *cf); static char* ngx_http_aws_auth_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child); static ngx_int_t register_variable(ngx_conf_t *cf); + +typedef struct { + ngx_array_t *lengths; + ngx_array_t *values; +} ngx_http_aws_auth_script_t; + +static char * +ngx_http_aws_auth_set_dynamic_variable(ngx_conf_t *cf, ngx_command_t *cmd, ngx_str_t *val, ngx_http_aws_auth_script_t **script); static char * ngx_http_aws_auth_set_access_key(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); static char * @@ -24,11 +32,6 @@ ngx_http_aws_auth_set_s3_bucket(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); static char * ngx_http_aws_auth_set_chop_prefix(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); -typedef struct { - ngx_array_t *lengths; - ngx_array_t *values; -} ngx_http_aws_auth_script_t; - typedef struct { ngx_str_t access_key; ngx_str_t secret; @@ -139,9 +142,8 @@ ngx_module_t ngx_http_aws_auth_module = { }; static char * -ngx_http_aws_auth_set_access_key(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) +ngx_http_aws_auth_set_dynamic_variable(ngx_conf_t *cf, ngx_command_t *cmd, ngx_str_t *retval, ngx_http_aws_auth_script_t **val_script) { - ngx_http_aws_auth_conf_t *aws_conf = conf; ngx_http_script_compile_t sc; ngx_str_t *value; ngx_uint_t n; @@ -149,13 +151,13 @@ ngx_http_aws_auth_set_access_key(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) n = ngx_http_script_variables_count(&value[1]); if (n == 0) { - // set access_key as string - aws_conf->access_key.data = value[1].data; - aws_conf->access_key.len = value[1].len; + // set as string + retval->data = value[1].data; + retval->len = value[1].len; } else { //add script to compile - aws_conf->access_key_script = ngx_pcalloc(cf->pool, sizeof(ngx_http_aws_auth_script_t)); - if (aws_conf->access_key_script == NULL) { + (*val_script) = ngx_pcalloc(cf->pool, sizeof(ngx_http_aws_auth_script_t)); + if (val_script == NULL) { return NGX_CONF_ERROR; } @@ -163,8 +165,8 @@ ngx_http_aws_auth_set_access_key(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) sc.cf = cf; sc.source = &value[1]; - sc.lengths = &aws_conf->access_key_script->lengths; - sc.values = &aws_conf->access_key_script->values; + sc.lengths = &((*val_script)->lengths); + sc.values = &((*val_script)->values); sc.variables = n; sc.complete_lengths = 1; sc.complete_values = 1; @@ -177,41 +179,17 @@ ngx_http_aws_auth_set_access_key(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) } static char * -ngx_http_aws_auth_set_secret(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) +ngx_http_aws_auth_set_access_key(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) { ngx_http_aws_auth_conf_t *aws_conf = conf; - ngx_http_script_compile_t sc; - ngx_str_t *value; - ngx_uint_t n; - value = cf->args->elts; - n = ngx_http_script_variables_count(&value[1]); - - if (n == 0) { - // set secret as string - aws_conf->secret.data = value[1].data; - aws_conf->secret.len = value[1].len; - } else { - //add script to compile - aws_conf->secret_script = ngx_pcalloc(cf->pool, sizeof(ngx_http_aws_auth_script_t)); - if (aws_conf->secret_script == NULL) { - return NGX_CONF_ERROR; - } - - ngx_memzero(&sc, sizeof(ngx_http_script_compile_t)); - - sc.cf = cf; - sc.source = &value[1]; - sc.lengths = &aws_conf->secret_script->lengths; - sc.values = &aws_conf->secret_script->values; - sc.variables = n; - sc.complete_lengths = 1; - sc.complete_values = 1; + return ngx_http_aws_auth_set_dynamic_variable(cf, cmd, &(aws_conf->access_key), &aws_conf->access_key_script); +} - if (ngx_http_script_compile(&sc) != NGX_OK) { - return NGX_CONF_ERROR; - } - } - return NGX_CONF_OK; +static char * +ngx_http_aws_auth_set_secret(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) +{ + ngx_http_aws_auth_conf_t *aws_conf = conf; + return ngx_http_aws_auth_set_dynamic_variable(cf, cmd, &(aws_conf->secret), &aws_conf->secret_script); } @@ -220,76 +198,14 @@ static char * ngx_http_aws_auth_set_s3_bucket(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) { ngx_http_aws_auth_conf_t *aws_conf = conf; - ngx_http_script_compile_t sc; - ngx_str_t *value; - ngx_uint_t n; - value = cf->args->elts; - n = ngx_http_script_variables_count(&value[1]); - - if (n == 0) { - // set s3_bucket as string - aws_conf->s3_bucket.data = value[1].data; - aws_conf->s3_bucket.len = value[1].len; - } else { - //add script to compile - aws_conf->s3_bucket_script = ngx_pcalloc(cf->pool, sizeof(ngx_http_aws_auth_script_t)); - if (aws_conf->s3_bucket_script == NULL) { - return NGX_CONF_ERROR; - } - - ngx_memzero(&sc, sizeof(ngx_http_script_compile_t)); - - sc.cf = cf; - sc.source = &value[1]; - sc.lengths = &aws_conf->s3_bucket_script->lengths; - sc.values = &aws_conf->s3_bucket_script->values; - sc.variables = n; - sc.complete_lengths = 1; - sc.complete_values = 1; - - if (ngx_http_script_compile(&sc) != NGX_OK) { - return NGX_CONF_ERROR; - } - } - return NGX_CONF_OK; + return ngx_http_aws_auth_set_dynamic_variable(cf, cmd, &(aws_conf->s3_bucket), &aws_conf->s3_bucket_script); } static char * ngx_http_aws_auth_set_chop_prefix(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) { ngx_http_aws_auth_conf_t *aws_conf = conf; - ngx_http_script_compile_t sc; - ngx_str_t *value; - ngx_uint_t n; - value = cf->args->elts; - n = ngx_http_script_variables_count(&value[1]); - - if (n == 0) { - // set chop_prefix as string - aws_conf->chop_prefix.data = value[1].data; - aws_conf->chop_prefix.len = value[1].len; - } else { - //add script to compile - aws_conf->chop_prefix_script = ngx_pcalloc(cf->pool, sizeof(ngx_http_aws_auth_script_t)); - if (aws_conf->chop_prefix_script == NULL) { - return NGX_CONF_ERROR; - } - - ngx_memzero(&sc, sizeof(ngx_http_script_compile_t)); - - sc.cf = cf; - sc.source = &value[1]; - sc.lengths = &aws_conf->chop_prefix_script->lengths; - sc.values = &aws_conf->chop_prefix_script->values; - sc.variables = n; - sc.complete_lengths = 1; - sc.complete_values = 1; - - if (ngx_http_script_compile(&sc) != NGX_OK) { - return NGX_CONF_ERROR; - } - } - return NGX_CONF_OK; + return ngx_http_aws_auth_set_dynamic_variable(cf, cmd, &(aws_conf->chop_prefix), &aws_conf->chop_prefix_script); } diff --git a/tests/nginx_sampleconf b/tests/nginx_sampleconf index f3c29d4..3505387 100644 --- a/tests/nginx_sampleconf +++ b/tests/nginx_sampleconf @@ -8,6 +8,7 @@ server { set $secret uGA3yy/NJqITgERIVmr9AgUZRBqUjPADvfQoxpKL; location /test1 { aws_access_key $key; + #aws_access_key 4WLAD43EZZ64EPK1CIRO; aws_secret_key $secret; #sign_encode_uri off; s3_bucket test1; From f57ad5eb4e1fee16d86e67e98c0befd75e0029e1 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 14 Oct 2014 17:15:55 +0200 Subject: [PATCH 3/4] disable output variable cache, possible second sign calculation --- ngx_http_aws_auth.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ngx_http_aws_auth.c b/ngx_http_aws_auth.c index e7b39e7..0c164ff 100644 --- a/ngx_http_aws_auth.c +++ b/ngx_http_aws_auth.c @@ -650,7 +650,7 @@ ngx_http_aws_auth_variable_s3(ngx_http_request_t *r, ngx_http_variable_value_t * v->len = ngx_strlen(signature); v->data = signature; v->valid = 1; - v->no_cacheable = 0; + v->no_cacheable = 1; v->not_found = 0; return NGX_OK; } @@ -663,7 +663,7 @@ ngx_http_aws_auth_variable_date(ngx_http_request_t *r, ngx_http_variable_value_t v->len = ngx_cached_http_time.len; v->data = ngx_cached_http_time.data; v->valid = 1; - v->no_cacheable = 0; + v->no_cacheable = 1; v->not_found = 0; aws_conf = ngx_http_get_module_loc_conf(r, ngx_http_aws_auth_module); //get this varialbe enable add x-amz-date to sign From e32b802dc3a27dec1852691cf3952b57205c3746 Mon Sep 17 00:00:00 2001 From: Dominik Mostowiec Date: Mon, 8 Jun 2015 13:22:32 +0200 Subject: [PATCH 4/4] ngx_http_arg2 fix --- ngx_http_aws_auth.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ngx_http_aws_auth.c b/ngx_http_aws_auth.c index 0c164ff..fb04a01 100644 --- a/ngx_http_aws_auth.c +++ b/ngx_http_aws_auth.c @@ -360,7 +360,7 @@ ngx_http_arg2(ngx_http_request_t *r, u_char *name, size_t len, ngx_str_t *value) if (ngx_strncasecmp(p, name, len) != 0) { continue; } - if (p == r->args.data || *(p - 1) == '&' || (p + len) == last || *(p + len) == '&' || *(p + len) == '=') { + if ((p == r->args.data || *(p - 1) == '&') && ((p + len) == last || *(p + len) == '&' || *(p + len) == '=')) { if ((p + len) < last && *(p + len) == '=') { value->data = p + len + 1; p = ngx_strlchr(p, last, '&');