From df50dc0feba87a40c029f96a7b5e17bb9aba648c Mon Sep 17 00:00:00 2001 From: AlmogBaku Date: Tue, 16 Aug 2016 18:43:43 +0300 Subject: [PATCH 1/5] Improve ES Sink: - support AWS authentication for ES servers - print errors if exists - change elastic library to the proper url(using v.3) - fixes ES Sink URI doesn't support single node config(only with ?nodes) - bring back the go-extpoint that was required by the doc --- Godeps/Godeps.json | 26 +- Makefile | 3 +- common/elasticsearch/elasticsearch.go | 51 ++- common/elasticsearch/elasticsearch_test.go | 2 +- docs/sink-configuration.md | 35 +- events/sinks/elasticsearch/driver.go | 13 +- events/sinks/elasticsearch/driver_test.go | 2 +- metrics/sinks/elasticsearch/driver.go | 16 +- metrics/sinks/elasticsearch/driver_test.go | 2 +- vendor/github.com/gedex/inflector/.travis.yml | 3 + .../gedex/inflector/CakePHP_LICENSE.txt | 28 ++ vendor/github.com/gedex/inflector/LICENSE.md | 29 ++ vendor/github.com/gedex/inflector/README.md | 32 ++ .../github.com/gedex/inflector/inflector.go | 355 ++++++++++++++++++ .../progrium/go-extpoints/.gitignore | 4 + .../github.com/progrium/go-extpoints/LICENSE | 19 + .../github.com/progrium/go-extpoints/Makefile | 8 + .../progrium/go-extpoints/README.md | 287 ++++++++++++++ .../github.com/progrium/go-extpoints/main.go | 179 +++++++++ .../progrium/go-extpoints/template.go | 183 +++++++++ .../smartystreets/go-aws-auth/.gitignore | 1 + .../smartystreets/go-aws-auth/CONTRIBUTING.md | 12 + .../go-aws-auth/LICENSE.md} | 21 +- .../smartystreets/go-aws-auth/README.md | 86 +++++ .../smartystreets/go-aws-auth/awsauth.go | 220 +++++++++++ .../smartystreets/go-aws-auth/common.go | 311 +++++++++++++++ .../smartystreets/go-aws-auth/s3.go | 121 ++++++ .../smartystreets/go-aws-auth/sign2.go | 50 +++ .../smartystreets/go-aws-auth/sign3.go | 58 +++ .../smartystreets/go-aws-auth/sign4.go | 117 ++++++ .../olivere/elastic.v3}/.gitignore | 0 .../olivere/elastic.v3}/.travis.yml | 0 .../olivere/elastic.v3}/CHANGELOG-3.0.md | 0 .../olivere/elastic.v3}/CONTRIBUTING.md | 0 .../olivere/elastic.v3}/CONTRIBUTORS | 0 .../olivere/elastic.v3}/ISSUE_TEMPLATE.md | 0 .../olivere/elastic.v3}/README.md | 0 .../olivere/elastic.v3}/bulk.go | 0 .../elastic.v3}/bulk_delete_request.go | 0 .../olivere/elastic.v3}/bulk_index_request.go | 0 .../olivere/elastic.v3}/bulk_processor.go | 0 .../olivere/elastic.v3}/bulk_request.go | 0 .../elastic.v3}/bulk_update_request.go | 0 .../olivere/elastic.v3}/canonicalize.go | 0 .../olivere/elastic.v3}/clear_scroll.go | 0 .../olivere/elastic.v3}/client.go | 0 .../olivere/elastic.v3}/cluster_health.go | 0 .../olivere/elastic.v3}/cluster_state.go | 0 .../olivere/elastic.v3}/cluster_stats.go | 0 .../olivere/elastic.v3}/connection.go | 0 .../olivere/elastic.v3}/count.go | 0 .../olivere/elastic.v3}/decoder.go | 0 .../olivere/elastic.v3}/delete.go | 0 .../olivere/elastic.v3}/delete_by_query.go | 0 .../olivere/elastic.v3}/delete_template.go | 0 .../olivere/elastic.v3}/doc.go | 0 .../olivere/elastic.v3}/errors.go | 0 .../olivere/elastic.v3}/exists.go | 0 .../olivere/elastic.v3}/explain.go | 0 .../elastic.v3}/fetch_source_context.go | 0 .../olivere/elastic.v3}/field_stats.go | 0 .../olivere/elastic.v3}/geo_point.go | 0 .../olivere/elastic.v3}/get.go | 0 .../olivere/elastic.v3}/get_template.go | 0 .../olivere/elastic.v3}/highlight.go | 0 .../olivere/elastic.v3}/index.go | 0 .../olivere/elastic.v3}/indices_close.go | 0 .../olivere/elastic.v3}/indices_create.go | 0 .../olivere/elastic.v3}/indices_delete.go | 0 .../elastic.v3}/indices_delete_template.go | 0 .../elastic.v3}/indices_delete_warmer.go | 0 .../olivere/elastic.v3}/indices_exists.go | 0 .../elastic.v3}/indices_exists_template.go | 0 .../elastic.v3}/indices_exists_type.go | 0 .../olivere/elastic.v3}/indices_flush.go | 0 .../olivere/elastic.v3}/indices_forcemerge.go | 0 .../olivere/elastic.v3}/indices_get.go | 0 .../elastic.v3}/indices_get_aliases.go | 0 .../elastic.v3}/indices_get_mapping.go | 0 .../elastic.v3}/indices_get_settings.go | 0 .../elastic.v3}/indices_get_template.go | 0 .../olivere/elastic.v3}/indices_get_warmer.go | 0 .../olivere/elastic.v3}/indices_open.go | 0 .../olivere/elastic.v3}/indices_put_alias.go | 0 .../elastic.v3}/indices_put_mapping.go | 0 .../elastic.v3}/indices_put_settings.go | 0 .../elastic.v3}/indices_put_template.go | 0 .../olivere/elastic.v3}/indices_put_warmer.go | 0 .../olivere/elastic.v3}/indices_refresh.go | 0 .../olivere/elastic.v3}/indices_stats.go | 0 .../olivere/elastic.v3}/inner_hit.go | 0 .../olivere/elastic.v3}/logger.go | 0 .../olivere/elastic.v3}/mget.go | 0 .../olivere/elastic.v3}/msearch.go | 0 .../olivere/elastic.v3}/mtermvectors.go | 0 .../olivere/elastic.v3}/nodes_info.go | 0 .../olivere/elastic.v3}/optimize.go | 0 .../olivere/elastic.v3}/percolate.go | 0 .../olivere/elastic.v3}/ping.go | 0 .../olivere/elastic.v3}/plugins.go | 0 .../olivere/elastic.v3}/query.go | 0 .../olivere/elastic.v3}/reindex.go | 0 .../olivere/elastic.v3}/reindexer.go | 0 .../olivere/elastic.v3}/request.go | 0 .../olivere/elastic.v3}/rescore.go | 0 .../olivere/elastic.v3}/rescorer.go | 0 .../olivere/elastic.v3}/response.go | 0 .../olivere/elastic.v3}/scan.go | 0 .../olivere/elastic.v3}/script.go | 0 .../olivere/elastic.v3}/scroll.go | 0 .../olivere/elastic.v3}/search.go | 0 .../olivere/elastic.v3}/search_aggs.go | 0 .../search_aggs_bucket_children.go | 0 .../search_aggs_bucket_date_histogram.go | 0 .../search_aggs_bucket_date_range.go | 0 .../elastic.v3}/search_aggs_bucket_filter.go | 0 .../elastic.v3}/search_aggs_bucket_filters.go | 0 .../search_aggs_bucket_geo_distance.go | 0 .../search_aggs_bucket_geohash_grid.go | 0 .../elastic.v3}/search_aggs_bucket_global.go | 0 .../search_aggs_bucket_histogram.go | 0 .../elastic.v3}/search_aggs_bucket_missing.go | 0 .../elastic.v3}/search_aggs_bucket_nested.go | 0 .../elastic.v3}/search_aggs_bucket_range.go | 0 .../search_aggs_bucket_reverse_nested.go | 0 .../elastic.v3}/search_aggs_bucket_sampler.go | 0 .../search_aggs_bucket_significant_terms.go | 0 .../elastic.v3}/search_aggs_bucket_terms.go | 0 .../elastic.v3}/search_aggs_metrics_avg.go | 0 .../search_aggs_metrics_cardinality.go | 0 .../search_aggs_metrics_extended_stats.go | 0 .../search_aggs_metrics_geo_bounds.go | 0 .../elastic.v3}/search_aggs_metrics_max.go | 0 .../elastic.v3}/search_aggs_metrics_min.go | 0 .../search_aggs_metrics_percentile_ranks.go | 0 .../search_aggs_metrics_percentiles.go | 0 .../elastic.v3}/search_aggs_metrics_stats.go | 0 .../elastic.v3}/search_aggs_metrics_sum.go | 0 .../search_aggs_metrics_top_hits.go | 0 .../search_aggs_metrics_value_count.go | 0 .../search_aggs_pipeline_avg_bucket.go | 0 .../search_aggs_pipeline_bucket_script.go | 0 .../search_aggs_pipeline_bucket_selector.go | 0 .../search_aggs_pipeline_cumulative_sum.go | 0 .../search_aggs_pipeline_derivative.go | 0 .../search_aggs_pipeline_max_bucket.go | 0 .../search_aggs_pipeline_min_bucket.go | 0 .../search_aggs_pipeline_mov_avg.go | 0 .../search_aggs_pipeline_serial_diff.go | 0 .../search_aggs_pipeline_sum_bucket.go | 0 .../elastic.v3}/search_queries_bool.go | 0 .../elastic.v3}/search_queries_boosting.go | 0 .../search_queries_common_terms.go | 0 .../search_queries_constant_score.go | 0 .../elastic.v3}/search_queries_dis_max.go | 0 .../elastic.v3}/search_queries_exists.go | 0 .../olivere/elastic.v3}/search_queries_fsq.go | 0 .../search_queries_fsq_score_funcs.go | 0 .../elastic.v3}/search_queries_fuzzy.go | 0 .../search_queries_geo_bounding_box.go | 0 .../search_queries_geo_distance.go | 0 .../elastic.v3}/search_queries_geo_polygon.go | 0 .../elastic.v3}/search_queries_has_child.go | 0 .../elastic.v3}/search_queries_has_parent.go | 0 .../olivere/elastic.v3}/search_queries_ids.go | 0 .../elastic.v3}/search_queries_indices.go | 0 .../elastic.v3}/search_queries_match.go | 0 .../elastic.v3}/search_queries_match_all.go | 0 .../elastic.v3}/search_queries_missing.go | 0 .../search_queries_more_like_this.go | 0 .../elastic.v3}/search_queries_multi_match.go | 0 .../elastic.v3}/search_queries_nested.go | 0 .../olivere/elastic.v3}/search_queries_not.go | 0 .../elastic.v3}/search_queries_prefix.go | 0 .../search_queries_query_string.go | 0 .../elastic.v3}/search_queries_range.go | 0 .../elastic.v3}/search_queries_regexp.go | 0 .../elastic.v3}/search_queries_script.go | 0 .../search_queries_simple_query_string.go | 0 .../search_queries_template_query.go | 0 .../elastic.v3}/search_queries_term.go | 0 .../elastic.v3}/search_queries_terms.go | 0 .../elastic.v3}/search_queries_type.go | 0 .../elastic.v3}/search_queries_wildcard.go | 0 .../olivere/elastic.v3}/search_request.go | 0 .../olivere/elastic.v3}/search_source.go | 0 .../olivere/elastic.v3}/search_template.go | 0 .../olivere/elastic.v3}/sort.go | 0 .../olivere/elastic.v3}/suggest.go | 0 .../olivere/elastic.v3}/suggest_field.go | 0 .../olivere/elastic.v3}/suggester.go | 0 .../elastic.v3}/suggester_completion.go | 0 .../elastic.v3}/suggester_completion_fuzzy.go | 0 .../olivere/elastic.v3}/suggester_context.go | 0 .../elastic.v3}/suggester_context_category.go | 0 .../elastic.v3}/suggester_context_geo.go | 0 .../olivere/elastic.v3}/suggester_phrase.go | 0 .../olivere/elastic.v3}/suggester_term.go | 0 .../olivere/elastic.v3}/tasks_cancel.go | 0 .../olivere/elastic.v3}/tasks_list.go | 0 .../olivere/elastic.v3}/termvectors.go | 0 .../olivere/elastic.v3}/update.go | 0 .../olivere/elastic.v3}/update_by_query.go | 0 203 files changed, 2226 insertions(+), 48 deletions(-) create mode 100644 vendor/github.com/gedex/inflector/.travis.yml create mode 100644 vendor/github.com/gedex/inflector/CakePHP_LICENSE.txt create mode 100644 vendor/github.com/gedex/inflector/LICENSE.md create mode 100644 vendor/github.com/gedex/inflector/README.md create mode 100644 vendor/github.com/gedex/inflector/inflector.go create mode 100644 vendor/github.com/progrium/go-extpoints/.gitignore create mode 100644 vendor/github.com/progrium/go-extpoints/LICENSE create mode 100644 vendor/github.com/progrium/go-extpoints/Makefile create mode 100644 vendor/github.com/progrium/go-extpoints/README.md create mode 100644 vendor/github.com/progrium/go-extpoints/main.go create mode 100644 vendor/github.com/progrium/go-extpoints/template.go create mode 100644 vendor/github.com/smartystreets/go-aws-auth/.gitignore create mode 100644 vendor/github.com/smartystreets/go-aws-auth/CONTRIBUTING.md rename vendor/github.com/{olivere/elastic/LICENSE => smartystreets/go-aws-auth/LICENSE.md} (55%) create mode 100644 vendor/github.com/smartystreets/go-aws-auth/README.md create mode 100644 vendor/github.com/smartystreets/go-aws-auth/awsauth.go create mode 100644 vendor/github.com/smartystreets/go-aws-auth/common.go create mode 100644 vendor/github.com/smartystreets/go-aws-auth/s3.go create mode 100644 vendor/github.com/smartystreets/go-aws-auth/sign2.go create mode 100644 vendor/github.com/smartystreets/go-aws-auth/sign3.go create mode 100644 vendor/github.com/smartystreets/go-aws-auth/sign4.go rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/.gitignore (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/.travis.yml (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/CHANGELOG-3.0.md (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/CONTRIBUTING.md (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/CONTRIBUTORS (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/ISSUE_TEMPLATE.md (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/README.md (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/bulk.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/bulk_delete_request.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/bulk_index_request.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/bulk_processor.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/bulk_request.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/bulk_update_request.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/canonicalize.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/clear_scroll.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/client.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/cluster_health.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/cluster_state.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/cluster_stats.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/connection.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/count.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/decoder.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/delete.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/delete_by_query.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/delete_template.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/doc.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/errors.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/exists.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/explain.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/fetch_source_context.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/field_stats.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/geo_point.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/get.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/get_template.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/highlight.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/index.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/indices_close.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/indices_create.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/indices_delete.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/indices_delete_template.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/indices_delete_warmer.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/indices_exists.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/indices_exists_template.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/indices_exists_type.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/indices_flush.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/indices_forcemerge.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/indices_get.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/indices_get_aliases.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/indices_get_mapping.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/indices_get_settings.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/indices_get_template.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/indices_get_warmer.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/indices_open.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/indices_put_alias.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/indices_put_mapping.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/indices_put_settings.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/indices_put_template.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/indices_put_warmer.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/indices_refresh.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/indices_stats.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/inner_hit.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/logger.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/mget.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/msearch.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/mtermvectors.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/nodes_info.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/optimize.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/percolate.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/ping.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/plugins.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/query.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/reindex.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/reindexer.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/request.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/rescore.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/rescorer.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/response.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/scan.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/script.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/scroll.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_aggs.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_aggs_bucket_children.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_aggs_bucket_date_histogram.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_aggs_bucket_date_range.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_aggs_bucket_filter.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_aggs_bucket_filters.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_aggs_bucket_geo_distance.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_aggs_bucket_geohash_grid.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_aggs_bucket_global.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_aggs_bucket_histogram.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_aggs_bucket_missing.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_aggs_bucket_nested.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_aggs_bucket_range.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_aggs_bucket_reverse_nested.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_aggs_bucket_sampler.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_aggs_bucket_significant_terms.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_aggs_bucket_terms.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_aggs_metrics_avg.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_aggs_metrics_cardinality.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_aggs_metrics_extended_stats.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_aggs_metrics_geo_bounds.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_aggs_metrics_max.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_aggs_metrics_min.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_aggs_metrics_percentile_ranks.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_aggs_metrics_percentiles.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_aggs_metrics_stats.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_aggs_metrics_sum.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_aggs_metrics_top_hits.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_aggs_metrics_value_count.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_aggs_pipeline_avg_bucket.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_aggs_pipeline_bucket_script.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_aggs_pipeline_bucket_selector.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_aggs_pipeline_cumulative_sum.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_aggs_pipeline_derivative.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_aggs_pipeline_max_bucket.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_aggs_pipeline_min_bucket.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_aggs_pipeline_mov_avg.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_aggs_pipeline_serial_diff.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_aggs_pipeline_sum_bucket.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_queries_bool.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_queries_boosting.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_queries_common_terms.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_queries_constant_score.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_queries_dis_max.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_queries_exists.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_queries_fsq.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_queries_fsq_score_funcs.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_queries_fuzzy.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_queries_geo_bounding_box.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_queries_geo_distance.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_queries_geo_polygon.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_queries_has_child.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_queries_has_parent.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_queries_ids.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_queries_indices.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_queries_match.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_queries_match_all.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_queries_missing.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_queries_more_like_this.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_queries_multi_match.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_queries_nested.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_queries_not.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_queries_prefix.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_queries_query_string.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_queries_range.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_queries_regexp.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_queries_script.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_queries_simple_query_string.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_queries_template_query.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_queries_term.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_queries_terms.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_queries_type.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_queries_wildcard.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_request.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_source.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/search_template.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/sort.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/suggest.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/suggest_field.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/suggester.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/suggester_completion.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/suggester_completion_fuzzy.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/suggester_context.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/suggester_context_category.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/suggester_context_geo.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/suggester_phrase.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/suggester_term.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/tasks_cancel.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/tasks_list.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/termvectors.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/update.go (100%) rename vendor/{github.com/olivere/elastic => gopkg.in/olivere/elastic.v3}/update_by_query.go (100%) diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 86ba8a8cbc..1655b21400 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -3,7 +3,8 @@ "GoVersion": "go1.6", "GodepVersion": "v74", "Packages": [ - "./..." + "./...", + "github.com/progrium/go-extpoints" ], "Deps": [ { @@ -105,6 +106,10 @@ "Comment": "v1.2-66-gc4afa8e", "Rev": "c4afa8e5421428d584b32d36d7b35f2c9ae5f823" }, + { + "ImportPath": "github.com/gedex/inflector", + "Rev": "91797f1712fd58f224781be1080b8613d096187e" + }, { "ImportPath": "github.com/ghodss/yaml", "Rev": "73d445a93680fa1a78ae23a5839bad48f32ba1ee" @@ -181,11 +186,6 @@ "ImportPath": "github.com/mitchellh/mapstructure", "Rev": "740c764bc6149d3f1806231418adb9f52c11bcbf" }, - { - "ImportPath": "github.com/olivere/elastic", - "Comment": "v3.0.41", - "Rev": "de60371c6ac39d035d1db8d64e6ae66552536950" - }, { "ImportPath": "github.com/optiopay/kafka", "Rev": "bc8e0950689f19fafd454acd517abe7b251cf54f" @@ -202,6 +202,11 @@ "ImportPath": "github.com/pmezard/go-difflib/difflib", "Rev": "d8ed2627bdf02c080bf22230dbb337003b7aba2d" }, + { + "ImportPath": "github.com/progrium/go-extpoints", + "Comment": "go-plugins-43-g529a176", + "Rev": "529a176f52394e48e8882dd70a01732f1bb480f3" + }, { "ImportPath": "github.com/prometheus/client_golang/prometheus", "Comment": "0.7.0-39-g3b78d7a", @@ -283,6 +288,10 @@ "Comment": "v1.0.0-920-g934dbf8", "Rev": "934dbf81977c67c521c75492dc1f55ca74dc5b04" }, + { + "ImportPath": "github.com/smartystreets/go-aws-auth", + "Rev": "2043e6d0bb7e4c18464a7bba562acbe482e3cabd" + }, { "ImportPath": "github.com/spf13/pflag", "Rev": "08b1a584251b5b62f458943640fc8ebd4d50aaa5" @@ -380,6 +389,11 @@ "Comment": "v0.9.0", "Rev": "3887ee99ecf07df5b447e9b00d9c0b2adaa9f3e4" }, + { + "ImportPath": "gopkg.in/olivere/elastic.v3", + "Comment": "v3.0.41", + "Rev": "de60371c6ac39d035d1db8d64e6ae66552536950" + }, { "ImportPath": "gopkg.in/olivere/elastic.v3/backoff", "Comment": "v3.0.41", diff --git a/Makefile b/Makefile index c0435c46b4..a0f4fe6190 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,8 @@ test-integration: clean deps build container: build cp heapster deploy/docker/heapster cp eventer deploy/docker/eventer - docker build -t $(PREFIX)/heapster:$(TAG) deploy/docker/ + docker build -t heapster deploy/docker/ + docker tag heapster:latest 032833530291.dkr.ecr.eu-west-1.amazonaws.com/heapster:latest grafana: docker build -t $(PREFIX)/heapster_grafana:$(TAG) grafana/ diff --git a/common/elasticsearch/elasticsearch.go b/common/elasticsearch/elasticsearch.go index 39092d2616..c04ab7a1dc 100644 --- a/common/elasticsearch/elasticsearch.go +++ b/common/elasticsearch/elasticsearch.go @@ -20,8 +20,10 @@ import ( "time" "github.com/golang/glog" - "github.com/olivere/elastic" "github.com/pborman/uuid" + + "gopkg.in/olivere/elastic.v3" + "os" ) const ( @@ -60,7 +62,7 @@ func SaveDataIntoES(esClient *elastic.Client, indexName string, typeName string, _, err = esClient.Index(). Index(indexName). Type(typeName). - Id(string(indexID)). + Id(indexID.String()). BodyJson(sinkData). Do() if err != nil { @@ -76,7 +78,7 @@ func CreateElasticSearchConfig(uri *url.URL) (*ElasticSearchConfig, error) { var esConfig ElasticSearchConfig opts, err := url.ParseQuery(uri.RawQuery) if err != nil { - return nil, fmt.Errorf("failed to parser url's query string: %s", err) + return nil, fmt.Errorf("fFailed to parser url's query string: %s", err) } // set the index for es,the default value is "heapster" @@ -87,12 +89,15 @@ func CreateElasticSearchConfig(uri *url.URL) (*ElasticSearchConfig, error) { // Set the URL endpoints of the ES's nodes. Notice that when sniffing is // enabled, these URLs are used to initially sniff the cluster on startup. - if len(opts["nodes"]) < 1 { + var startupFns []elastic.ClientOptionFunc + if len(opts["nodes"]) > 0 { + startupFns = append(startupFns, elastic.SetURL(opts["nodes"]...)) + } else if uri.Opaque != "" { + startupFns = append(startupFns, elastic.SetURL(uri.Opaque)) + } else { return nil, fmt.Errorf("There is no node assigned for connecting ES cluster") } - startupFns := []elastic.ClientOptionFunc{elastic.SetURL(opts["nodes"]...)} - // If the ES cluster needs authentication, the username and secret // should be set in sink config.Else, set the Authenticate flag to false if len(opts["esUserName"]) > 0 && len(opts["esUserSecret"]) > 0 { @@ -115,14 +120,6 @@ func CreateElasticSearchConfig(uri *url.URL) (*ElasticSearchConfig, error) { startupFns = append(startupFns, elastic.SetHealthcheck(healthCheck)) } - if len(opts["sniff"]) > 0 { - sniff, err := strconv.ParseBool(opts["sniff"][0]) - if err != nil { - return nil, fmt.Errorf("Failed to parse URL's sniff value into a bool") - } - startupFns = append(startupFns, elastic.SetSniff(sniff)) - } - if len(opts["startupHealthcheckTimeout"]) > 0 { timeout, err := time.ParseDuration(opts["startupHealthcheckTimeout"][0] + "s") if err != nil { @@ -131,12 +128,34 @@ func CreateElasticSearchConfig(uri *url.URL) (*ElasticSearchConfig, error) { startupFns = append(startupFns, elastic.SetHealthcheckTimeoutStartup(timeout)) } + if os.Getenv("AWS_ACCESS_KEY_ID") != "" || os.Getenv("AWS_ACCESS_KEY") != "" || + os.Getenv("AWS_SECRET_ACCESS_KEY") != "" || os.Getenv("AWS_SECRET_KEY") != "" { + + glog.Info("Configuring with AWS credentials..") + + awsClient, err := createAWSClient() + if err != nil { + return nil, err + } + + startupFns = append(startupFns, elastic.SetHttpClient(awsClient)) + startupFns = append(startupFns, elastic.SetSniff(false)) + } else { + if len(opts["sniff"]) > 0 { + sniff, err := strconv.ParseBool(opts["sniff"][0]) + if err != nil { + return nil, fmt.Errorf("Failed to parse URL's sniff value into a bool") + } + startupFns = append(startupFns, elastic.SetSniff(sniff)) + } + } + esConfig.EsClient, err = elastic.NewClient(startupFns...) if err != nil { - return nil, fmt.Errorf("failed to create ElasticSearch client: %v", err) + return nil, fmt.Errorf("Failed to create ElasticSearch client: %v", err) } - glog.V(2).Infof("elasticsearch sink configure successfully") + glog.V(2).Infof("ElasticSearch sink configure successfully") return &esConfig, nil } diff --git a/common/elasticsearch/elasticsearch_test.go b/common/elasticsearch/elasticsearch_test.go index 7d437a6b73..78e49f5dd5 100644 --- a/common/elasticsearch/elasticsearch_test.go +++ b/common/elasticsearch/elasticsearch_test.go @@ -20,7 +20,7 @@ import ( "testing" "time" - "github.com/olivere/elastic" + "gopkg.in/olivere/elastic.v3" ) func TestCreateElasticSearchConfig(t *testing.T) { diff --git a/docs/sink-configuration.md b/docs/sink-configuration.md index 0d6152ef98..d61e80d6b9 100644 --- a/docs/sink-configuration.md +++ b/docs/sink-configuration.md @@ -155,15 +155,18 @@ The following options are available: ### Elasticsearch This sink supports monitoring metrics and events. To use the ElasticSearch sink add the following flag: - +``` --sink=elasticsearch:[?] - +``` Normally an ElasticSearch cluster has multiple nodes or a proxy, so these need to be configured for the ElasticSearch sink. To do this, you can set `ES_SERVER_URL` to a dummy value, and use the `?nodes=` query value for each additional node in the cluster. For example: - +``` --sink=elasticsearch:?nodes=foo.com:9200&nodes=bar.com:9200 +``` +(*) Notice that using the `?nodes` notation will override the `ES_SERVER_URL` + Besides this, the following options can be set in query string: @@ -189,6 +192,32 @@ Like this: --sink="elasticsearch:?nodes=0.0.0.0:9200&Index=testEvent" +#### AWS Integration +In order to use AWS Managed Elastic we need to use one of the following methods: + +1. Making sure the public IPs of the Heapster are allowed on the ElasticSearch's Access Policy + +-OR- + +2. Configuring an Access Policy with IAM + 1. Configure the ElasticSearch cluster policy with IAM User + 2. Create a secret that stores the IAM credentials + 3. Expose the credentials to the environment variables: `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` + + ``` + env: + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: aws-heapster + key: aws.id + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: aws-heapster + key: aws.secret + ``` + ## Using multiple sinks Heapster can be configured to send k8s metrics and events to multiple sinks by specifying the`--sink=...` flag multiple times. diff --git a/events/sinks/elasticsearch/driver.go b/events/sinks/elasticsearch/driver.go index be2204df72..de974fc81e 100644 --- a/events/sinks/elasticsearch/driver.go +++ b/events/sinks/elasticsearch/driver.go @@ -22,7 +22,7 @@ import ( "encoding/json" "github.com/golang/glog" - "github.com/olivere/elastic" + "gopkg.in/olivere/elastic.v3" esCommon "k8s.io/heapster/common/elasticsearch" event_core "k8s.io/heapster/events/core" "k8s.io/heapster/metrics/core" @@ -84,9 +84,12 @@ func (sink *elasticSearchSink) ExportEvents(eventBatch *event_core.EventBatch) { for _, event := range eventBatch.Events { point, err := eventToPoint(event) if err != nil { - glog.Warningf("Failed to convert event to point: %v", err) + glog.Info("Failed to convert event to point: %v", err) + } + err = sink.saveDataFunc(sink.esConfig.EsClient, sink.esConfig.Index, typeName, point) + if err != nil { + glog.V(3).Info("Failed to export data to ElasticSearch sink: ", err) } - sink.saveDataFunc(sink.esConfig.EsClient, sink.esConfig.Index, typeName, point) } } @@ -102,12 +105,12 @@ func NewElasticSearchSink(uri *url.URL) (event_core.EventSink, error) { var esSink elasticSearchSink elasticsearchConfig, err := esCommon.CreateElasticSearchConfig(uri) if err != nil { - glog.V(2).Infof("failed to config elasticsearch") + glog.V(2).Infof("Failed to config ElasticSearch") return nil, err } esSink.esConfig = *elasticsearchConfig esSink.saveDataFunc = esCommon.SaveDataIntoES - glog.V(2).Infof("elasticsearch sink setup successfully") + glog.V(2).Infof("ElasticSearch sink setup successfully") return &esSink, nil } diff --git a/events/sinks/elasticsearch/driver_test.go b/events/sinks/elasticsearch/driver_test.go index cdd7ddbc50..33910fe502 100644 --- a/events/sinks/elasticsearch/driver_test.go +++ b/events/sinks/elasticsearch/driver_test.go @@ -20,8 +20,8 @@ import ( "testing" "time" - "github.com/olivere/elastic" "github.com/stretchr/testify/assert" + "gopkg.in/olivere/elastic.v3" esCommon "k8s.io/heapster/common/elasticsearch" "k8s.io/heapster/events/core" kube_api "k8s.io/kubernetes/pkg/api" diff --git a/metrics/sinks/elasticsearch/driver.go b/metrics/sinks/elasticsearch/driver.go index 8df8aa8567..61c1e0d25d 100644 --- a/metrics/sinks/elasticsearch/driver.go +++ b/metrics/sinks/elasticsearch/driver.go @@ -20,7 +20,7 @@ import ( "time" "github.com/golang/glog" - "github.com/olivere/elastic" + "gopkg.in/olivere/elastic.v3" esCommon "k8s.io/heapster/common/elasticsearch" "k8s.io/heapster/metrics/core" ) @@ -58,7 +58,10 @@ func (sink *elasticSearchSink) ExportData(dataBatch *core.DataBatch) { }, MetricsTimestamp: dataBatch.Timestamp.UTC(), } - sink.saveDataFunc(sink.esConfig.EsClient, sink.esConfig.Index, typeName, point) + err := sink.saveDataFunc(sink.esConfig.EsClient, sink.esConfig.Index, typeName, point) + if err != nil { + glog.V(3).Info("Failed to export data to ElasticSearch sink: ", err) + } } for _, metric := range metricSet.LabeledMetrics { labels := make(map[string]string) @@ -76,7 +79,10 @@ func (sink *elasticSearchSink) ExportData(dataBatch *core.DataBatch) { }, MetricsTimestamp: dataBatch.Timestamp.UTC(), } - sink.saveDataFunc(sink.esConfig.EsClient, sink.esConfig.Index, typeName, point) + err := sink.saveDataFunc(sink.esConfig.EsClient, sink.esConfig.Index, typeName, point) + if err != nil { + glog.V(3).Info("Failed to export data to ElasticSearch sink: ", err) + } } } } @@ -93,12 +99,12 @@ func NewElasticSearchSink(uri *url.URL) (core.DataSink, error) { var esSink elasticSearchSink elasticsearchConfig, err := esCommon.CreateElasticSearchConfig(uri) if err != nil { - glog.V(2).Infof("failed to config elasticsearch") + glog.V(2).Infof("Failed to config ElasticSearch") return nil, err } esSink.esConfig = *elasticsearchConfig esSink.saveDataFunc = esCommon.SaveDataIntoES - glog.V(2).Infof("elasticsearch sink setup successfully") + glog.V(2).Infof("ElasticSearch sink setup successfully") return &esSink, nil } diff --git a/metrics/sinks/elasticsearch/driver_test.go b/metrics/sinks/elasticsearch/driver_test.go index 8abc9f6aa7..77727f654e 100644 --- a/metrics/sinks/elasticsearch/driver_test.go +++ b/metrics/sinks/elasticsearch/driver_test.go @@ -20,8 +20,8 @@ import ( "testing" "time" - "github.com/olivere/elastic" "github.com/stretchr/testify/assert" + "gopkg.in/olivere/elastic.v3" esCommon "k8s.io/heapster/common/elasticsearch" "k8s.io/heapster/metrics/core" ) diff --git a/vendor/github.com/gedex/inflector/.travis.yml b/vendor/github.com/gedex/inflector/.travis.yml new file mode 100644 index 0000000000..0079167161 --- /dev/null +++ b/vendor/github.com/gedex/inflector/.travis.yml @@ -0,0 +1,3 @@ +language: Go +go: + - 1.1 diff --git a/vendor/github.com/gedex/inflector/CakePHP_LICENSE.txt b/vendor/github.com/gedex/inflector/CakePHP_LICENSE.txt new file mode 100644 index 0000000000..414ab1e723 --- /dev/null +++ b/vendor/github.com/gedex/inflector/CakePHP_LICENSE.txt @@ -0,0 +1,28 @@ +The MIT License + +CakePHP(tm) : The Rapid Development PHP Framework (http://cakephp.org) +Copyright (c) 2005-2013, Cake Software Foundation, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +Cake Software Foundation, Inc. +1785 E. Sahara Avenue, +Suite 490-204 +Las Vegas, Nevada 89104, +United States of America. \ No newline at end of file diff --git a/vendor/github.com/gedex/inflector/LICENSE.md b/vendor/github.com/gedex/inflector/LICENSE.md new file mode 100644 index 0000000000..cdbc217671 --- /dev/null +++ b/vendor/github.com/gedex/inflector/LICENSE.md @@ -0,0 +1,29 @@ +Copyright (c) 2013 Akeda Bagus . All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------- + +Much of this library was inspired from CakePHP's inflector, a PHP +framework licensed under MIT license (see CakePHP_LICENSE.txt). diff --git a/vendor/github.com/gedex/inflector/README.md b/vendor/github.com/gedex/inflector/README.md new file mode 100644 index 0000000000..a3a9fd4236 --- /dev/null +++ b/vendor/github.com/gedex/inflector/README.md @@ -0,0 +1,32 @@ +Inflector +========= + +Inflector pluralizes and singularizes English nouns. + +**Documentation:** + +[![Build Status](https://travis-ci.org/gedex/inflector.png?branch=master)](https://travis-ci.org/gedex/inflector) +[![Build Status](https://drone.io/github.com/gedex/inflector/status.png)](https://drone.io/github.com/gedex/inflector/latest) +[![Coverage Status](https://coveralls.io/repos/gedex/inflector/badge.png?branch=master)](https://coveralls.io/r/gedex/inflector?branch=master) + +## Basic Usage + +There are only two exported functions: `Pluralize` and `Singularize`. + +~~~go +s := "People" +fmt.Println(inflector.Singularize(s)) // will print "Person" + +s2 := "octopus" +fmt.Println(inflector.Pluralize(s2)) // will print "octopuses" +~~~ + +Please see [example/example.go](./example/example.go) for a complete example. + +## Credits + +* [CakePHP's Inflector](https://github.com/cakephp/cakephp/blob/master/lib/Cake/Utility/Inflector.php) + +## License + +This library is distributed under the BSD-style license found in the LICENSE.md file. diff --git a/vendor/github.com/gedex/inflector/inflector.go b/vendor/github.com/gedex/inflector/inflector.go new file mode 100644 index 0000000000..17bc887e73 --- /dev/null +++ b/vendor/github.com/gedex/inflector/inflector.go @@ -0,0 +1,355 @@ +// Copyright 2013 Akeda Bagus . All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +Package inflector pluralizes and singularizes English nouns. + +There are only two exported functions: `Pluralize` and `Singularize`. + + s := "People" + fmt.Println(inflector.Singularize(s)) // will print "Person" + + s2 := "octopus" + fmt.Println(inflector.Pluralize(s2)) // will print "octopuses" + +*/ +package inflector + +import ( + "bytes" + "fmt" + "regexp" + "strings" + "sync" +) + +// Rule represents name of the inflector rule, can be +// Plural or Singular +type Rule int + +const ( + Plural = iota + Singular +) + +// InflectorRule represents inflector rule +type InflectorRule struct { + Rules []*ruleItem + Irregular []*irregularItem + Uninflected []string + compiledIrregular *regexp.Regexp + compiledUninflected *regexp.Regexp + compiledRules []*compiledRule +} + +type ruleItem struct { + pattern string + replacement string +} + +type irregularItem struct { + word string + replacement string +} + +// compiledRule represents compiled version of Inflector.Rules. +type compiledRule struct { + replacement string + *regexp.Regexp +} + +// threadsafe access to rules and caches +var mutex sync.Mutex +var rules = make(map[Rule]*InflectorRule) + +// Words that should not be inflected +var uninflected = []string{ + `Amoyese`, `bison`, `Borghese`, `bream`, `breeches`, `britches`, `buffalo`, + `cantus`, `carp`, `chassis`, `clippers`, `cod`, `coitus`, `Congoese`, + `contretemps`, `corps`, `debris`, `diabetes`, `djinn`, `eland`, `elk`, + `equipment`, `Faroese`, `flounder`, `Foochowese`, `gallows`, `Genevese`, + `Genoese`, `Gilbertese`, `graffiti`, `headquarters`, `herpes`, `hijinks`, + `Hottentotese`, `information`, `innings`, `jackanapes`, `Kiplingese`, + `Kongoese`, `Lucchese`, `mackerel`, `Maltese`, `.*?media`, `mews`, `moose`, + `mumps`, `Nankingese`, `news`, `nexus`, `Niasese`, `Pekingese`, + `Piedmontese`, `pincers`, `Pistoiese`, `pliers`, `Portuguese`, `proceedings`, + `rabies`, `rice`, `rhinoceros`, `salmon`, `Sarawakese`, `scissors`, + `sea[- ]bass`, `series`, `Shavese`, `shears`, `siemens`, `species`, `swine`, + `testes`, `trousers`, `trout`, `tuna`, `Vermontese`, `Wenchowese`, `whiting`, + `wildebeest`, `Yengeese`, +} + +// Plural words that should not be inflected +var uninflectedPlurals = []string{ + `.*[nrlm]ese`, `.*deer`, `.*fish`, `.*measles`, `.*ois`, `.*pox`, `.*sheep`, + `people`, +} + +// Singular words that should not be inflected +var uninflectedSingulars = []string{ + `.*[nrlm]ese`, `.*deer`, `.*fish`, `.*measles`, `.*ois`, `.*pox`, `.*sheep`, + `.*ss`, +} + +type cache map[string]string + +// Inflected words that already cached for immediate retrieval from a given Rule +var caches = make(map[Rule]cache) + +// map of irregular words where its key is a word and its value is the replacement +var irregularMaps = make(map[Rule]cache) + +func init() { + + rules[Plural] = &InflectorRule{ + Rules: []*ruleItem{ + &ruleItem{`(?i)(s)tatus$`, `${1}${2}tatuses`}, + &ruleItem{`(?i)(quiz)$`, `${1}zes`}, + &ruleItem{`(?i)^(ox)$`, `${1}${2}en`}, + &ruleItem{`(?i)([m|l])ouse$`, `${1}ice`}, + &ruleItem{`(?i)(matr|vert|ind)(ix|ex)$`, `${1}ices`}, + &ruleItem{`(?i)(x|ch|ss|sh)$`, `${1}es`}, + &ruleItem{`(?i)([^aeiouy]|qu)y$`, `${1}ies`}, + &ruleItem{`(?i)(hive)$`, `$1s`}, + &ruleItem{`(?i)(?:([^f])fe|([lre])f)$`, `${1}${2}ves`}, + &ruleItem{`(?i)sis$`, `ses`}, + &ruleItem{`(?i)([ti])um$`, `${1}a`}, + &ruleItem{`(?i)(p)erson$`, `${1}eople`}, + &ruleItem{`(?i)(m)an$`, `${1}en`}, + &ruleItem{`(?i)(c)hild$`, `${1}hildren`}, + &ruleItem{`(?i)(buffal|tomat)o$`, `${1}${2}oes`}, + &ruleItem{`(?i)(alumn|bacill|cact|foc|fung|nucle|radi|stimul|syllab|termin|vir)us$`, `${1}i`}, + &ruleItem{`(?i)us$`, `uses`}, + &ruleItem{`(?i)(alias)$`, `${1}es`}, + &ruleItem{`(?i)(ax|cris|test)is$`, `${1}es`}, + &ruleItem{`s$`, `s`}, + &ruleItem{`^$`, ``}, + &ruleItem{`$`, `s`}, + }, + Irregular: []*irregularItem{ + &irregularItem{`atlas`, `atlases`}, + &irregularItem{`beef`, `beefs`}, + &irregularItem{`brother`, `brothers`}, + &irregularItem{`cafe`, `cafes`}, + &irregularItem{`child`, `children`}, + &irregularItem{`cookie`, `cookies`}, + &irregularItem{`corpus`, `corpuses`}, + &irregularItem{`cow`, `cows`}, + &irregularItem{`ganglion`, `ganglions`}, + &irregularItem{`genie`, `genies`}, + &irregularItem{`genus`, `genera`}, + &irregularItem{`graffito`, `graffiti`}, + &irregularItem{`hoof`, `hoofs`}, + &irregularItem{`loaf`, `loaves`}, + &irregularItem{`man`, `men`}, + &irregularItem{`money`, `monies`}, + &irregularItem{`mongoose`, `mongooses`}, + &irregularItem{`move`, `moves`}, + &irregularItem{`mythos`, `mythoi`}, + &irregularItem{`niche`, `niches`}, + &irregularItem{`numen`, `numina`}, + &irregularItem{`occiput`, `occiputs`}, + &irregularItem{`octopus`, `octopuses`}, + &irregularItem{`opus`, `opuses`}, + &irregularItem{`ox`, `oxen`}, + &irregularItem{`penis`, `penises`}, + &irregularItem{`person`, `people`}, + &irregularItem{`sex`, `sexes`}, + &irregularItem{`soliloquy`, `soliloquies`}, + &irregularItem{`testis`, `testes`}, + &irregularItem{`trilby`, `trilbys`}, + &irregularItem{`turf`, `turfs`}, + &irregularItem{`potato`, `potatoes`}, + &irregularItem{`hero`, `heroes`}, + &irregularItem{`tooth`, `teeth`}, + &irregularItem{`goose`, `geese`}, + &irregularItem{`foot`, `feet`}, + }, + } + prepare(Plural) + + rules[Singular] = &InflectorRule{ + Rules: []*ruleItem{ + &ruleItem{`(?i)(s)tatuses$`, `${1}${2}tatus`}, + &ruleItem{`(?i)^(.*)(menu)s$`, `${1}${2}`}, + &ruleItem{`(?i)(quiz)zes$`, `$1`}, + &ruleItem{`(?i)(matr)ices$`, `${1}ix`}, + &ruleItem{`(?i)(vert|ind)ices$`, `${1}ex`}, + &ruleItem{`(?i)^(ox)en`, `$1`}, + &ruleItem{`(?i)(alias)(es)*$`, `$1`}, + &ruleItem{`(?i)(alumn|bacill|cact|foc|fung|nucle|radi|stimul|syllab|termin|viri?)i$`, `${1}us`}, + &ruleItem{`(?i)([ftw]ax)es`, `$1`}, + &ruleItem{`(?i)(cris|ax|test)es$`, `${1}is`}, + &ruleItem{`(?i)(shoe|slave)s$`, `$1`}, + &ruleItem{`(?i)(o)es$`, `$1`}, + &ruleItem{`ouses$`, `ouse`}, + &ruleItem{`([^a])uses$`, `${1}us`}, + &ruleItem{`(?i)([m|l])ice$`, `${1}ouse`}, + &ruleItem{`(?i)(x|ch|ss|sh)es$`, `$1`}, + &ruleItem{`(?i)(m)ovies$`, `${1}${2}ovie`}, + &ruleItem{`(?i)(s)eries$`, `${1}${2}eries`}, + &ruleItem{`(?i)([^aeiouy]|qu)ies$`, `${1}y`}, + &ruleItem{`(?i)(tive)s$`, `$1`}, + &ruleItem{`(?i)([lre])ves$`, `${1}f`}, + &ruleItem{`(?i)([^fo])ves$`, `${1}fe`}, + &ruleItem{`(?i)(hive)s$`, `$1`}, + &ruleItem{`(?i)(drive)s$`, `$1`}, + &ruleItem{`(?i)(^analy)ses$`, `${1}sis`}, + &ruleItem{`(?i)(analy|diagno|^ba|(p)arenthe|(p)rogno|(s)ynop|(t)he)ses$`, `${1}${2}sis`}, + &ruleItem{`(?i)([ti])a$`, `${1}um`}, + &ruleItem{`(?i)(p)eople$`, `${1}${2}erson`}, + &ruleItem{`(?i)(m)en$`, `${1}an`}, + &ruleItem{`(?i)(c)hildren$`, `${1}${2}hild`}, + &ruleItem{`(?i)(n)ews$`, `${1}${2}ews`}, + &ruleItem{`eaus$`, `eau`}, + &ruleItem{`^(.*us)$`, `$1`}, + &ruleItem{`(?i)s$`, ``}, + }, + Irregular: []*irregularItem{ + &irregularItem{`foes`, `foe`}, + &irregularItem{`waves`, `wave`}, + &irregularItem{`curves`, `curve`}, + &irregularItem{`atlases`, `atlas`}, + &irregularItem{`beefs`, `beef`}, + &irregularItem{`brothers`, `brother`}, + &irregularItem{`cafes`, `cafe`}, + &irregularItem{`children`, `child`}, + &irregularItem{`cookies`, `cookie`}, + &irregularItem{`corpuses`, `corpus`}, + &irregularItem{`cows`, `cow`}, + &irregularItem{`ganglions`, `ganglion`}, + &irregularItem{`genies`, `genie`}, + &irregularItem{`genera`, `genus`}, + &irregularItem{`graffiti`, `graffito`}, + &irregularItem{`hoofs`, `hoof`}, + &irregularItem{`loaves`, `loaf`}, + &irregularItem{`men`, `man`}, + &irregularItem{`monies`, `money`}, + &irregularItem{`mongooses`, `mongoose`}, + &irregularItem{`moves`, `move`}, + &irregularItem{`mythoi`, `mythos`}, + &irregularItem{`niches`, `niche`}, + &irregularItem{`numina`, `numen`}, + &irregularItem{`occiputs`, `occiput`}, + &irregularItem{`octopuses`, `octopus`}, + &irregularItem{`opuses`, `opus`}, + &irregularItem{`oxen`, `ox`}, + &irregularItem{`penises`, `penis`}, + &irregularItem{`people`, `person`}, + &irregularItem{`sexes`, `sex`}, + &irregularItem{`soliloquies`, `soliloquy`}, + &irregularItem{`testes`, `testis`}, + &irregularItem{`trilbys`, `trilby`}, + &irregularItem{`turfs`, `turf`}, + &irregularItem{`potatoes`, `potato`}, + &irregularItem{`heroes`, `hero`}, + &irregularItem{`teeth`, `tooth`}, + &irregularItem{`geese`, `goose`}, + &irregularItem{`feet`, `foot`}, + }, + } + prepare(Singular) +} + +// prepare rule, e.g., compile the pattern. +func prepare(r Rule) error { + var reString string + + switch r { + case Plural: + // Merge global uninflected with singularsUninflected + rules[r].Uninflected = merge(uninflected, uninflectedPlurals) + case Singular: + // Merge global uninflected with singularsUninflected + rules[r].Uninflected = merge(uninflected, uninflectedSingulars) + } + + // Set InflectorRule.compiledUninflected by joining InflectorRule.Uninflected into + // a single string then compile it. + reString = fmt.Sprintf(`(?i)(^(?:%s))$`, strings.Join(rules[r].Uninflected, `|`)) + rules[r].compiledUninflected = regexp.MustCompile(reString) + + // Prepare irregularMaps + irregularMaps[r] = make(cache, len(rules[r].Irregular)) + + // Set InflectorRule.compiledIrregular by joining the irregularItem.word of Inflector.Irregular + // into a single string then compile it. + vIrregulars := make([]string, len(rules[r].Irregular)) + for i, item := range rules[r].Irregular { + vIrregulars[i] = item.word + irregularMaps[r][item.word] = item.replacement + } + reString = fmt.Sprintf(`(?i)(.*)\b((?:%s))$`, strings.Join(vIrregulars, `|`)) + rules[r].compiledIrregular = regexp.MustCompile(reString) + + // Compile all patterns in InflectorRule.Rules + rules[r].compiledRules = make([]*compiledRule, len(rules[r].Rules)) + for i, item := range rules[r].Rules { + rules[r].compiledRules[i] = &compiledRule{item.replacement, regexp.MustCompile(item.pattern)} + } + + // Prepare caches + caches[r] = make(cache) + + return nil +} + +// merge slice a and slice b +func merge(a []string, b []string) []string { + result := make([]string, len(a)+len(b)) + copy(result, a) + copy(result[len(a):], b) + + return result +} + +// Pluralize returns string s in plural form. +func Pluralize(s string) string { + return getInflected(Plural, s) +} + +// Singularize returns string s in singular form. +func Singularize(s string) string { + return getInflected(Singular, s) +} + +func getInflected(r Rule, s string) string { + mutex.Lock() + defer mutex.Unlock() + if v, ok := caches[r][s]; ok { + return v + } + + // Check for irregular words + if res := rules[r].compiledIrregular.FindStringSubmatch(s); len(res) >= 3 { + var buf bytes.Buffer + + buf.WriteString(res[1]) + buf.WriteString(s[0:1]) + buf.WriteString(irregularMaps[r][strings.ToLower(res[2])][1:]) + + // Cache it then returns + caches[r][s] = buf.String() + return caches[r][s] + } + + // Check for uninflected words + if rules[r].compiledUninflected.MatchString(s) { + caches[r][s] = s + return caches[r][s] + } + + // Check each rule + for _, re := range rules[r].compiledRules { + if re.MatchString(s) { + caches[r][s] = re.ReplaceAllString(s, re.replacement) + return caches[r][s] + } + } + + // Returns unaltered + caches[r][s] = s + return caches[r][s] +} diff --git a/vendor/github.com/progrium/go-extpoints/.gitignore b/vendor/github.com/progrium/go-extpoints/.gitignore new file mode 100644 index 0000000000..b5c4dba585 --- /dev/null +++ b/vendor/github.com/progrium/go-extpoints/.gitignore @@ -0,0 +1,4 @@ +examples/tool/tool +examples/daemon/daemon +go-extpoints +tests/extpoints/extpoints.go diff --git a/vendor/github.com/progrium/go-extpoints/LICENSE b/vendor/github.com/progrium/go-extpoints/LICENSE new file mode 100644 index 0000000000..3a487b3c3a --- /dev/null +++ b/vendor/github.com/progrium/go-extpoints/LICENSE @@ -0,0 +1,19 @@ +Copyright (C) 2015 Jeff Lindsay + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/progrium/go-extpoints/Makefile b/vendor/github.com/progrium/go-extpoints/Makefile new file mode 100644 index 0000000000..ffe98176e5 --- /dev/null +++ b/vendor/github.com/progrium/go-extpoints/Makefile @@ -0,0 +1,8 @@ +GO ?= go + +test: + $(GO) run *.go ./tests/extpoints + $(GO) test -v ./tests + +install: + $(GO) install diff --git a/vendor/github.com/progrium/go-extpoints/README.md b/vendor/github.com/progrium/go-extpoints/README.md new file mode 100644 index 0000000000..f95223efb2 --- /dev/null +++ b/vendor/github.com/progrium/go-extpoints/README.md @@ -0,0 +1,287 @@ +# go-extpoints + +This Go generator, named short for "extension points", provides a generic [inversion of control](http://en.wikipedia.org/wiki/Inversion_of_control) model for making extensible Go packages, libraries, and applications. + +It generates package extension point singletons from extension types you define. Extension points are then used to both register extensions and use the registered extensions with a common [meta-API](#extension-point-api). + +[Logspout](https://github.com/gliderlabs/logspout) is a real application built using go-extpoints. [Read about it here.](http://gliderlabs.com/blog/2015/03/31/new-logspout-extensible-docker-logging/) + +## Getting the tool + + $ go install github.com/progrium/go-extpoints + +## Concepts + +#### Extension Types + +These define your hooks. They can be Go interfaces or simple function signature types. Here are some generic examples: + +```go +type ConfigStore interface { + Get(key string) (string, error) + Set(key, value string) error + Del(key string) error +} + +type AuthProvider interface { + Authenticate(user, pass string) bool +} + +type EventListener interface { + Notify(event Event) +} + +type HttpEndpoint func() http.Handler + +type RequestModifier func(request *http.Request) + +``` + +#### Extension Points + +With types defined, go-extpoints generates package singletons for each type. When your program starts, extensions are registered with extension points. Then you can then use registered extensions in a number of ways: + +```go +// Lookup a single registered extension for drivers +config := ConfigStores.Lookup(configStore) +if config == nil { + log.Fatalf("config store '%s' not registered", configStore) +} +config.Set("foo", "bar") + +// Iterate until you get what you need +func authenticate(user, pass string) bool { + for _, provider := range AuthProviders.All() { + if provide.Authenticate(user, pass) { + return true + } + } + return false +} + +// Fire and forget events to all extensions +for _, listener := range EventListeners.All() { + listener.Notify(event) +} + +// Use name and return value of all extensions for registration +for name, handler := range HttpEndpoints.All() { + http.Handle("/"+name, handler()) +} + +// Pass by reference to all extensions for middleware +for _, modifier := range RequestModifiers.All() { + modifier(req) +} + +``` + +## Extension Point API + +All extension types passed to go-extpoints will be turned into extension point singletons, using the pluralized name of the extension type. These extension point objects implement this simple meta-API: + +```go +type interface { + // if name is "", the specific extension type is used. + // returns false if doesn't implement type or already registered. + Register(extension , name string) bool + + // returns false if not registered to start with + Unregister(name string) bool + + // returns nil if not registered + Lookup(name string) + + // for sorted subsets. each name is looked up in order, nil or not + Select(names []string) [] + + // all registered, keyed by name + All() map[string] + + // convenient list of names + Names() []string + +} +``` + +It also generates top-level registration functions that will run extensions through all known extension points, registering or unregistering with any that are based on an interface the extension implements. They return the names of the interfaces they were registered/unregistered with. + +```go + +func RegisterExtension(extension interface{}, name string) []string + +func UnregisterExtension(name string) []string + +``` + +## Example Application + +Here is a full Go application that lets extensions hook into `main()` as subcommands simply by implementing an interface we'll make called `Subcommand`. This interface will have just one method `Run()`, but you can make extension points based on any interface. + +Assuming our package lives under `$GOPATH/src/github.com/quick/example`, here is our `main.go`: + +```go +//go:generate go-extpoints +package main + +import ( + "fmt" + "os" + + "github.com/quick/example/extpoints" +) + +var subcommands = extpoints.Subcommands + +func usage() { + fmt.Println("Available commands:\n") + for name, _ := range subcommands.All() { + fmt.Println(" - ", name) + } + os.Exit(2) +} + +func main() { + if len(os.Args) < 2 { + usage() + } + cmd := subcommands.Lookup(os.Args[1]) + if cmd == nil { + usage() + } + cmd.Run(os.Args[2:]) +} +``` +Two things to note. First, the `go:generate` directive at the top. This tells `go generate` it needs to run `go-extpoints`, which will happen in a moment. + +Another thing to note, the extension point is accessed by a variable named by the plural of our interface `Subcommand` and in this case lives under a separate `extpoints` subpackage. + +We need to create that subpackage with a Go file in it to define our interface used for this extension point. This is our `extpoints/interfaces.go`: + +```go +package extpoints + +type Subcommand interface { + Run(args []string) +} +``` + +We use `go generate` now to produce the extension point code in our `extpoints` subpackage. These extension points are based on any Go interfaces you've defined in there. In our case, just `Subcommand`. + + $ go generate + ... + $ go install + +Okay, but it doesn't *do* anything! Let's make a builtin command extension that implements `Subcommand`. Add a `hello.go` file: + +```go +package main + +import ( + "fmt" + "github.com/quick/example/extpoints" +) + +func init() { + extpoints.Register(new(HelloComponent), "hello") +} + +type HelloComponent struct {} + +func (p *HelloComponent) Run(args []string) { + fmt.Println("Hello world!") +} +``` + +Now when we build and run the app, it shows `hello` as a subcommand. We've registered the component with the name `hello`, which we happen to use to identify the name of the subcommand. Component names are optional, but can be handy for situations like this one. + +Certainly, the value of extension points becomes clearer with larger applications and more interesting interfaces. But just consider now that the component defined in `hello.go` *could* exist in another package in another repo. You'd just have to import it and rebuild to let it hook into our application. + +There are two more in-deptch example applications in this repo to take a look at: + + * [tool](https://github.com/progrium/go-extpoints/tree/master/examples/tool) ([extpoints](http://godoc.org/github.com/progrium/go-extpoints/examples/tool/extpoints)), a more realistic CLI tool with subcommands and lifecycle hooks + * [daemon](https://github.com/progrium/go-extpoints/tree/master/examples/daemon), ... doesn't exist yet + + + +## Making it easy to install extensions + +Assuming you tell third-party developers to call your package or extension point `Register` in their `init()`, you can link them with a side-effect import (using a blank import name). + +You can make this easy for users to enable/disable via comments, or add their own without worrying about messing with your code by having a separate `extensions.go` or `plugins.go` file with just these imports: + +```go +package yourpackage + +import ( + _ "github.com/you/some-extension" + _ "github.com/third-party/another-extension" +) + +``` + +Users can now just edit this file and `go build` or `go install`. + +## Usage Patterns + +Here are different example ways to use extension points to interact with extensions: + +#### Simple Iteration +```go +for _, listener := range extpoints.EventListeners.All() { + listener.Notify(&MyEvent{}) +} +``` + +#### Lookup Only One +```go +driverName := config.Get("storage-driver") +driver := extpoints.StorageDrivers.Lookup(driverName) +if driver == nil { + log.Fatalf("storage driver '%s' not installed", driverName) +} +driver.StoreObject(object) +``` + +#### Passing by Reference +```go +for _, filter := range extpoints.RequestFilters.All() { + filter.FilterRequest(req) +} +``` + +#### Match and Use +```go +for _, handler := range extpoints.RequestHandlers.All() { + if handler.MatchRequest(req) { + handler.HandleRequest(req) + break + } +} +``` + +## Why the `extpoints` subpackage? + +Since we encourage the convention of a subpackage called `extpoints`, it makes it very easy to identify a package as having extension points from looking at the project tree. You then know where to look to find the interfaces that are exposed as extension points. + +Third-party packages have a well known package to import for registering. Whether you have extension points for a library package or a command with just a `main` package, there's always a definite `extpoints` package there to import. + +It also makes it clearer in your code when you're using extension points. You have to explicitly import the package, then call `extpoints.` when using them. This helps identify where extension points actually hook into your program. + +## Groundwork for Dynamic Extensions + +Although this only seems to allow for compile-time extensibility, this itself is quite a win. It means power users can build and compile in their own extensions that live outside your repository. + +However, it also lays the groundwork for other dynamic extensions. I've used this model to wrap extension points for components in embedded scripting languages, as hook scripts, as remote plugin daemons via RPC, or all of the above implemented as components themselves! + +No matter how you're thinking about dynamic extensions later on, using `go-extpoints` gives you a lot of options. Once Go supports dynamic libraries? This will work perfectly with that, too. + +## Inspiration + +This project and component model is a lightweight, Go idiomatic port of the [component architecture](http://trac.edgewall.org/wiki/TracDev/ComponentArchitecture) used in Trac, which is written in Python. It's taken about a year to get this right in Go. + +![Trac Component Architecture](http://trac.edgewall.org/raw-attachment/wiki/TracDev/ComponentArchitecture/xtnpt.png) + +## License + +BSD diff --git a/vendor/github.com/progrium/go-extpoints/main.go b/vendor/github.com/progrium/go-extpoints/main.go new file mode 100644 index 0000000000..c6a2cf25ec --- /dev/null +++ b/vendor/github.com/progrium/go-extpoints/main.go @@ -0,0 +1,179 @@ +package main + +import ( + "go/ast" + "go/parser" + "go/token" + "io/ioutil" + "log" + "os" + "path/filepath" + "strings" + "text/template" + + "github.com/gedex/inflector" +) + +func processFile(inputPath string) (string, []string) { + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, inputPath, nil, parser.ParseComments) + if err != nil { + log.Fatalf("Could not parse file: %s", err) + } + + packageName := identifyPackage(f) + if packageName == "" { + log.Fatalf("Could not determine package name of %s", inputPath) + } + + var ifaces []string + for _, decl := range f.Decls { + if typeName, ok := identifyInterface(decl); ok { + ifaces = append(ifaces, typeName) + continue + } + if typeName, ok := identifyFuncType(decl); ok { + ifaces = append(ifaces, typeName) + continue + } + } + + return packageName, ifaces +} + +func identifyPackage(f *ast.File) string { + if f.Name == nil { + return "" + } + return f.Name.Name +} + +func identifyFuncType(decl ast.Decl) (typeName string, match bool) { + genDecl, ok := decl.(*ast.GenDecl) + if !ok { + return + } + for _, spec := range genDecl.Specs { + if typeSpec, ok := spec.(*ast.TypeSpec); ok { + if _, ok := typeSpec.Type.(*ast.FuncType); ok { + if typeSpec.Name != nil { + typeName = typeSpec.Name.Name + break + } + } + } + } + if typeName == "" { + return + } + match = true + return +} + +func identifyInterface(decl ast.Decl) (typeName string, match bool) { + genDecl, ok := decl.(*ast.GenDecl) + if !ok { + return + } + for _, spec := range genDecl.Specs { + if typeSpec, ok := spec.(*ast.TypeSpec); ok { + if _, ok := typeSpec.Type.(*ast.InterfaceType); ok { + if typeSpec.Name != nil { + typeName = typeSpec.Name.Name + break + } + } + } + } + if typeName == "" { + return + } + match = true + return +} + +type extensionPoint struct { + Name string +} + +func (i *extensionPoint) Var() string { + return inflector.Pluralize(i.Name) +} + +func (i *extensionPoint) Type() string { + return strings.ToLower(i.Name[0:1]) + i.Name[1:] + "Ext" +} + +func extensionPoints(ifaces []string) []extensionPoint { + var extpoints []extensionPoint + for _, iface := range ifaces { + extpoints = append(extpoints, extensionPoint{iface}) + } + return extpoints +} + +type templateData struct { + Package string + ExtensionPoints []extensionPoint +} + +func renderExtpoints(path, packageName string, ifaces []string) error { + output, err := os.Create(path) + if err != nil { + log.Fatalf("Could not open output file: %s", err) + } + defer output.Close() + outputTemplate := template.Must(template.New("render").Parse(extpointsTemplate)) + return outputTemplate.Execute(output, templateData{packageName, extensionPoints(ifaces)}) +} + +func main() { + log.SetFlags(0) + log.SetPrefix("extpoints: ") + + packagePath := "./extpoints" + if len(os.Args) > 1 { + packagePath = os.Args[1] + } + if _, err := os.Stat(packagePath); os.IsNotExist(err) { + log.Fatal("Unable to find package for extpoints, not found:", packagePath) + } + + var packageName string + var ifaces, ifacesAll []string + ifacesAllowed := make(map[string]struct{}) + + if len(os.Args) > 2 { + for _, iface := range os.Args[2:] { + ifacesAllowed[iface] = struct{}{} + } + } + + files, _ := ioutil.ReadDir(packagePath) + for _, file := range files { + if file.Name() != "extpoints.go" && !strings.HasSuffix(file.Name(), "_ext.go") { + path := filepath.Join(packagePath, file.Name()) + log.Printf("Processing file %s", path) + packageName, ifaces = processFile(path) + if len(ifacesAllowed) > 0 { + var ifacesFiltered []string + for _, iface := range ifaces { + _, allowed := ifacesAllowed[iface] + if allowed { + ifacesFiltered = append(ifacesFiltered, iface) + } + } + ifaces = ifacesFiltered + } + log.Printf("Found interfaces: %#v", ifaces) + ifacesAll = append(ifacesAll, ifaces...) + } + } + + path := filepath.Join(packagePath, "extpoints.go") + log.Printf("Writing file %s", path) + err := renderExtpoints(path, packageName, ifacesAll) + if err != nil { + log.Fatalf("Could not write extpoints.go file: %s", err) + } +} diff --git a/vendor/github.com/progrium/go-extpoints/template.go b/vendor/github.com/progrium/go-extpoints/template.go new file mode 100644 index 0000000000..101af5f82d --- /dev/null +++ b/vendor/github.com/progrium/go-extpoints/template.go @@ -0,0 +1,183 @@ +package main + +var extpointsTemplate = `// generated by go-extpoints -- DO NOT EDIT +package {{.Package}} + +import ( + "reflect" + "runtime" + "strings" + "sync" +) + +var extRegistry = ®istryType{m: make(map[string]*extensionPoint)} + +type registryType struct { + sync.Mutex + m map[string]*extensionPoint +} + +// Top level registration + +func extensionTypes(extension interface{}) []string { + var ifaces []string + typ := reflect.TypeOf(extension) + for name, ep := range extRegistry.m { + if ep.iface.Kind() == reflect.Func && typ.AssignableTo(ep.iface) { + ifaces = append(ifaces, name) + } + if ep.iface.Kind() != reflect.Func && typ.Implements(ep.iface) { + ifaces = append(ifaces, name) + } + } + return ifaces +} + +func RegisterExtension(extension interface{}, name string) []string { + extRegistry.Lock() + defer extRegistry.Unlock() + var ifaces []string + for _, iface := range extensionTypes(extension) { + if extRegistry.m[iface].register(extension, name) { + ifaces = append(ifaces, iface) + } + } + return ifaces +} + +func UnregisterExtension(name string) []string { + extRegistry.Lock() + defer extRegistry.Unlock() + var ifaces []string + for iface, extpoint := range extRegistry.m { + if extpoint.unregister(name) { + ifaces = append(ifaces, iface) + } + } + return ifaces +} + + +// Base extension point + +type extensionPoint struct { + sync.Mutex + iface reflect.Type + extensions map[string]interface{} +} + +func newExtensionPoint(iface interface{}) *extensionPoint { + ep := &extensionPoint{ + iface: reflect.TypeOf(iface).Elem(), + extensions: make(map[string]interface{}), + } + extRegistry.Lock() + extRegistry.m[ep.iface.Name()] = ep + extRegistry.Unlock() + return ep +} + +func (ep *extensionPoint) lookup(name string) interface{} { + ep.Lock() + defer ep.Unlock() + ext, ok := ep.extensions[name] + if !ok { + return nil + } + return ext +} + +func (ep *extensionPoint) all() map[string]interface{} { + ep.Lock() + defer ep.Unlock() + all := make(map[string]interface{}) + for k, v := range ep.extensions { + all[k] = v + } + return all +} + +func (ep *extensionPoint) register(extension interface{}, name string) bool { + ep.Lock() + defer ep.Unlock() + if name == "" { + typ := reflect.TypeOf(extension) + if typ.Kind() == reflect.Func { + nameParts := strings.Split(runtime.FuncForPC( + reflect.ValueOf(extension).Pointer()).Name(), ".") + name = nameParts[len(nameParts)-1] + } else { + name = typ.Elem().Name() + } + } + _, exists := ep.extensions[name] + if exists { + return false + } + ep.extensions[name] = extension + return true +} + +func (ep *extensionPoint) unregister(name string) bool { + ep.Lock() + defer ep.Unlock() + _, exists := ep.extensions[name] + if !exists { + return false + } + delete(ep.extensions, name) + return true +} + +{{range .ExtensionPoints}}// {{.Name}} + +var {{.Var}} = &{{.Type}}{ + newExtensionPoint(new({{.Name}})), +} + +type {{.Type}} struct { + *extensionPoint +} + +func (ep *{{.Type}}) Unregister(name string) bool { + return ep.unregister(name) +} + +func (ep *{{.Type}}) Register(extension {{.Name}}, name string) bool { + return ep.register(extension, name) +} + +func (ep *{{.Type}}) Lookup(name string) {{.Name}} { + ext := ep.lookup(name) + if ext == nil { + return nil + } + return ext.({{.Name}}) +} + +func (ep *{{.Type}}) Select(names []string) []{{.Name}} { + var selected []{{.Name}} + for _, name := range names { + selected = append(selected, ep.Lookup(name)) + } + return selected +} + +func (ep *{{.Type}}) All() map[string]{{.Name}} { + all := make(map[string]{{.Name}}) + for k, v := range ep.all() { + all[k] = v.({{.Name}}) + } + return all +} + +func (ep *{{.Type}}) Names() []string { + var names []string + for k := range ep.all() { + names = append(names, k) + } + return names +} + + +{{end}}` diff --git a/vendor/github.com/smartystreets/go-aws-auth/.gitignore b/vendor/github.com/smartystreets/go-aws-auth/.gitignore new file mode 100644 index 0000000000..a09c56df5c --- /dev/null +++ b/vendor/github.com/smartystreets/go-aws-auth/.gitignore @@ -0,0 +1 @@ +/.idea diff --git a/vendor/github.com/smartystreets/go-aws-auth/CONTRIBUTING.md b/vendor/github.com/smartystreets/go-aws-auth/CONTRIBUTING.md new file mode 100644 index 0000000000..1820ecb331 --- /dev/null +++ b/vendor/github.com/smartystreets/go-aws-auth/CONTRIBUTING.md @@ -0,0 +1,12 @@ +# Contributing + +In general, the code posted to the [SmartyStreets github organization](https://github.com/smartystreets) is created to solve specific problems at SmartyStreets that are ancillary to our core products in the address verification industry and may or may not be useful to other organizations or developers. Our reason for posting said code isn't necessarily to solicit feedback or contributions from the community but more as a showcase of some of the approaches to solving problems we have adopted. + +Having stated that, we do consider issues raised by other githubbers as well as contributions submitted via pull requests. When submitting such a pull request, please follow these guidelines: + +- _Look before you leap:_ If the changes you plan to make are significant, it's in everyone's best interest for you to discuss them with a SmartyStreets team member prior to opening a pull request. +- _License and ownership:_ If modifying the `LICENSE.md` file, limit your changes to fixing typographical mistakes. Do NOT modify the actual terms in the license or the copyright by **SmartyStreets, LLC**. Code submitted to SmartyStreets projects becomes property of SmartyStreets and must be compatible with the associated license. +- _Testing:_ If the code you are submitting resides in packages/modules covered by automated tests, be sure to add passing tests that cover your changes and assert expected behavior and state. Submit the additional test cases as part of your change set. +- _Style:_ Match your approach to **naming** and **formatting** with the surrounding code. Basically, the code you submit shouldn't stand out. + - "Naming" refers to such constructs as variables, methods, functions, classes, structs, interfaces, packages, modules, directories, files, etc... + - "Formatting" refers to such constructs as whitespace, horizontal line length, vertical function length, vertical file length, indentation, curly braces, etc... diff --git a/vendor/github.com/olivere/elastic/LICENSE b/vendor/github.com/smartystreets/go-aws-auth/LICENSE.md similarity index 55% rename from vendor/github.com/olivere/elastic/LICENSE rename to vendor/github.com/smartystreets/go-aws-auth/LICENSE.md index 8b22cdb601..92959a64aa 100644 --- a/vendor/github.com/olivere/elastic/LICENSE +++ b/vendor/github.com/smartystreets/go-aws-auth/LICENSE.md @@ -1,20 +1,23 @@ -The MIT License (MIT) -Copyright © 2012-2015 Oliver Eilhard +Copyright (c) 2016 SmartyStreets Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the “Software”), to deal +of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be included -in all copies or substantial portions of the Software. +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. -THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -IN THE SOFTWARE. +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +NOTE: Various optional and subordinate components carry their own licensing +requirements and restrictions. Use of those components is subject to the terms +and conditions outlined the respective license of each component. diff --git a/vendor/github.com/smartystreets/go-aws-auth/README.md b/vendor/github.com/smartystreets/go-aws-auth/README.md new file mode 100644 index 0000000000..e2f6e405e6 --- /dev/null +++ b/vendor/github.com/smartystreets/go-aws-auth/README.md @@ -0,0 +1,86 @@ +go-aws-auth +=========== + +[![GoDoc](https://godoc.org/github.com/smartystreets/go-aws-auth?status.svg)](http://godoc.org/github.com/smartystreets/go-aws-auth) + +Go-AWS-Auth is a comprehensive, lightweight library for signing requests to Amazon Web Services. + +It's easy to use: simply build your HTTP request and call `awsauth.Sign(req)` before sending your request over the wire. + + + +### Supported signing mechanisms + +- [Signed Signature Version 2](http://docs.aws.amazon.com/general/latest/gr/signature-version-2.html) +- [Signed Signature Version 3](http://docs.aws.amazon.com/general/latest/gr/signing_aws_api_requests.html) +- [Signed Signature Version 4](http://docs.aws.amazon.com/general/latest/gr/signature-version-4.html) +- [Custom S3 Authentication Scheme](http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html) +- [Security Token Service](http://docs.aws.amazon.com/STS/latest/APIReference/Welcome.html) +- [S3 Query String Authentication](http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html#RESTAuthenticationQueryStringAuth) +- [IAM Role](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html#instance-metadata-security-credentials) + +For more info about AWS authentication, see the [comprehensive docs](http://docs.aws.amazon.com/general/latest/gr/signing_aws_api_requests.html) at AWS. + + +### Install + +Go get it: + + $ go get github.com/smartystreets/go-aws-auth + +Then import it: + + import "github.com/smartystreets/go-aws-auth" + + +### Using your AWS Credentials + +The library looks for credentials in this order: + +1. **Hard-code:** You can manually pass in an instance of `awsauth.Credentials` to any call to a signing function as a second argument: + + ```go + awsauth.Sign(req, awsauth.Credentials{ + AccessKeyID: "Access Key ID", + SecretAccessKey: "Secret Access Key", + SecurityToken: "Security Token", // STS (optional) + }) + ``` + + +2. **Environment variables:** Set the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables with your credentials. The library will automatically detect and use them. Optionally, you may also set the `AWS_SECURITY_TOKEN` environment variable if you are using temporary credentials from [STS](http://docs.aws.amazon.com/STS/latest/APIReference/Welcome.html). + +3. **IAM Role:** If running on EC2 and the credentials are neither hard-coded nor in the environment, go-aws-auth will detect the first IAM role assigned to the current EC2 instance and use those credentials. + +(Be especially careful hard-coding credentials into your application if the code is committed to source control.) + + + +### Signing requests + +Just make the request, have it signed, and perform the request as you normally would. + +```go +url := "https://iam.amazonaws.com/?Action=ListRoles&Version=2010-05-08" +client := new(http.Client) + +req, err := http.NewRequest("GET", url, nil) + +awsauth.Sign(req) // Automatically chooses the best signing mechanism for the service + +resp, err := client.Do(req) +``` + +You can use `Sign` to have the library choose the best signing algorithm depending on the service, or you can specify it manually if you know what you need: + +- `Sign2` +- `Sign3` +- `Sign4` +- `SignS3` (deprecated for Sign4) +- `SignS3Url` (for pre-signed S3 URLs; GETs only) + + + +### Contributing + +Please feel free to contribute! Bug fixes are more than welcome any time, as long as tests assert correct behavior. If you'd like to change an existing implementation or see a new feature, open an issue first so we can discuss it. Thanks to all contributors! diff --git a/vendor/github.com/smartystreets/go-aws-auth/awsauth.go b/vendor/github.com/smartystreets/go-aws-auth/awsauth.go new file mode 100644 index 0000000000..1f05d3b469 --- /dev/null +++ b/vendor/github.com/smartystreets/go-aws-auth/awsauth.go @@ -0,0 +1,220 @@ +// Package awsauth implements AWS request signing using Signed Signature Version 2, +// Signed Signature Version 3, and Signed Signature Version 4. Supports S3 and STS. +package awsauth + +import ( + "net/http" + "net/url" + "time" +) + +// Credentials stores the information necessary to authorize with AWS and it +// is from this information that requests are signed. +type Credentials struct { + AccessKeyID string + SecretAccessKey string + SecurityToken string `json:"Token"` + Expiration time.Time +} + +// Sign signs a request bound for AWS. It automatically chooses the best +// authentication scheme based on the service the request is going to. +func Sign(request *http.Request, credentials ...Credentials) *http.Request { + service, _ := serviceAndRegion(request.URL.Host) + signVersion := awsSignVersion[service] + + switch signVersion { + case 2: + return Sign2(request, credentials...) + case 3: + return Sign3(request, credentials...) + case 4: + return Sign4(request, credentials...) + case -1: + return SignS3(request, credentials...) + } + + return nil +} + +// Sign4 signs a request with Signed Signature Version 4. +func Sign4(request *http.Request, credentials ...Credentials) *http.Request { + keys := chooseKeys(credentials) + + // Add the X-Amz-Security-Token header when using STS + if keys.SecurityToken != "" { + request.Header.Set("X-Amz-Security-Token", keys.SecurityToken) + } + + prepareRequestV4(request) + meta := new(metadata) + + // Task 1 + hashedCanonReq := hashedCanonicalRequestV4(request, meta) + + // Task 2 + stringToSign := stringToSignV4(request, hashedCanonReq, meta) + + // Task 3 + signingKey := signingKeyV4(keys.SecretAccessKey, meta.date, meta.region, meta.service) + signature := signatureV4(signingKey, stringToSign) + + request.Header.Set("Authorization", buildAuthHeaderV4(signature, meta, keys)) + + return request +} + +// Sign3 signs a request with Signed Signature Version 3. +// If the service you're accessing supports Version 4, use that instead. +func Sign3(request *http.Request, credentials ...Credentials) *http.Request { + keys := chooseKeys(credentials) + + // Add the X-Amz-Security-Token header when using STS + if keys.SecurityToken != "" { + request.Header.Set("X-Amz-Security-Token", keys.SecurityToken) + } + + prepareRequestV3(request) + + // Task 1 + stringToSign := stringToSignV3(request) + + // Task 2 + signature := signatureV3(stringToSign, keys) + + // Task 3 + request.Header.Set("X-Amzn-Authorization", buildAuthHeaderV3(signature, keys)) + + return request +} + +// Sign2 signs a request with Signed Signature Version 2. +// If the service you're accessing supports Version 4, use that instead. +func Sign2(request *http.Request, credentials ...Credentials) *http.Request { + keys := chooseKeys(credentials) + + // Add the SecurityToken parameter when using STS + // This must be added before the signature is calculated + if keys.SecurityToken != "" { + values := url.Values{} + values.Set("SecurityToken", keys.SecurityToken) + augmentRequestQuery(request, values) + } + + prepareRequestV2(request, keys) + + stringToSign := stringToSignV2(request) + signature := signatureV2(stringToSign, keys) + + values := url.Values{} + values.Set("Signature", signature) + + augmentRequestQuery(request, values) + + return request +} + +// SignS3 signs a request bound for Amazon S3 using their custom +// HTTP authentication scheme. +func SignS3(request *http.Request, credentials ...Credentials) *http.Request { + keys := chooseKeys(credentials) + + // Add the X-Amz-Security-Token header when using STS + if keys.SecurityToken != "" { + request.Header.Set("X-Amz-Security-Token", keys.SecurityToken) + } + + prepareRequestS3(request) + + stringToSign := stringToSignS3(request) + signature := signatureS3(stringToSign, keys) + + authHeader := "AWS " + keys.AccessKeyID + ":" + signature + request.Header.Set("Authorization", authHeader) + + return request +} + +// SignS3Url signs a GET request for a resource on Amazon S3 by appending +// query string parameters containing credentials and signature. You must +// specify an expiration date for these signed requests. After that date, +// a request signed with this method will be rejected by S3. +func SignS3Url(request *http.Request, expire time.Time, credentials ...Credentials) *http.Request { + keys := chooseKeys(credentials) + + stringToSign := stringToSignS3Url("GET", expire, request.URL.Path) + signature := signatureS3(stringToSign, keys) + + query := request.URL.Query() + query.Set("AWSAccessKeyId", keys.AccessKeyID) + query.Set("Signature", signature) + query.Set("Expires", timeToUnixEpochString(expire)) + request.URL.RawQuery = query.Encode() + + return request +} + +// expired checks to see if the temporary credentials from an IAM role are +// within 4 minutes of expiration (The IAM documentation says that new keys +// will be provisioned 5 minutes before the old keys expire). Credentials +// that do not have an Expiration cannot expire. +func (this *Credentials) expired() bool { + if this.Expiration.IsZero() { + // Credentials with no expiration can't expire + return false + } + expireTime := this.Expiration.Add(-4 * time.Minute) + // if t - 4 mins is before now, true + if expireTime.Before(time.Now()) { + return true + } else { + return false + } +} + +type metadata struct { + algorithm string + credentialScope string + signedHeaders string + date string + region string + service string +} + +const ( + envAccessKey = "AWS_ACCESS_KEY" + envAccessKeyID = "AWS_ACCESS_KEY_ID" + envSecretKey = "AWS_SECRET_KEY" + envSecretAccessKey = "AWS_SECRET_ACCESS_KEY" + envSecurityToken = "AWS_SECURITY_TOKEN" +) + +var ( + awsSignVersion = map[string]int{ + "autoscaling": 4, + "cloudfront": 4, + "cloudformation": 4, + "cloudsearch": 4, + "monitoring": 4, + "dynamodb": 4, + "ec2": 2, + "elasticmapreduce": 4, + "elastictranscoder": 4, + "elasticache": 2, + "es": 4, + "glacier": 4, + "kinesis": 4, + "redshift": 4, + "rds": 4, + "sdb": 2, + "sns": 4, + "sqs": 4, + "s3": 4, + "elasticbeanstalk": 4, + "importexport": 2, + "iam": 4, + "route53": 3, + "elasticloadbalancing": 4, + "email": 3, + } +) diff --git a/vendor/github.com/smartystreets/go-aws-auth/common.go b/vendor/github.com/smartystreets/go-aws-auth/common.go new file mode 100644 index 0000000000..d4bea4703c --- /dev/null +++ b/vendor/github.com/smartystreets/go-aws-auth/common.go @@ -0,0 +1,311 @@ +package awsauth + +import ( + "bufio" + "bytes" + "crypto/hmac" + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "fmt" + "io/ioutil" + "net" + "net/http" + "net/url" + "os" + "strings" + "time" +) + +type location struct { + ec2 bool + checked bool +} + +var loc *location + +// serviceAndRegion parsers a hostname to find out which ones it is. +// http://docs.aws.amazon.com/general/latest/gr/rande.html +func serviceAndRegion(host string) (service string, region string) { + // These are the defaults if the hostname doesn't suggest something else + region = "us-east-1" + service = "s3" + + parts := strings.Split(host, ".") + if len(parts) == 4 { + // Either service.region.amazonaws.com or virtual-host.region.amazonaws.com + if parts[1] == "s3" { + service = "s3" + } else if strings.HasPrefix(parts[1], "s3-") { + region = parts[1][3:] + service = "s3" + } else { + service = parts[0] + region = parts[1] + } + } else if len(parts) == 5 { + service = parts[2] + region = parts[1] + } else { + // Either service.amazonaws.com or s3-region.amazonaws.com + if strings.HasPrefix(parts[0], "s3-") { + region = parts[0][3:] + } else { + service = parts[0] + } + } + + if region == "external-1" { + region = "us-east-1" + } + + return +} + +// newKeys produces a set of credentials based on the environment +func newKeys() (newCredentials Credentials) { + // First use credentials from environment variables + newCredentials.AccessKeyID = os.Getenv(envAccessKeyID) + if newCredentials.AccessKeyID == "" { + newCredentials.AccessKeyID = os.Getenv(envAccessKey) + } + + newCredentials.SecretAccessKey = os.Getenv(envSecretAccessKey) + if newCredentials.SecretAccessKey == "" { + newCredentials.SecretAccessKey = os.Getenv(envSecretKey) + } + + newCredentials.SecurityToken = os.Getenv(envSecurityToken) + + // If there is no Access Key and you are on EC2, get the key from the role + if (newCredentials.AccessKeyID == "" || newCredentials.SecretAccessKey == "") && onEC2() { + newCredentials = *getIAMRoleCredentials() + } + + // If the key is expiring, get a new key + if newCredentials.expired() && onEC2() { + newCredentials = *getIAMRoleCredentials() + } + + return newCredentials +} + +// checkKeys gets credentials depending on if any were passed in as an argument +// or it makes new ones based on the environment. +func chooseKeys(cred []Credentials) Credentials { + if len(cred) == 0 { + return newKeys() + } else { + return cred[0] + } +} + +// onEC2 checks to see if the program is running on an EC2 instance. +// It does this by looking for the EC2 metadata service. +// This caches that information in a struct so that it doesn't waste time. +func onEC2() bool { + if loc == nil { + loc = &location{} + } + if !(loc.checked) { + c, err := net.DialTimeout("tcp", "169.254.169.254:80", time.Millisecond*100) + + if err != nil { + loc.ec2 = false + } else { + c.Close() + loc.ec2 = true + } + loc.checked = true + } + + return loc.ec2 +} + +// getIAMRoleList gets a list of the roles that are available to this instance +func getIAMRoleList() []string { + + var roles []string + url := "http://169.254.169.254/latest/meta-data/iam/security-credentials/" + + client := &http.Client{} + + request, err := http.NewRequest("GET", url, nil) + + if err != nil { + return roles + } + + response, err := client.Do(request) + + if err != nil { + return roles + } + defer response.Body.Close() + + scanner := bufio.NewScanner(response.Body) + for scanner.Scan() { + roles = append(roles, scanner.Text()) + } + return roles +} + +func getIAMRoleCredentials() *Credentials { + + roles := getIAMRoleList() + + if len(roles) < 1 { + return &Credentials{} + } + + // Use the first role in the list + role := roles[0] + + url := "http://169.254.169.254/latest/meta-data/iam/security-credentials/" + + // Create the full URL of the role + var buffer bytes.Buffer + buffer.WriteString(url) + buffer.WriteString(role) + roleURL := buffer.String() + + // Get the role + roleRequest, err := http.NewRequest("GET", roleURL, nil) + + if err != nil { + return &Credentials{} + } + + client := &http.Client{} + roleResponse, err := client.Do(roleRequest) + + if err != nil { + return &Credentials{} + } + defer roleResponse.Body.Close() + + roleBuffer := new(bytes.Buffer) + roleBuffer.ReadFrom(roleResponse.Body) + + credentials := Credentials{} + + err = json.Unmarshal(roleBuffer.Bytes(), &credentials) + + if err != nil { + return &Credentials{} + } + + return &credentials + +} + +func augmentRequestQuery(request *http.Request, values url.Values) *http.Request { + for key, array := range request.URL.Query() { + for _, value := range array { + values.Set(key, value) + } + } + + request.URL.RawQuery = values.Encode() + + return request +} + +func hmacSHA256(key []byte, content string) []byte { + mac := hmac.New(sha256.New, key) + mac.Write([]byte(content)) + return mac.Sum(nil) +} + +func hmacSHA1(key []byte, content string) []byte { + mac := hmac.New(sha1.New, key) + mac.Write([]byte(content)) + return mac.Sum(nil) +} + +func hashSHA256(content []byte) string { + h := sha256.New() + h.Write(content) + return fmt.Sprintf("%x", h.Sum(nil)) +} + +func hashMD5(content []byte) string { + h := md5.New() + h.Write(content) + return base64.StdEncoding.EncodeToString(h.Sum(nil)) +} + +func readAndReplaceBody(request *http.Request) []byte { + if request.Body == nil { + return []byte{} + } + payload, _ := ioutil.ReadAll(request.Body) + request.Body = ioutil.NopCloser(bytes.NewReader(payload)) + return payload +} + +func concat(delim string, str ...string) string { + return strings.Join(str, delim) +} + +var now = func() time.Time { + return time.Now().UTC() +} + +func normuri(uri string) string { + parts := strings.Split(uri, "/") + for i := range parts { + parts[i] = encodePathFrag(parts[i]) + } + return strings.Join(parts, "/") +} + +func encodePathFrag(s string) string { + hexCount := 0 + for i := 0; i < len(s); i++ { + c := s[i] + if shouldEscape(c) { + hexCount++ + } + } + t := make([]byte, len(s)+2*hexCount) + j := 0 + for i := 0; i < len(s); i++ { + c := s[i] + if shouldEscape(c) { + t[j] = '%' + t[j+1] = "0123456789ABCDEF"[c>>4] + t[j+2] = "0123456789ABCDEF"[c&15] + j += 3 + } else { + t[j] = c + j++ + } + } + return string(t) +} + +func shouldEscape(c byte) bool { + if 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' { + return false + } + if '0' <= c && c <= '9' { + return false + } + if c == '-' || c == '_' || c == '.' || c == '~' { + return false + } + return true +} + +func normquery(v url.Values) string { + queryString := v.Encode() + + // Go encodes a space as '+' but Amazon requires '%20'. Luckily any '+' in the + // original query string has been percent escaped so all '+' chars that are left + // were originally spaces. + + return strings.Replace(queryString, "+", "%20", -1) +} diff --git a/vendor/github.com/smartystreets/go-aws-auth/s3.go b/vendor/github.com/smartystreets/go-aws-auth/s3.go new file mode 100644 index 0000000000..9be5ac92a0 --- /dev/null +++ b/vendor/github.com/smartystreets/go-aws-auth/s3.go @@ -0,0 +1,121 @@ +package awsauth + +import ( + "encoding/base64" + "net/http" + "sort" + "strconv" + "strings" + "time" +) + +func signatureS3(stringToSign string, keys Credentials) string { + hashed := hmacSHA1([]byte(keys.SecretAccessKey), stringToSign) + return base64.StdEncoding.EncodeToString(hashed) +} + +func stringToSignS3(request *http.Request) string { + str := request.Method + "\n" + + if request.Header.Get("Content-Md5") != "" { + str += request.Header.Get("Content-Md5") + } else { + body := readAndReplaceBody(request) + if len(body) > 0 { + str += hashMD5(body) + } + } + str += "\n" + + str += request.Header.Get("Content-Type") + "\n" + + if request.Header.Get("Date") != "" { + str += request.Header.Get("Date") + } else { + str += timestampS3() + } + + str += "\n" + + canonicalHeaders := canonicalAmzHeadersS3(request) + if canonicalHeaders != "" { + str += canonicalHeaders + } + + str += canonicalResourceS3(request) + + return str +} + +func stringToSignS3Url(method string, expire time.Time, path string) string { + return method + "\n\n\n" + timeToUnixEpochString(expire) + "\n" + path +} + +func timeToUnixEpochString(t time.Time) string { + return strconv.FormatInt(t.Unix(), 10) +} + +func canonicalAmzHeadersS3(request *http.Request) string { + var headers []string + + for header := range request.Header { + standardized := strings.ToLower(strings.TrimSpace(header)) + if strings.HasPrefix(standardized, "x-amz") { + headers = append(headers, standardized) + } + } + + sort.Strings(headers) + + for i, header := range headers { + headers[i] = header + ":" + strings.Replace(request.Header.Get(header), "\n", " ", -1) + } + + if len(headers) > 0 { + return strings.Join(headers, "\n") + "\n" + } else { + return "" + } +} + +func canonicalResourceS3(request *http.Request) string { + res := "" + + if isS3VirtualHostedStyle(request) { + bucketname := strings.Split(request.Host, ".")[0] + res += "/" + bucketname + } + + res += request.URL.Path + + for _, subres := range strings.Split(subresourcesS3, ",") { + if strings.HasPrefix(request.URL.RawQuery, subres) { + res += "?" + subres + } + } + + return res +} + +func prepareRequestS3(request *http.Request) *http.Request { + request.Header.Set("Date", timestampS3()) + if request.URL.Path == "" { + request.URL.Path += "/" + } + return request +} + +// Info: http://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html +func isS3VirtualHostedStyle(request *http.Request) bool { + service, _ := serviceAndRegion(request.Host) + return service == "s3" && strings.Count(request.Host, ".") == 3 +} + +func timestampS3() string { + return now().Format(timeFormatS3) +} + +const ( + timeFormatS3 = time.RFC1123Z + subresourcesS3 = "acl,lifecycle,location,logging,notification,partNumber,policy,requestPayment,torrent,uploadId,uploads,versionId,versioning,versions,website" +) diff --git a/vendor/github.com/smartystreets/go-aws-auth/sign2.go b/vendor/github.com/smartystreets/go-aws-auth/sign2.go new file mode 100644 index 0000000000..ecfa394ba2 --- /dev/null +++ b/vendor/github.com/smartystreets/go-aws-auth/sign2.go @@ -0,0 +1,50 @@ +package awsauth + +import ( + "encoding/base64" + "net/http" + "net/url" + "strings" +) + +func prepareRequestV2(request *http.Request, keys Credentials) *http.Request { + + keyID := keys.AccessKeyID + + values := url.Values{} + values.Set("AWSAccessKeyId", keyID) + values.Set("SignatureVersion", "2") + values.Set("SignatureMethod", "HmacSHA256") + values.Set("Timestamp", timestampV2()) + + augmentRequestQuery(request, values) + + if request.URL.Path == "" { + request.URL.Path += "/" + } + + return request +} + +func stringToSignV2(request *http.Request) string { + str := request.Method + "\n" + str += strings.ToLower(request.URL.Host) + "\n" + str += request.URL.Path + "\n" + str += canonicalQueryStringV2(request) + return str +} + +func signatureV2(strToSign string, keys Credentials) string { + hashed := hmacSHA256([]byte(keys.SecretAccessKey), strToSign) + return base64.StdEncoding.EncodeToString(hashed) +} + +func canonicalQueryStringV2(request *http.Request) string { + return request.URL.RawQuery +} + +func timestampV2() string { + return now().Format(timeFormatV2) +} + +const timeFormatV2 = "2006-01-02T15:04:05" diff --git a/vendor/github.com/smartystreets/go-aws-auth/sign3.go b/vendor/github.com/smartystreets/go-aws-auth/sign3.go new file mode 100644 index 0000000000..509e2a19f1 --- /dev/null +++ b/vendor/github.com/smartystreets/go-aws-auth/sign3.go @@ -0,0 +1,58 @@ +// Thanks to Michael Vierling for contributing sign3.go + +package awsauth + +import ( + "encoding/base64" + "net/http" + "time" +) + +func stringToSignV3(request *http.Request) string { + // TASK 1. http://docs.aws.amazon.com/Route53/latest/DeveloperGuide/RESTAuthentication.html#StringToSign + + return request.Header.Get("Date") + request.Header.Get("x-amz-nonce") +} + +func signatureV3(stringToSign string, keys Credentials) string { + // TASK 2. http://docs.aws.amazon.com/Route53/latest/DeveloperGuide/RESTAuthentication.html#Signature + + hash := hmacSHA256([]byte(keys.SecretAccessKey), stringToSign) + return base64.StdEncoding.EncodeToString(hash) +} + +func buildAuthHeaderV3(signature string, keys Credentials) string { + // TASK 3. http://docs.aws.amazon.com/Route53/latest/DeveloperGuide/RESTAuthentication.html#AuthorizationHeader + + return "AWS3-HTTPS AWSAccessKeyId=" + keys.AccessKeyID + + ", Algorithm=HmacSHA256" + + ", Signature=" + signature +} + +func prepareRequestV3(request *http.Request) *http.Request { + ts := timestampV3() + necessaryDefaults := map[string]string{ + "Content-Type": "application/x-www-form-urlencoded; charset=utf-8", + "x-amz-date": ts, + "Date": ts, + "x-amz-nonce": "", + } + + for header, value := range necessaryDefaults { + if request.Header.Get(header) == "" { + request.Header.Set(header, value) + } + } + + if request.URL.Path == "" { + request.URL.Path += "/" + } + + return request +} + +func timestampV3() string { + return now().Format(timeFormatV3) +} + +const timeFormatV3 = time.RFC1123 diff --git a/vendor/github.com/smartystreets/go-aws-auth/sign4.go b/vendor/github.com/smartystreets/go-aws-auth/sign4.go new file mode 100644 index 0000000000..89770bdca8 --- /dev/null +++ b/vendor/github.com/smartystreets/go-aws-auth/sign4.go @@ -0,0 +1,117 @@ +package awsauth + +import ( + "encoding/hex" + "net/http" + "sort" + "strings" +) + +func hashedCanonicalRequestV4(request *http.Request, meta *metadata) string { + // TASK 1. http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html + + payload := readAndReplaceBody(request) + payloadHash := hashSHA256(payload) + request.Header.Set("X-Amz-Content-Sha256", payloadHash) + + // Set this in header values to make it appear in the range of headers to sign + request.Header.Set("Host", request.Host) + + var sortedHeaderKeys []string + for key, _ := range request.Header { + switch key { + case "Content-Type", "Content-Md5", "Host": + default: + if !strings.HasPrefix(key, "X-Amz-") { + continue + } + } + sortedHeaderKeys = append(sortedHeaderKeys, strings.ToLower(key)) + } + sort.Strings(sortedHeaderKeys) + + var headersToSign string + for _, key := range sortedHeaderKeys { + value := strings.TrimSpace(request.Header.Get(key)) + if key == "host" { + //AWS does not include port in signing request. + if strings.Contains(value, ":") { + split := strings.Split(value, ":") + port := split[1] + if port == "80" || port == "443" { + value = split[0] + } + } + } + headersToSign += key + ":" + value + "\n" + } + meta.signedHeaders = concat(";", sortedHeaderKeys...) + canonicalRequest := concat("\n", request.Method, normuri(request.URL.Path), normquery(request.URL.Query()), headersToSign, meta.signedHeaders, payloadHash) + + return hashSHA256([]byte(canonicalRequest)) +} + +func stringToSignV4(request *http.Request, hashedCanonReq string, meta *metadata) string { + // TASK 2. http://docs.aws.amazon.com/general/latest/gr/sigv4-create-string-to-sign.html + + requestTs := request.Header.Get("X-Amz-Date") + + meta.algorithm = "AWS4-HMAC-SHA256" + meta.service, meta.region = serviceAndRegion(request.Host) + meta.date = tsDateV4(requestTs) + meta.credentialScope = concat("/", meta.date, meta.region, meta.service, "aws4_request") + + return concat("\n", meta.algorithm, requestTs, meta.credentialScope, hashedCanonReq) +} + +func signatureV4(signingKey []byte, stringToSign string) string { + // TASK 3. http://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html + + return hex.EncodeToString(hmacSHA256(signingKey, stringToSign)) +} + +func prepareRequestV4(request *http.Request) *http.Request { + necessaryDefaults := map[string]string{ + "Content-Type": "application/x-www-form-urlencoded; charset=utf-8", + "X-Amz-Date": timestampV4(), + } + + for header, value := range necessaryDefaults { + if request.Header.Get(header) == "" { + request.Header.Set(header, value) + } + } + + if request.URL.Path == "" { + request.URL.Path += "/" + } + + return request +} + +func signingKeyV4(secretKey, date, region, service string) []byte { + kDate := hmacSHA256([]byte("AWS4"+secretKey), date) + kRegion := hmacSHA256(kDate, region) + kService := hmacSHA256(kRegion, service) + kSigning := hmacSHA256(kService, "aws4_request") + return kSigning +} + +func buildAuthHeaderV4(signature string, meta *metadata, keys Credentials) string { + credential := keys.AccessKeyID + "/" + meta.credentialScope + + return meta.algorithm + + " Credential=" + credential + + ", SignedHeaders=" + meta.signedHeaders + + ", Signature=" + signature +} + +func timestampV4() string { + return now().Format(timeFormatV4) +} + +func tsDateV4(timestamp string) string { + return timestamp[:8] +} + +const timeFormatV4 = "20060102T150405Z" diff --git a/vendor/github.com/olivere/elastic/.gitignore b/vendor/gopkg.in/olivere/elastic.v3/.gitignore similarity index 100% rename from vendor/github.com/olivere/elastic/.gitignore rename to vendor/gopkg.in/olivere/elastic.v3/.gitignore diff --git a/vendor/github.com/olivere/elastic/.travis.yml b/vendor/gopkg.in/olivere/elastic.v3/.travis.yml similarity index 100% rename from vendor/github.com/olivere/elastic/.travis.yml rename to vendor/gopkg.in/olivere/elastic.v3/.travis.yml diff --git a/vendor/github.com/olivere/elastic/CHANGELOG-3.0.md b/vendor/gopkg.in/olivere/elastic.v3/CHANGELOG-3.0.md similarity index 100% rename from vendor/github.com/olivere/elastic/CHANGELOG-3.0.md rename to vendor/gopkg.in/olivere/elastic.v3/CHANGELOG-3.0.md diff --git a/vendor/github.com/olivere/elastic/CONTRIBUTING.md b/vendor/gopkg.in/olivere/elastic.v3/CONTRIBUTING.md similarity index 100% rename from vendor/github.com/olivere/elastic/CONTRIBUTING.md rename to vendor/gopkg.in/olivere/elastic.v3/CONTRIBUTING.md diff --git a/vendor/github.com/olivere/elastic/CONTRIBUTORS b/vendor/gopkg.in/olivere/elastic.v3/CONTRIBUTORS similarity index 100% rename from vendor/github.com/olivere/elastic/CONTRIBUTORS rename to vendor/gopkg.in/olivere/elastic.v3/CONTRIBUTORS diff --git a/vendor/github.com/olivere/elastic/ISSUE_TEMPLATE.md b/vendor/gopkg.in/olivere/elastic.v3/ISSUE_TEMPLATE.md similarity index 100% rename from vendor/github.com/olivere/elastic/ISSUE_TEMPLATE.md rename to vendor/gopkg.in/olivere/elastic.v3/ISSUE_TEMPLATE.md diff --git a/vendor/github.com/olivere/elastic/README.md b/vendor/gopkg.in/olivere/elastic.v3/README.md similarity index 100% rename from vendor/github.com/olivere/elastic/README.md rename to vendor/gopkg.in/olivere/elastic.v3/README.md diff --git a/vendor/github.com/olivere/elastic/bulk.go b/vendor/gopkg.in/olivere/elastic.v3/bulk.go similarity index 100% rename from vendor/github.com/olivere/elastic/bulk.go rename to vendor/gopkg.in/olivere/elastic.v3/bulk.go diff --git a/vendor/github.com/olivere/elastic/bulk_delete_request.go b/vendor/gopkg.in/olivere/elastic.v3/bulk_delete_request.go similarity index 100% rename from vendor/github.com/olivere/elastic/bulk_delete_request.go rename to vendor/gopkg.in/olivere/elastic.v3/bulk_delete_request.go diff --git a/vendor/github.com/olivere/elastic/bulk_index_request.go b/vendor/gopkg.in/olivere/elastic.v3/bulk_index_request.go similarity index 100% rename from vendor/github.com/olivere/elastic/bulk_index_request.go rename to vendor/gopkg.in/olivere/elastic.v3/bulk_index_request.go diff --git a/vendor/github.com/olivere/elastic/bulk_processor.go b/vendor/gopkg.in/olivere/elastic.v3/bulk_processor.go similarity index 100% rename from vendor/github.com/olivere/elastic/bulk_processor.go rename to vendor/gopkg.in/olivere/elastic.v3/bulk_processor.go diff --git a/vendor/github.com/olivere/elastic/bulk_request.go b/vendor/gopkg.in/olivere/elastic.v3/bulk_request.go similarity index 100% rename from vendor/github.com/olivere/elastic/bulk_request.go rename to vendor/gopkg.in/olivere/elastic.v3/bulk_request.go diff --git a/vendor/github.com/olivere/elastic/bulk_update_request.go b/vendor/gopkg.in/olivere/elastic.v3/bulk_update_request.go similarity index 100% rename from vendor/github.com/olivere/elastic/bulk_update_request.go rename to vendor/gopkg.in/olivere/elastic.v3/bulk_update_request.go diff --git a/vendor/github.com/olivere/elastic/canonicalize.go b/vendor/gopkg.in/olivere/elastic.v3/canonicalize.go similarity index 100% rename from vendor/github.com/olivere/elastic/canonicalize.go rename to vendor/gopkg.in/olivere/elastic.v3/canonicalize.go diff --git a/vendor/github.com/olivere/elastic/clear_scroll.go b/vendor/gopkg.in/olivere/elastic.v3/clear_scroll.go similarity index 100% rename from vendor/github.com/olivere/elastic/clear_scroll.go rename to vendor/gopkg.in/olivere/elastic.v3/clear_scroll.go diff --git a/vendor/github.com/olivere/elastic/client.go b/vendor/gopkg.in/olivere/elastic.v3/client.go similarity index 100% rename from vendor/github.com/olivere/elastic/client.go rename to vendor/gopkg.in/olivere/elastic.v3/client.go diff --git a/vendor/github.com/olivere/elastic/cluster_health.go b/vendor/gopkg.in/olivere/elastic.v3/cluster_health.go similarity index 100% rename from vendor/github.com/olivere/elastic/cluster_health.go rename to vendor/gopkg.in/olivere/elastic.v3/cluster_health.go diff --git a/vendor/github.com/olivere/elastic/cluster_state.go b/vendor/gopkg.in/olivere/elastic.v3/cluster_state.go similarity index 100% rename from vendor/github.com/olivere/elastic/cluster_state.go rename to vendor/gopkg.in/olivere/elastic.v3/cluster_state.go diff --git a/vendor/github.com/olivere/elastic/cluster_stats.go b/vendor/gopkg.in/olivere/elastic.v3/cluster_stats.go similarity index 100% rename from vendor/github.com/olivere/elastic/cluster_stats.go rename to vendor/gopkg.in/olivere/elastic.v3/cluster_stats.go diff --git a/vendor/github.com/olivere/elastic/connection.go b/vendor/gopkg.in/olivere/elastic.v3/connection.go similarity index 100% rename from vendor/github.com/olivere/elastic/connection.go rename to vendor/gopkg.in/olivere/elastic.v3/connection.go diff --git a/vendor/github.com/olivere/elastic/count.go b/vendor/gopkg.in/olivere/elastic.v3/count.go similarity index 100% rename from vendor/github.com/olivere/elastic/count.go rename to vendor/gopkg.in/olivere/elastic.v3/count.go diff --git a/vendor/github.com/olivere/elastic/decoder.go b/vendor/gopkg.in/olivere/elastic.v3/decoder.go similarity index 100% rename from vendor/github.com/olivere/elastic/decoder.go rename to vendor/gopkg.in/olivere/elastic.v3/decoder.go diff --git a/vendor/github.com/olivere/elastic/delete.go b/vendor/gopkg.in/olivere/elastic.v3/delete.go similarity index 100% rename from vendor/github.com/olivere/elastic/delete.go rename to vendor/gopkg.in/olivere/elastic.v3/delete.go diff --git a/vendor/github.com/olivere/elastic/delete_by_query.go b/vendor/gopkg.in/olivere/elastic.v3/delete_by_query.go similarity index 100% rename from vendor/github.com/olivere/elastic/delete_by_query.go rename to vendor/gopkg.in/olivere/elastic.v3/delete_by_query.go diff --git a/vendor/github.com/olivere/elastic/delete_template.go b/vendor/gopkg.in/olivere/elastic.v3/delete_template.go similarity index 100% rename from vendor/github.com/olivere/elastic/delete_template.go rename to vendor/gopkg.in/olivere/elastic.v3/delete_template.go diff --git a/vendor/github.com/olivere/elastic/doc.go b/vendor/gopkg.in/olivere/elastic.v3/doc.go similarity index 100% rename from vendor/github.com/olivere/elastic/doc.go rename to vendor/gopkg.in/olivere/elastic.v3/doc.go diff --git a/vendor/github.com/olivere/elastic/errors.go b/vendor/gopkg.in/olivere/elastic.v3/errors.go similarity index 100% rename from vendor/github.com/olivere/elastic/errors.go rename to vendor/gopkg.in/olivere/elastic.v3/errors.go diff --git a/vendor/github.com/olivere/elastic/exists.go b/vendor/gopkg.in/olivere/elastic.v3/exists.go similarity index 100% rename from vendor/github.com/olivere/elastic/exists.go rename to vendor/gopkg.in/olivere/elastic.v3/exists.go diff --git a/vendor/github.com/olivere/elastic/explain.go b/vendor/gopkg.in/olivere/elastic.v3/explain.go similarity index 100% rename from vendor/github.com/olivere/elastic/explain.go rename to vendor/gopkg.in/olivere/elastic.v3/explain.go diff --git a/vendor/github.com/olivere/elastic/fetch_source_context.go b/vendor/gopkg.in/olivere/elastic.v3/fetch_source_context.go similarity index 100% rename from vendor/github.com/olivere/elastic/fetch_source_context.go rename to vendor/gopkg.in/olivere/elastic.v3/fetch_source_context.go diff --git a/vendor/github.com/olivere/elastic/field_stats.go b/vendor/gopkg.in/olivere/elastic.v3/field_stats.go similarity index 100% rename from vendor/github.com/olivere/elastic/field_stats.go rename to vendor/gopkg.in/olivere/elastic.v3/field_stats.go diff --git a/vendor/github.com/olivere/elastic/geo_point.go b/vendor/gopkg.in/olivere/elastic.v3/geo_point.go similarity index 100% rename from vendor/github.com/olivere/elastic/geo_point.go rename to vendor/gopkg.in/olivere/elastic.v3/geo_point.go diff --git a/vendor/github.com/olivere/elastic/get.go b/vendor/gopkg.in/olivere/elastic.v3/get.go similarity index 100% rename from vendor/github.com/olivere/elastic/get.go rename to vendor/gopkg.in/olivere/elastic.v3/get.go diff --git a/vendor/github.com/olivere/elastic/get_template.go b/vendor/gopkg.in/olivere/elastic.v3/get_template.go similarity index 100% rename from vendor/github.com/olivere/elastic/get_template.go rename to vendor/gopkg.in/olivere/elastic.v3/get_template.go diff --git a/vendor/github.com/olivere/elastic/highlight.go b/vendor/gopkg.in/olivere/elastic.v3/highlight.go similarity index 100% rename from vendor/github.com/olivere/elastic/highlight.go rename to vendor/gopkg.in/olivere/elastic.v3/highlight.go diff --git a/vendor/github.com/olivere/elastic/index.go b/vendor/gopkg.in/olivere/elastic.v3/index.go similarity index 100% rename from vendor/github.com/olivere/elastic/index.go rename to vendor/gopkg.in/olivere/elastic.v3/index.go diff --git a/vendor/github.com/olivere/elastic/indices_close.go b/vendor/gopkg.in/olivere/elastic.v3/indices_close.go similarity index 100% rename from vendor/github.com/olivere/elastic/indices_close.go rename to vendor/gopkg.in/olivere/elastic.v3/indices_close.go diff --git a/vendor/github.com/olivere/elastic/indices_create.go b/vendor/gopkg.in/olivere/elastic.v3/indices_create.go similarity index 100% rename from vendor/github.com/olivere/elastic/indices_create.go rename to vendor/gopkg.in/olivere/elastic.v3/indices_create.go diff --git a/vendor/github.com/olivere/elastic/indices_delete.go b/vendor/gopkg.in/olivere/elastic.v3/indices_delete.go similarity index 100% rename from vendor/github.com/olivere/elastic/indices_delete.go rename to vendor/gopkg.in/olivere/elastic.v3/indices_delete.go diff --git a/vendor/github.com/olivere/elastic/indices_delete_template.go b/vendor/gopkg.in/olivere/elastic.v3/indices_delete_template.go similarity index 100% rename from vendor/github.com/olivere/elastic/indices_delete_template.go rename to vendor/gopkg.in/olivere/elastic.v3/indices_delete_template.go diff --git a/vendor/github.com/olivere/elastic/indices_delete_warmer.go b/vendor/gopkg.in/olivere/elastic.v3/indices_delete_warmer.go similarity index 100% rename from vendor/github.com/olivere/elastic/indices_delete_warmer.go rename to vendor/gopkg.in/olivere/elastic.v3/indices_delete_warmer.go diff --git a/vendor/github.com/olivere/elastic/indices_exists.go b/vendor/gopkg.in/olivere/elastic.v3/indices_exists.go similarity index 100% rename from vendor/github.com/olivere/elastic/indices_exists.go rename to vendor/gopkg.in/olivere/elastic.v3/indices_exists.go diff --git a/vendor/github.com/olivere/elastic/indices_exists_template.go b/vendor/gopkg.in/olivere/elastic.v3/indices_exists_template.go similarity index 100% rename from vendor/github.com/olivere/elastic/indices_exists_template.go rename to vendor/gopkg.in/olivere/elastic.v3/indices_exists_template.go diff --git a/vendor/github.com/olivere/elastic/indices_exists_type.go b/vendor/gopkg.in/olivere/elastic.v3/indices_exists_type.go similarity index 100% rename from vendor/github.com/olivere/elastic/indices_exists_type.go rename to vendor/gopkg.in/olivere/elastic.v3/indices_exists_type.go diff --git a/vendor/github.com/olivere/elastic/indices_flush.go b/vendor/gopkg.in/olivere/elastic.v3/indices_flush.go similarity index 100% rename from vendor/github.com/olivere/elastic/indices_flush.go rename to vendor/gopkg.in/olivere/elastic.v3/indices_flush.go diff --git a/vendor/github.com/olivere/elastic/indices_forcemerge.go b/vendor/gopkg.in/olivere/elastic.v3/indices_forcemerge.go similarity index 100% rename from vendor/github.com/olivere/elastic/indices_forcemerge.go rename to vendor/gopkg.in/olivere/elastic.v3/indices_forcemerge.go diff --git a/vendor/github.com/olivere/elastic/indices_get.go b/vendor/gopkg.in/olivere/elastic.v3/indices_get.go similarity index 100% rename from vendor/github.com/olivere/elastic/indices_get.go rename to vendor/gopkg.in/olivere/elastic.v3/indices_get.go diff --git a/vendor/github.com/olivere/elastic/indices_get_aliases.go b/vendor/gopkg.in/olivere/elastic.v3/indices_get_aliases.go similarity index 100% rename from vendor/github.com/olivere/elastic/indices_get_aliases.go rename to vendor/gopkg.in/olivere/elastic.v3/indices_get_aliases.go diff --git a/vendor/github.com/olivere/elastic/indices_get_mapping.go b/vendor/gopkg.in/olivere/elastic.v3/indices_get_mapping.go similarity index 100% rename from vendor/github.com/olivere/elastic/indices_get_mapping.go rename to vendor/gopkg.in/olivere/elastic.v3/indices_get_mapping.go diff --git a/vendor/github.com/olivere/elastic/indices_get_settings.go b/vendor/gopkg.in/olivere/elastic.v3/indices_get_settings.go similarity index 100% rename from vendor/github.com/olivere/elastic/indices_get_settings.go rename to vendor/gopkg.in/olivere/elastic.v3/indices_get_settings.go diff --git a/vendor/github.com/olivere/elastic/indices_get_template.go b/vendor/gopkg.in/olivere/elastic.v3/indices_get_template.go similarity index 100% rename from vendor/github.com/olivere/elastic/indices_get_template.go rename to vendor/gopkg.in/olivere/elastic.v3/indices_get_template.go diff --git a/vendor/github.com/olivere/elastic/indices_get_warmer.go b/vendor/gopkg.in/olivere/elastic.v3/indices_get_warmer.go similarity index 100% rename from vendor/github.com/olivere/elastic/indices_get_warmer.go rename to vendor/gopkg.in/olivere/elastic.v3/indices_get_warmer.go diff --git a/vendor/github.com/olivere/elastic/indices_open.go b/vendor/gopkg.in/olivere/elastic.v3/indices_open.go similarity index 100% rename from vendor/github.com/olivere/elastic/indices_open.go rename to vendor/gopkg.in/olivere/elastic.v3/indices_open.go diff --git a/vendor/github.com/olivere/elastic/indices_put_alias.go b/vendor/gopkg.in/olivere/elastic.v3/indices_put_alias.go similarity index 100% rename from vendor/github.com/olivere/elastic/indices_put_alias.go rename to vendor/gopkg.in/olivere/elastic.v3/indices_put_alias.go diff --git a/vendor/github.com/olivere/elastic/indices_put_mapping.go b/vendor/gopkg.in/olivere/elastic.v3/indices_put_mapping.go similarity index 100% rename from vendor/github.com/olivere/elastic/indices_put_mapping.go rename to vendor/gopkg.in/olivere/elastic.v3/indices_put_mapping.go diff --git a/vendor/github.com/olivere/elastic/indices_put_settings.go b/vendor/gopkg.in/olivere/elastic.v3/indices_put_settings.go similarity index 100% rename from vendor/github.com/olivere/elastic/indices_put_settings.go rename to vendor/gopkg.in/olivere/elastic.v3/indices_put_settings.go diff --git a/vendor/github.com/olivere/elastic/indices_put_template.go b/vendor/gopkg.in/olivere/elastic.v3/indices_put_template.go similarity index 100% rename from vendor/github.com/olivere/elastic/indices_put_template.go rename to vendor/gopkg.in/olivere/elastic.v3/indices_put_template.go diff --git a/vendor/github.com/olivere/elastic/indices_put_warmer.go b/vendor/gopkg.in/olivere/elastic.v3/indices_put_warmer.go similarity index 100% rename from vendor/github.com/olivere/elastic/indices_put_warmer.go rename to vendor/gopkg.in/olivere/elastic.v3/indices_put_warmer.go diff --git a/vendor/github.com/olivere/elastic/indices_refresh.go b/vendor/gopkg.in/olivere/elastic.v3/indices_refresh.go similarity index 100% rename from vendor/github.com/olivere/elastic/indices_refresh.go rename to vendor/gopkg.in/olivere/elastic.v3/indices_refresh.go diff --git a/vendor/github.com/olivere/elastic/indices_stats.go b/vendor/gopkg.in/olivere/elastic.v3/indices_stats.go similarity index 100% rename from vendor/github.com/olivere/elastic/indices_stats.go rename to vendor/gopkg.in/olivere/elastic.v3/indices_stats.go diff --git a/vendor/github.com/olivere/elastic/inner_hit.go b/vendor/gopkg.in/olivere/elastic.v3/inner_hit.go similarity index 100% rename from vendor/github.com/olivere/elastic/inner_hit.go rename to vendor/gopkg.in/olivere/elastic.v3/inner_hit.go diff --git a/vendor/github.com/olivere/elastic/logger.go b/vendor/gopkg.in/olivere/elastic.v3/logger.go similarity index 100% rename from vendor/github.com/olivere/elastic/logger.go rename to vendor/gopkg.in/olivere/elastic.v3/logger.go diff --git a/vendor/github.com/olivere/elastic/mget.go b/vendor/gopkg.in/olivere/elastic.v3/mget.go similarity index 100% rename from vendor/github.com/olivere/elastic/mget.go rename to vendor/gopkg.in/olivere/elastic.v3/mget.go diff --git a/vendor/github.com/olivere/elastic/msearch.go b/vendor/gopkg.in/olivere/elastic.v3/msearch.go similarity index 100% rename from vendor/github.com/olivere/elastic/msearch.go rename to vendor/gopkg.in/olivere/elastic.v3/msearch.go diff --git a/vendor/github.com/olivere/elastic/mtermvectors.go b/vendor/gopkg.in/olivere/elastic.v3/mtermvectors.go similarity index 100% rename from vendor/github.com/olivere/elastic/mtermvectors.go rename to vendor/gopkg.in/olivere/elastic.v3/mtermvectors.go diff --git a/vendor/github.com/olivere/elastic/nodes_info.go b/vendor/gopkg.in/olivere/elastic.v3/nodes_info.go similarity index 100% rename from vendor/github.com/olivere/elastic/nodes_info.go rename to vendor/gopkg.in/olivere/elastic.v3/nodes_info.go diff --git a/vendor/github.com/olivere/elastic/optimize.go b/vendor/gopkg.in/olivere/elastic.v3/optimize.go similarity index 100% rename from vendor/github.com/olivere/elastic/optimize.go rename to vendor/gopkg.in/olivere/elastic.v3/optimize.go diff --git a/vendor/github.com/olivere/elastic/percolate.go b/vendor/gopkg.in/olivere/elastic.v3/percolate.go similarity index 100% rename from vendor/github.com/olivere/elastic/percolate.go rename to vendor/gopkg.in/olivere/elastic.v3/percolate.go diff --git a/vendor/github.com/olivere/elastic/ping.go b/vendor/gopkg.in/olivere/elastic.v3/ping.go similarity index 100% rename from vendor/github.com/olivere/elastic/ping.go rename to vendor/gopkg.in/olivere/elastic.v3/ping.go diff --git a/vendor/github.com/olivere/elastic/plugins.go b/vendor/gopkg.in/olivere/elastic.v3/plugins.go similarity index 100% rename from vendor/github.com/olivere/elastic/plugins.go rename to vendor/gopkg.in/olivere/elastic.v3/plugins.go diff --git a/vendor/github.com/olivere/elastic/query.go b/vendor/gopkg.in/olivere/elastic.v3/query.go similarity index 100% rename from vendor/github.com/olivere/elastic/query.go rename to vendor/gopkg.in/olivere/elastic.v3/query.go diff --git a/vendor/github.com/olivere/elastic/reindex.go b/vendor/gopkg.in/olivere/elastic.v3/reindex.go similarity index 100% rename from vendor/github.com/olivere/elastic/reindex.go rename to vendor/gopkg.in/olivere/elastic.v3/reindex.go diff --git a/vendor/github.com/olivere/elastic/reindexer.go b/vendor/gopkg.in/olivere/elastic.v3/reindexer.go similarity index 100% rename from vendor/github.com/olivere/elastic/reindexer.go rename to vendor/gopkg.in/olivere/elastic.v3/reindexer.go diff --git a/vendor/github.com/olivere/elastic/request.go b/vendor/gopkg.in/olivere/elastic.v3/request.go similarity index 100% rename from vendor/github.com/olivere/elastic/request.go rename to vendor/gopkg.in/olivere/elastic.v3/request.go diff --git a/vendor/github.com/olivere/elastic/rescore.go b/vendor/gopkg.in/olivere/elastic.v3/rescore.go similarity index 100% rename from vendor/github.com/olivere/elastic/rescore.go rename to vendor/gopkg.in/olivere/elastic.v3/rescore.go diff --git a/vendor/github.com/olivere/elastic/rescorer.go b/vendor/gopkg.in/olivere/elastic.v3/rescorer.go similarity index 100% rename from vendor/github.com/olivere/elastic/rescorer.go rename to vendor/gopkg.in/olivere/elastic.v3/rescorer.go diff --git a/vendor/github.com/olivere/elastic/response.go b/vendor/gopkg.in/olivere/elastic.v3/response.go similarity index 100% rename from vendor/github.com/olivere/elastic/response.go rename to vendor/gopkg.in/olivere/elastic.v3/response.go diff --git a/vendor/github.com/olivere/elastic/scan.go b/vendor/gopkg.in/olivere/elastic.v3/scan.go similarity index 100% rename from vendor/github.com/olivere/elastic/scan.go rename to vendor/gopkg.in/olivere/elastic.v3/scan.go diff --git a/vendor/github.com/olivere/elastic/script.go b/vendor/gopkg.in/olivere/elastic.v3/script.go similarity index 100% rename from vendor/github.com/olivere/elastic/script.go rename to vendor/gopkg.in/olivere/elastic.v3/script.go diff --git a/vendor/github.com/olivere/elastic/scroll.go b/vendor/gopkg.in/olivere/elastic.v3/scroll.go similarity index 100% rename from vendor/github.com/olivere/elastic/scroll.go rename to vendor/gopkg.in/olivere/elastic.v3/scroll.go diff --git a/vendor/github.com/olivere/elastic/search.go b/vendor/gopkg.in/olivere/elastic.v3/search.go similarity index 100% rename from vendor/github.com/olivere/elastic/search.go rename to vendor/gopkg.in/olivere/elastic.v3/search.go diff --git a/vendor/github.com/olivere/elastic/search_aggs.go b/vendor/gopkg.in/olivere/elastic.v3/search_aggs.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_aggs.go rename to vendor/gopkg.in/olivere/elastic.v3/search_aggs.go diff --git a/vendor/github.com/olivere/elastic/search_aggs_bucket_children.go b/vendor/gopkg.in/olivere/elastic.v3/search_aggs_bucket_children.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_aggs_bucket_children.go rename to vendor/gopkg.in/olivere/elastic.v3/search_aggs_bucket_children.go diff --git a/vendor/github.com/olivere/elastic/search_aggs_bucket_date_histogram.go b/vendor/gopkg.in/olivere/elastic.v3/search_aggs_bucket_date_histogram.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_aggs_bucket_date_histogram.go rename to vendor/gopkg.in/olivere/elastic.v3/search_aggs_bucket_date_histogram.go diff --git a/vendor/github.com/olivere/elastic/search_aggs_bucket_date_range.go b/vendor/gopkg.in/olivere/elastic.v3/search_aggs_bucket_date_range.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_aggs_bucket_date_range.go rename to vendor/gopkg.in/olivere/elastic.v3/search_aggs_bucket_date_range.go diff --git a/vendor/github.com/olivere/elastic/search_aggs_bucket_filter.go b/vendor/gopkg.in/olivere/elastic.v3/search_aggs_bucket_filter.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_aggs_bucket_filter.go rename to vendor/gopkg.in/olivere/elastic.v3/search_aggs_bucket_filter.go diff --git a/vendor/github.com/olivere/elastic/search_aggs_bucket_filters.go b/vendor/gopkg.in/olivere/elastic.v3/search_aggs_bucket_filters.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_aggs_bucket_filters.go rename to vendor/gopkg.in/olivere/elastic.v3/search_aggs_bucket_filters.go diff --git a/vendor/github.com/olivere/elastic/search_aggs_bucket_geo_distance.go b/vendor/gopkg.in/olivere/elastic.v3/search_aggs_bucket_geo_distance.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_aggs_bucket_geo_distance.go rename to vendor/gopkg.in/olivere/elastic.v3/search_aggs_bucket_geo_distance.go diff --git a/vendor/github.com/olivere/elastic/search_aggs_bucket_geohash_grid.go b/vendor/gopkg.in/olivere/elastic.v3/search_aggs_bucket_geohash_grid.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_aggs_bucket_geohash_grid.go rename to vendor/gopkg.in/olivere/elastic.v3/search_aggs_bucket_geohash_grid.go diff --git a/vendor/github.com/olivere/elastic/search_aggs_bucket_global.go b/vendor/gopkg.in/olivere/elastic.v3/search_aggs_bucket_global.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_aggs_bucket_global.go rename to vendor/gopkg.in/olivere/elastic.v3/search_aggs_bucket_global.go diff --git a/vendor/github.com/olivere/elastic/search_aggs_bucket_histogram.go b/vendor/gopkg.in/olivere/elastic.v3/search_aggs_bucket_histogram.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_aggs_bucket_histogram.go rename to vendor/gopkg.in/olivere/elastic.v3/search_aggs_bucket_histogram.go diff --git a/vendor/github.com/olivere/elastic/search_aggs_bucket_missing.go b/vendor/gopkg.in/olivere/elastic.v3/search_aggs_bucket_missing.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_aggs_bucket_missing.go rename to vendor/gopkg.in/olivere/elastic.v3/search_aggs_bucket_missing.go diff --git a/vendor/github.com/olivere/elastic/search_aggs_bucket_nested.go b/vendor/gopkg.in/olivere/elastic.v3/search_aggs_bucket_nested.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_aggs_bucket_nested.go rename to vendor/gopkg.in/olivere/elastic.v3/search_aggs_bucket_nested.go diff --git a/vendor/github.com/olivere/elastic/search_aggs_bucket_range.go b/vendor/gopkg.in/olivere/elastic.v3/search_aggs_bucket_range.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_aggs_bucket_range.go rename to vendor/gopkg.in/olivere/elastic.v3/search_aggs_bucket_range.go diff --git a/vendor/github.com/olivere/elastic/search_aggs_bucket_reverse_nested.go b/vendor/gopkg.in/olivere/elastic.v3/search_aggs_bucket_reverse_nested.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_aggs_bucket_reverse_nested.go rename to vendor/gopkg.in/olivere/elastic.v3/search_aggs_bucket_reverse_nested.go diff --git a/vendor/github.com/olivere/elastic/search_aggs_bucket_sampler.go b/vendor/gopkg.in/olivere/elastic.v3/search_aggs_bucket_sampler.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_aggs_bucket_sampler.go rename to vendor/gopkg.in/olivere/elastic.v3/search_aggs_bucket_sampler.go diff --git a/vendor/github.com/olivere/elastic/search_aggs_bucket_significant_terms.go b/vendor/gopkg.in/olivere/elastic.v3/search_aggs_bucket_significant_terms.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_aggs_bucket_significant_terms.go rename to vendor/gopkg.in/olivere/elastic.v3/search_aggs_bucket_significant_terms.go diff --git a/vendor/github.com/olivere/elastic/search_aggs_bucket_terms.go b/vendor/gopkg.in/olivere/elastic.v3/search_aggs_bucket_terms.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_aggs_bucket_terms.go rename to vendor/gopkg.in/olivere/elastic.v3/search_aggs_bucket_terms.go diff --git a/vendor/github.com/olivere/elastic/search_aggs_metrics_avg.go b/vendor/gopkg.in/olivere/elastic.v3/search_aggs_metrics_avg.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_aggs_metrics_avg.go rename to vendor/gopkg.in/olivere/elastic.v3/search_aggs_metrics_avg.go diff --git a/vendor/github.com/olivere/elastic/search_aggs_metrics_cardinality.go b/vendor/gopkg.in/olivere/elastic.v3/search_aggs_metrics_cardinality.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_aggs_metrics_cardinality.go rename to vendor/gopkg.in/olivere/elastic.v3/search_aggs_metrics_cardinality.go diff --git a/vendor/github.com/olivere/elastic/search_aggs_metrics_extended_stats.go b/vendor/gopkg.in/olivere/elastic.v3/search_aggs_metrics_extended_stats.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_aggs_metrics_extended_stats.go rename to vendor/gopkg.in/olivere/elastic.v3/search_aggs_metrics_extended_stats.go diff --git a/vendor/github.com/olivere/elastic/search_aggs_metrics_geo_bounds.go b/vendor/gopkg.in/olivere/elastic.v3/search_aggs_metrics_geo_bounds.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_aggs_metrics_geo_bounds.go rename to vendor/gopkg.in/olivere/elastic.v3/search_aggs_metrics_geo_bounds.go diff --git a/vendor/github.com/olivere/elastic/search_aggs_metrics_max.go b/vendor/gopkg.in/olivere/elastic.v3/search_aggs_metrics_max.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_aggs_metrics_max.go rename to vendor/gopkg.in/olivere/elastic.v3/search_aggs_metrics_max.go diff --git a/vendor/github.com/olivere/elastic/search_aggs_metrics_min.go b/vendor/gopkg.in/olivere/elastic.v3/search_aggs_metrics_min.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_aggs_metrics_min.go rename to vendor/gopkg.in/olivere/elastic.v3/search_aggs_metrics_min.go diff --git a/vendor/github.com/olivere/elastic/search_aggs_metrics_percentile_ranks.go b/vendor/gopkg.in/olivere/elastic.v3/search_aggs_metrics_percentile_ranks.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_aggs_metrics_percentile_ranks.go rename to vendor/gopkg.in/olivere/elastic.v3/search_aggs_metrics_percentile_ranks.go diff --git a/vendor/github.com/olivere/elastic/search_aggs_metrics_percentiles.go b/vendor/gopkg.in/olivere/elastic.v3/search_aggs_metrics_percentiles.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_aggs_metrics_percentiles.go rename to vendor/gopkg.in/olivere/elastic.v3/search_aggs_metrics_percentiles.go diff --git a/vendor/github.com/olivere/elastic/search_aggs_metrics_stats.go b/vendor/gopkg.in/olivere/elastic.v3/search_aggs_metrics_stats.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_aggs_metrics_stats.go rename to vendor/gopkg.in/olivere/elastic.v3/search_aggs_metrics_stats.go diff --git a/vendor/github.com/olivere/elastic/search_aggs_metrics_sum.go b/vendor/gopkg.in/olivere/elastic.v3/search_aggs_metrics_sum.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_aggs_metrics_sum.go rename to vendor/gopkg.in/olivere/elastic.v3/search_aggs_metrics_sum.go diff --git a/vendor/github.com/olivere/elastic/search_aggs_metrics_top_hits.go b/vendor/gopkg.in/olivere/elastic.v3/search_aggs_metrics_top_hits.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_aggs_metrics_top_hits.go rename to vendor/gopkg.in/olivere/elastic.v3/search_aggs_metrics_top_hits.go diff --git a/vendor/github.com/olivere/elastic/search_aggs_metrics_value_count.go b/vendor/gopkg.in/olivere/elastic.v3/search_aggs_metrics_value_count.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_aggs_metrics_value_count.go rename to vendor/gopkg.in/olivere/elastic.v3/search_aggs_metrics_value_count.go diff --git a/vendor/github.com/olivere/elastic/search_aggs_pipeline_avg_bucket.go b/vendor/gopkg.in/olivere/elastic.v3/search_aggs_pipeline_avg_bucket.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_aggs_pipeline_avg_bucket.go rename to vendor/gopkg.in/olivere/elastic.v3/search_aggs_pipeline_avg_bucket.go diff --git a/vendor/github.com/olivere/elastic/search_aggs_pipeline_bucket_script.go b/vendor/gopkg.in/olivere/elastic.v3/search_aggs_pipeline_bucket_script.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_aggs_pipeline_bucket_script.go rename to vendor/gopkg.in/olivere/elastic.v3/search_aggs_pipeline_bucket_script.go diff --git a/vendor/github.com/olivere/elastic/search_aggs_pipeline_bucket_selector.go b/vendor/gopkg.in/olivere/elastic.v3/search_aggs_pipeline_bucket_selector.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_aggs_pipeline_bucket_selector.go rename to vendor/gopkg.in/olivere/elastic.v3/search_aggs_pipeline_bucket_selector.go diff --git a/vendor/github.com/olivere/elastic/search_aggs_pipeline_cumulative_sum.go b/vendor/gopkg.in/olivere/elastic.v3/search_aggs_pipeline_cumulative_sum.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_aggs_pipeline_cumulative_sum.go rename to vendor/gopkg.in/olivere/elastic.v3/search_aggs_pipeline_cumulative_sum.go diff --git a/vendor/github.com/olivere/elastic/search_aggs_pipeline_derivative.go b/vendor/gopkg.in/olivere/elastic.v3/search_aggs_pipeline_derivative.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_aggs_pipeline_derivative.go rename to vendor/gopkg.in/olivere/elastic.v3/search_aggs_pipeline_derivative.go diff --git a/vendor/github.com/olivere/elastic/search_aggs_pipeline_max_bucket.go b/vendor/gopkg.in/olivere/elastic.v3/search_aggs_pipeline_max_bucket.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_aggs_pipeline_max_bucket.go rename to vendor/gopkg.in/olivere/elastic.v3/search_aggs_pipeline_max_bucket.go diff --git a/vendor/github.com/olivere/elastic/search_aggs_pipeline_min_bucket.go b/vendor/gopkg.in/olivere/elastic.v3/search_aggs_pipeline_min_bucket.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_aggs_pipeline_min_bucket.go rename to vendor/gopkg.in/olivere/elastic.v3/search_aggs_pipeline_min_bucket.go diff --git a/vendor/github.com/olivere/elastic/search_aggs_pipeline_mov_avg.go b/vendor/gopkg.in/olivere/elastic.v3/search_aggs_pipeline_mov_avg.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_aggs_pipeline_mov_avg.go rename to vendor/gopkg.in/olivere/elastic.v3/search_aggs_pipeline_mov_avg.go diff --git a/vendor/github.com/olivere/elastic/search_aggs_pipeline_serial_diff.go b/vendor/gopkg.in/olivere/elastic.v3/search_aggs_pipeline_serial_diff.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_aggs_pipeline_serial_diff.go rename to vendor/gopkg.in/olivere/elastic.v3/search_aggs_pipeline_serial_diff.go diff --git a/vendor/github.com/olivere/elastic/search_aggs_pipeline_sum_bucket.go b/vendor/gopkg.in/olivere/elastic.v3/search_aggs_pipeline_sum_bucket.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_aggs_pipeline_sum_bucket.go rename to vendor/gopkg.in/olivere/elastic.v3/search_aggs_pipeline_sum_bucket.go diff --git a/vendor/github.com/olivere/elastic/search_queries_bool.go b/vendor/gopkg.in/olivere/elastic.v3/search_queries_bool.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_queries_bool.go rename to vendor/gopkg.in/olivere/elastic.v3/search_queries_bool.go diff --git a/vendor/github.com/olivere/elastic/search_queries_boosting.go b/vendor/gopkg.in/olivere/elastic.v3/search_queries_boosting.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_queries_boosting.go rename to vendor/gopkg.in/olivere/elastic.v3/search_queries_boosting.go diff --git a/vendor/github.com/olivere/elastic/search_queries_common_terms.go b/vendor/gopkg.in/olivere/elastic.v3/search_queries_common_terms.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_queries_common_terms.go rename to vendor/gopkg.in/olivere/elastic.v3/search_queries_common_terms.go diff --git a/vendor/github.com/olivere/elastic/search_queries_constant_score.go b/vendor/gopkg.in/olivere/elastic.v3/search_queries_constant_score.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_queries_constant_score.go rename to vendor/gopkg.in/olivere/elastic.v3/search_queries_constant_score.go diff --git a/vendor/github.com/olivere/elastic/search_queries_dis_max.go b/vendor/gopkg.in/olivere/elastic.v3/search_queries_dis_max.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_queries_dis_max.go rename to vendor/gopkg.in/olivere/elastic.v3/search_queries_dis_max.go diff --git a/vendor/github.com/olivere/elastic/search_queries_exists.go b/vendor/gopkg.in/olivere/elastic.v3/search_queries_exists.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_queries_exists.go rename to vendor/gopkg.in/olivere/elastic.v3/search_queries_exists.go diff --git a/vendor/github.com/olivere/elastic/search_queries_fsq.go b/vendor/gopkg.in/olivere/elastic.v3/search_queries_fsq.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_queries_fsq.go rename to vendor/gopkg.in/olivere/elastic.v3/search_queries_fsq.go diff --git a/vendor/github.com/olivere/elastic/search_queries_fsq_score_funcs.go b/vendor/gopkg.in/olivere/elastic.v3/search_queries_fsq_score_funcs.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_queries_fsq_score_funcs.go rename to vendor/gopkg.in/olivere/elastic.v3/search_queries_fsq_score_funcs.go diff --git a/vendor/github.com/olivere/elastic/search_queries_fuzzy.go b/vendor/gopkg.in/olivere/elastic.v3/search_queries_fuzzy.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_queries_fuzzy.go rename to vendor/gopkg.in/olivere/elastic.v3/search_queries_fuzzy.go diff --git a/vendor/github.com/olivere/elastic/search_queries_geo_bounding_box.go b/vendor/gopkg.in/olivere/elastic.v3/search_queries_geo_bounding_box.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_queries_geo_bounding_box.go rename to vendor/gopkg.in/olivere/elastic.v3/search_queries_geo_bounding_box.go diff --git a/vendor/github.com/olivere/elastic/search_queries_geo_distance.go b/vendor/gopkg.in/olivere/elastic.v3/search_queries_geo_distance.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_queries_geo_distance.go rename to vendor/gopkg.in/olivere/elastic.v3/search_queries_geo_distance.go diff --git a/vendor/github.com/olivere/elastic/search_queries_geo_polygon.go b/vendor/gopkg.in/olivere/elastic.v3/search_queries_geo_polygon.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_queries_geo_polygon.go rename to vendor/gopkg.in/olivere/elastic.v3/search_queries_geo_polygon.go diff --git a/vendor/github.com/olivere/elastic/search_queries_has_child.go b/vendor/gopkg.in/olivere/elastic.v3/search_queries_has_child.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_queries_has_child.go rename to vendor/gopkg.in/olivere/elastic.v3/search_queries_has_child.go diff --git a/vendor/github.com/olivere/elastic/search_queries_has_parent.go b/vendor/gopkg.in/olivere/elastic.v3/search_queries_has_parent.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_queries_has_parent.go rename to vendor/gopkg.in/olivere/elastic.v3/search_queries_has_parent.go diff --git a/vendor/github.com/olivere/elastic/search_queries_ids.go b/vendor/gopkg.in/olivere/elastic.v3/search_queries_ids.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_queries_ids.go rename to vendor/gopkg.in/olivere/elastic.v3/search_queries_ids.go diff --git a/vendor/github.com/olivere/elastic/search_queries_indices.go b/vendor/gopkg.in/olivere/elastic.v3/search_queries_indices.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_queries_indices.go rename to vendor/gopkg.in/olivere/elastic.v3/search_queries_indices.go diff --git a/vendor/github.com/olivere/elastic/search_queries_match.go b/vendor/gopkg.in/olivere/elastic.v3/search_queries_match.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_queries_match.go rename to vendor/gopkg.in/olivere/elastic.v3/search_queries_match.go diff --git a/vendor/github.com/olivere/elastic/search_queries_match_all.go b/vendor/gopkg.in/olivere/elastic.v3/search_queries_match_all.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_queries_match_all.go rename to vendor/gopkg.in/olivere/elastic.v3/search_queries_match_all.go diff --git a/vendor/github.com/olivere/elastic/search_queries_missing.go b/vendor/gopkg.in/olivere/elastic.v3/search_queries_missing.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_queries_missing.go rename to vendor/gopkg.in/olivere/elastic.v3/search_queries_missing.go diff --git a/vendor/github.com/olivere/elastic/search_queries_more_like_this.go b/vendor/gopkg.in/olivere/elastic.v3/search_queries_more_like_this.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_queries_more_like_this.go rename to vendor/gopkg.in/olivere/elastic.v3/search_queries_more_like_this.go diff --git a/vendor/github.com/olivere/elastic/search_queries_multi_match.go b/vendor/gopkg.in/olivere/elastic.v3/search_queries_multi_match.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_queries_multi_match.go rename to vendor/gopkg.in/olivere/elastic.v3/search_queries_multi_match.go diff --git a/vendor/github.com/olivere/elastic/search_queries_nested.go b/vendor/gopkg.in/olivere/elastic.v3/search_queries_nested.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_queries_nested.go rename to vendor/gopkg.in/olivere/elastic.v3/search_queries_nested.go diff --git a/vendor/github.com/olivere/elastic/search_queries_not.go b/vendor/gopkg.in/olivere/elastic.v3/search_queries_not.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_queries_not.go rename to vendor/gopkg.in/olivere/elastic.v3/search_queries_not.go diff --git a/vendor/github.com/olivere/elastic/search_queries_prefix.go b/vendor/gopkg.in/olivere/elastic.v3/search_queries_prefix.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_queries_prefix.go rename to vendor/gopkg.in/olivere/elastic.v3/search_queries_prefix.go diff --git a/vendor/github.com/olivere/elastic/search_queries_query_string.go b/vendor/gopkg.in/olivere/elastic.v3/search_queries_query_string.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_queries_query_string.go rename to vendor/gopkg.in/olivere/elastic.v3/search_queries_query_string.go diff --git a/vendor/github.com/olivere/elastic/search_queries_range.go b/vendor/gopkg.in/olivere/elastic.v3/search_queries_range.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_queries_range.go rename to vendor/gopkg.in/olivere/elastic.v3/search_queries_range.go diff --git a/vendor/github.com/olivere/elastic/search_queries_regexp.go b/vendor/gopkg.in/olivere/elastic.v3/search_queries_regexp.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_queries_regexp.go rename to vendor/gopkg.in/olivere/elastic.v3/search_queries_regexp.go diff --git a/vendor/github.com/olivere/elastic/search_queries_script.go b/vendor/gopkg.in/olivere/elastic.v3/search_queries_script.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_queries_script.go rename to vendor/gopkg.in/olivere/elastic.v3/search_queries_script.go diff --git a/vendor/github.com/olivere/elastic/search_queries_simple_query_string.go b/vendor/gopkg.in/olivere/elastic.v3/search_queries_simple_query_string.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_queries_simple_query_string.go rename to vendor/gopkg.in/olivere/elastic.v3/search_queries_simple_query_string.go diff --git a/vendor/github.com/olivere/elastic/search_queries_template_query.go b/vendor/gopkg.in/olivere/elastic.v3/search_queries_template_query.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_queries_template_query.go rename to vendor/gopkg.in/olivere/elastic.v3/search_queries_template_query.go diff --git a/vendor/github.com/olivere/elastic/search_queries_term.go b/vendor/gopkg.in/olivere/elastic.v3/search_queries_term.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_queries_term.go rename to vendor/gopkg.in/olivere/elastic.v3/search_queries_term.go diff --git a/vendor/github.com/olivere/elastic/search_queries_terms.go b/vendor/gopkg.in/olivere/elastic.v3/search_queries_terms.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_queries_terms.go rename to vendor/gopkg.in/olivere/elastic.v3/search_queries_terms.go diff --git a/vendor/github.com/olivere/elastic/search_queries_type.go b/vendor/gopkg.in/olivere/elastic.v3/search_queries_type.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_queries_type.go rename to vendor/gopkg.in/olivere/elastic.v3/search_queries_type.go diff --git a/vendor/github.com/olivere/elastic/search_queries_wildcard.go b/vendor/gopkg.in/olivere/elastic.v3/search_queries_wildcard.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_queries_wildcard.go rename to vendor/gopkg.in/olivere/elastic.v3/search_queries_wildcard.go diff --git a/vendor/github.com/olivere/elastic/search_request.go b/vendor/gopkg.in/olivere/elastic.v3/search_request.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_request.go rename to vendor/gopkg.in/olivere/elastic.v3/search_request.go diff --git a/vendor/github.com/olivere/elastic/search_source.go b/vendor/gopkg.in/olivere/elastic.v3/search_source.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_source.go rename to vendor/gopkg.in/olivere/elastic.v3/search_source.go diff --git a/vendor/github.com/olivere/elastic/search_template.go b/vendor/gopkg.in/olivere/elastic.v3/search_template.go similarity index 100% rename from vendor/github.com/olivere/elastic/search_template.go rename to vendor/gopkg.in/olivere/elastic.v3/search_template.go diff --git a/vendor/github.com/olivere/elastic/sort.go b/vendor/gopkg.in/olivere/elastic.v3/sort.go similarity index 100% rename from vendor/github.com/olivere/elastic/sort.go rename to vendor/gopkg.in/olivere/elastic.v3/sort.go diff --git a/vendor/github.com/olivere/elastic/suggest.go b/vendor/gopkg.in/olivere/elastic.v3/suggest.go similarity index 100% rename from vendor/github.com/olivere/elastic/suggest.go rename to vendor/gopkg.in/olivere/elastic.v3/suggest.go diff --git a/vendor/github.com/olivere/elastic/suggest_field.go b/vendor/gopkg.in/olivere/elastic.v3/suggest_field.go similarity index 100% rename from vendor/github.com/olivere/elastic/suggest_field.go rename to vendor/gopkg.in/olivere/elastic.v3/suggest_field.go diff --git a/vendor/github.com/olivere/elastic/suggester.go b/vendor/gopkg.in/olivere/elastic.v3/suggester.go similarity index 100% rename from vendor/github.com/olivere/elastic/suggester.go rename to vendor/gopkg.in/olivere/elastic.v3/suggester.go diff --git a/vendor/github.com/olivere/elastic/suggester_completion.go b/vendor/gopkg.in/olivere/elastic.v3/suggester_completion.go similarity index 100% rename from vendor/github.com/olivere/elastic/suggester_completion.go rename to vendor/gopkg.in/olivere/elastic.v3/suggester_completion.go diff --git a/vendor/github.com/olivere/elastic/suggester_completion_fuzzy.go b/vendor/gopkg.in/olivere/elastic.v3/suggester_completion_fuzzy.go similarity index 100% rename from vendor/github.com/olivere/elastic/suggester_completion_fuzzy.go rename to vendor/gopkg.in/olivere/elastic.v3/suggester_completion_fuzzy.go diff --git a/vendor/github.com/olivere/elastic/suggester_context.go b/vendor/gopkg.in/olivere/elastic.v3/suggester_context.go similarity index 100% rename from vendor/github.com/olivere/elastic/suggester_context.go rename to vendor/gopkg.in/olivere/elastic.v3/suggester_context.go diff --git a/vendor/github.com/olivere/elastic/suggester_context_category.go b/vendor/gopkg.in/olivere/elastic.v3/suggester_context_category.go similarity index 100% rename from vendor/github.com/olivere/elastic/suggester_context_category.go rename to vendor/gopkg.in/olivere/elastic.v3/suggester_context_category.go diff --git a/vendor/github.com/olivere/elastic/suggester_context_geo.go b/vendor/gopkg.in/olivere/elastic.v3/suggester_context_geo.go similarity index 100% rename from vendor/github.com/olivere/elastic/suggester_context_geo.go rename to vendor/gopkg.in/olivere/elastic.v3/suggester_context_geo.go diff --git a/vendor/github.com/olivere/elastic/suggester_phrase.go b/vendor/gopkg.in/olivere/elastic.v3/suggester_phrase.go similarity index 100% rename from vendor/github.com/olivere/elastic/suggester_phrase.go rename to vendor/gopkg.in/olivere/elastic.v3/suggester_phrase.go diff --git a/vendor/github.com/olivere/elastic/suggester_term.go b/vendor/gopkg.in/olivere/elastic.v3/suggester_term.go similarity index 100% rename from vendor/github.com/olivere/elastic/suggester_term.go rename to vendor/gopkg.in/olivere/elastic.v3/suggester_term.go diff --git a/vendor/github.com/olivere/elastic/tasks_cancel.go b/vendor/gopkg.in/olivere/elastic.v3/tasks_cancel.go similarity index 100% rename from vendor/github.com/olivere/elastic/tasks_cancel.go rename to vendor/gopkg.in/olivere/elastic.v3/tasks_cancel.go diff --git a/vendor/github.com/olivere/elastic/tasks_list.go b/vendor/gopkg.in/olivere/elastic.v3/tasks_list.go similarity index 100% rename from vendor/github.com/olivere/elastic/tasks_list.go rename to vendor/gopkg.in/olivere/elastic.v3/tasks_list.go diff --git a/vendor/github.com/olivere/elastic/termvectors.go b/vendor/gopkg.in/olivere/elastic.v3/termvectors.go similarity index 100% rename from vendor/github.com/olivere/elastic/termvectors.go rename to vendor/gopkg.in/olivere/elastic.v3/termvectors.go diff --git a/vendor/github.com/olivere/elastic/update.go b/vendor/gopkg.in/olivere/elastic.v3/update.go similarity index 100% rename from vendor/github.com/olivere/elastic/update.go rename to vendor/gopkg.in/olivere/elastic.v3/update.go diff --git a/vendor/github.com/olivere/elastic/update_by_query.go b/vendor/gopkg.in/olivere/elastic.v3/update_by_query.go similarity index 100% rename from vendor/github.com/olivere/elastic/update_by_query.go rename to vendor/gopkg.in/olivere/elastic.v3/update_by_query.go From 9d05351759c0b7c4db77d69cbbf6f3ec4e144301 Mon Sep 17 00:00:00 2001 From: AlmogBaku Date: Wed, 17 Aug 2016 12:05:53 +0300 Subject: [PATCH 2/5] add missing files revert Makefile change [mistakenly commited] --- Makefile | 3 +-- common/elasticsearch/aws.go | 44 +++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) create mode 100644 common/elasticsearch/aws.go diff --git a/Makefile b/Makefile index a0f4fe6190..c0435c46b4 100644 --- a/Makefile +++ b/Makefile @@ -32,8 +32,7 @@ test-integration: clean deps build container: build cp heapster deploy/docker/heapster cp eventer deploy/docker/eventer - docker build -t heapster deploy/docker/ - docker tag heapster:latest 032833530291.dkr.ecr.eu-west-1.amazonaws.com/heapster:latest + docker build -t $(PREFIX)/heapster:$(TAG) deploy/docker/ grafana: docker build -t $(PREFIX)/heapster_grafana:$(TAG) grafana/ diff --git a/common/elasticsearch/aws.go b/common/elasticsearch/aws.go new file mode 100644 index 0000000000..43817800bf --- /dev/null +++ b/common/elasticsearch/aws.go @@ -0,0 +1,44 @@ +package elasticsearch + +import ( + "fmt" + awsauth "github.com/smartystreets/go-aws-auth" + "net/http" + "os" +) + +type AWSSigningTransport struct { + HTTPClient *http.Client + Credentials awsauth.Credentials +} + +// RoundTrip implementation +func (a AWSSigningTransport) RoundTrip(req *http.Request) (*http.Response, error) { + return a.HTTPClient.Do(awsauth.Sign4(req, a.Credentials)) +} + +func createAWSClient() (*http.Client, error) { + id := os.Getenv("AWS_ACCESS_KEY_ID") + if id == "" { + id = os.Getenv("AWS_ACCESS_KEY") + } + + secret := os.Getenv("AWS_SECRET_ACCESS_KEY") + if secret == "" { + secret = os.Getenv("AWS_SECRET_KEY") + } + + if id == "" || secret == "" { + return nil, fmt.Errorf("Failed to configure AWS authentication. Both `AWS_ACCESS_KEY_ID` and " + + "`AWS_SECRET_ACCESS_KEY` environment veriables required") + } + + signingTransport := AWSSigningTransport{ + Credentials: awsauth.Credentials{ + AccessKeyID: id, + SecretAccessKey: secret, + }, + HTTPClient: http.DefaultClient, + } + return &http.Client{Transport: http.RoundTripper(signingTransport)}, nil +} From 97869bffcf1e467cbce430a29ea3ff69d2d57fd9 Mon Sep 17 00:00:00 2001 From: AlmogBaku Date: Wed, 17 Aug 2016 12:48:26 +0300 Subject: [PATCH 3/5] copyright notice in the file --- common/elasticsearch/aws.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/common/elasticsearch/aws.go b/common/elasticsearch/aws.go index 43817800bf..4075e94216 100644 --- a/common/elasticsearch/aws.go +++ b/common/elasticsearch/aws.go @@ -1,3 +1,16 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package elasticsearch import ( From 8b28b5dd0bf3550920ad91db9d05f58e9e398216 Mon Sep 17 00:00:00 2001 From: AlmogBaku Date: Tue, 23 Aug 2016 13:22:48 +0300 Subject: [PATCH 4/5] remove go-extpoints --- Godeps/Godeps.json | 8 +- .../progrium/go-extpoints/.gitignore | 4 - .../github.com/progrium/go-extpoints/LICENSE | 19 -- .../github.com/progrium/go-extpoints/Makefile | 8 - .../progrium/go-extpoints/README.md | 287 ------------------ .../github.com/progrium/go-extpoints/main.go | 179 ----------- .../progrium/go-extpoints/template.go | 183 ----------- 7 files changed, 1 insertion(+), 687 deletions(-) delete mode 100644 vendor/github.com/progrium/go-extpoints/.gitignore delete mode 100644 vendor/github.com/progrium/go-extpoints/LICENSE delete mode 100644 vendor/github.com/progrium/go-extpoints/Makefile delete mode 100644 vendor/github.com/progrium/go-extpoints/README.md delete mode 100644 vendor/github.com/progrium/go-extpoints/main.go delete mode 100644 vendor/github.com/progrium/go-extpoints/template.go diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 1655b21400..6bb87f5d8f 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -3,8 +3,7 @@ "GoVersion": "go1.6", "GodepVersion": "v74", "Packages": [ - "./...", - "github.com/progrium/go-extpoints" + "./..." ], "Deps": [ { @@ -202,11 +201,6 @@ "ImportPath": "github.com/pmezard/go-difflib/difflib", "Rev": "d8ed2627bdf02c080bf22230dbb337003b7aba2d" }, - { - "ImportPath": "github.com/progrium/go-extpoints", - "Comment": "go-plugins-43-g529a176", - "Rev": "529a176f52394e48e8882dd70a01732f1bb480f3" - }, { "ImportPath": "github.com/prometheus/client_golang/prometheus", "Comment": "0.7.0-39-g3b78d7a", diff --git a/vendor/github.com/progrium/go-extpoints/.gitignore b/vendor/github.com/progrium/go-extpoints/.gitignore deleted file mode 100644 index b5c4dba585..0000000000 --- a/vendor/github.com/progrium/go-extpoints/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -examples/tool/tool -examples/daemon/daemon -go-extpoints -tests/extpoints/extpoints.go diff --git a/vendor/github.com/progrium/go-extpoints/LICENSE b/vendor/github.com/progrium/go-extpoints/LICENSE deleted file mode 100644 index 3a487b3c3a..0000000000 --- a/vendor/github.com/progrium/go-extpoints/LICENSE +++ /dev/null @@ -1,19 +0,0 @@ -Copyright (C) 2015 Jeff Lindsay - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/vendor/github.com/progrium/go-extpoints/Makefile b/vendor/github.com/progrium/go-extpoints/Makefile deleted file mode 100644 index ffe98176e5..0000000000 --- a/vendor/github.com/progrium/go-extpoints/Makefile +++ /dev/null @@ -1,8 +0,0 @@ -GO ?= go - -test: - $(GO) run *.go ./tests/extpoints - $(GO) test -v ./tests - -install: - $(GO) install diff --git a/vendor/github.com/progrium/go-extpoints/README.md b/vendor/github.com/progrium/go-extpoints/README.md deleted file mode 100644 index f95223efb2..0000000000 --- a/vendor/github.com/progrium/go-extpoints/README.md +++ /dev/null @@ -1,287 +0,0 @@ -# go-extpoints - -This Go generator, named short for "extension points", provides a generic [inversion of control](http://en.wikipedia.org/wiki/Inversion_of_control) model for making extensible Go packages, libraries, and applications. - -It generates package extension point singletons from extension types you define. Extension points are then used to both register extensions and use the registered extensions with a common [meta-API](#extension-point-api). - -[Logspout](https://github.com/gliderlabs/logspout) is a real application built using go-extpoints. [Read about it here.](http://gliderlabs.com/blog/2015/03/31/new-logspout-extensible-docker-logging/) - -## Getting the tool - - $ go install github.com/progrium/go-extpoints - -## Concepts - -#### Extension Types - -These define your hooks. They can be Go interfaces or simple function signature types. Here are some generic examples: - -```go -type ConfigStore interface { - Get(key string) (string, error) - Set(key, value string) error - Del(key string) error -} - -type AuthProvider interface { - Authenticate(user, pass string) bool -} - -type EventListener interface { - Notify(event Event) -} - -type HttpEndpoint func() http.Handler - -type RequestModifier func(request *http.Request) - -``` - -#### Extension Points - -With types defined, go-extpoints generates package singletons for each type. When your program starts, extensions are registered with extension points. Then you can then use registered extensions in a number of ways: - -```go -// Lookup a single registered extension for drivers -config := ConfigStores.Lookup(configStore) -if config == nil { - log.Fatalf("config store '%s' not registered", configStore) -} -config.Set("foo", "bar") - -// Iterate until you get what you need -func authenticate(user, pass string) bool { - for _, provider := range AuthProviders.All() { - if provide.Authenticate(user, pass) { - return true - } - } - return false -} - -// Fire and forget events to all extensions -for _, listener := range EventListeners.All() { - listener.Notify(event) -} - -// Use name and return value of all extensions for registration -for name, handler := range HttpEndpoints.All() { - http.Handle("/"+name, handler()) -} - -// Pass by reference to all extensions for middleware -for _, modifier := range RequestModifiers.All() { - modifier(req) -} - -``` - -## Extension Point API - -All extension types passed to go-extpoints will be turned into extension point singletons, using the pluralized name of the extension type. These extension point objects implement this simple meta-API: - -```go -type interface { - // if name is "", the specific extension type is used. - // returns false if doesn't implement type or already registered. - Register(extension , name string) bool - - // returns false if not registered to start with - Unregister(name string) bool - - // returns nil if not registered - Lookup(name string) - - // for sorted subsets. each name is looked up in order, nil or not - Select(names []string) [] - - // all registered, keyed by name - All() map[string] - - // convenient list of names - Names() []string - -} -``` - -It also generates top-level registration functions that will run extensions through all known extension points, registering or unregistering with any that are based on an interface the extension implements. They return the names of the interfaces they were registered/unregistered with. - -```go - -func RegisterExtension(extension interface{}, name string) []string - -func UnregisterExtension(name string) []string - -``` - -## Example Application - -Here is a full Go application that lets extensions hook into `main()` as subcommands simply by implementing an interface we'll make called `Subcommand`. This interface will have just one method `Run()`, but you can make extension points based on any interface. - -Assuming our package lives under `$GOPATH/src/github.com/quick/example`, here is our `main.go`: - -```go -//go:generate go-extpoints -package main - -import ( - "fmt" - "os" - - "github.com/quick/example/extpoints" -) - -var subcommands = extpoints.Subcommands - -func usage() { - fmt.Println("Available commands:\n") - for name, _ := range subcommands.All() { - fmt.Println(" - ", name) - } - os.Exit(2) -} - -func main() { - if len(os.Args) < 2 { - usage() - } - cmd := subcommands.Lookup(os.Args[1]) - if cmd == nil { - usage() - } - cmd.Run(os.Args[2:]) -} -``` -Two things to note. First, the `go:generate` directive at the top. This tells `go generate` it needs to run `go-extpoints`, which will happen in a moment. - -Another thing to note, the extension point is accessed by a variable named by the plural of our interface `Subcommand` and in this case lives under a separate `extpoints` subpackage. - -We need to create that subpackage with a Go file in it to define our interface used for this extension point. This is our `extpoints/interfaces.go`: - -```go -package extpoints - -type Subcommand interface { - Run(args []string) -} -``` - -We use `go generate` now to produce the extension point code in our `extpoints` subpackage. These extension points are based on any Go interfaces you've defined in there. In our case, just `Subcommand`. - - $ go generate - ... - $ go install - -Okay, but it doesn't *do* anything! Let's make a builtin command extension that implements `Subcommand`. Add a `hello.go` file: - -```go -package main - -import ( - "fmt" - "github.com/quick/example/extpoints" -) - -func init() { - extpoints.Register(new(HelloComponent), "hello") -} - -type HelloComponent struct {} - -func (p *HelloComponent) Run(args []string) { - fmt.Println("Hello world!") -} -``` - -Now when we build and run the app, it shows `hello` as a subcommand. We've registered the component with the name `hello`, which we happen to use to identify the name of the subcommand. Component names are optional, but can be handy for situations like this one. - -Certainly, the value of extension points becomes clearer with larger applications and more interesting interfaces. But just consider now that the component defined in `hello.go` *could* exist in another package in another repo. You'd just have to import it and rebuild to let it hook into our application. - -There are two more in-deptch example applications in this repo to take a look at: - - * [tool](https://github.com/progrium/go-extpoints/tree/master/examples/tool) ([extpoints](http://godoc.org/github.com/progrium/go-extpoints/examples/tool/extpoints)), a more realistic CLI tool with subcommands and lifecycle hooks - * [daemon](https://github.com/progrium/go-extpoints/tree/master/examples/daemon), ... doesn't exist yet - - - -## Making it easy to install extensions - -Assuming you tell third-party developers to call your package or extension point `Register` in their `init()`, you can link them with a side-effect import (using a blank import name). - -You can make this easy for users to enable/disable via comments, or add their own without worrying about messing with your code by having a separate `extensions.go` or `plugins.go` file with just these imports: - -```go -package yourpackage - -import ( - _ "github.com/you/some-extension" - _ "github.com/third-party/another-extension" -) - -``` - -Users can now just edit this file and `go build` or `go install`. - -## Usage Patterns - -Here are different example ways to use extension points to interact with extensions: - -#### Simple Iteration -```go -for _, listener := range extpoints.EventListeners.All() { - listener.Notify(&MyEvent{}) -} -``` - -#### Lookup Only One -```go -driverName := config.Get("storage-driver") -driver := extpoints.StorageDrivers.Lookup(driverName) -if driver == nil { - log.Fatalf("storage driver '%s' not installed", driverName) -} -driver.StoreObject(object) -``` - -#### Passing by Reference -```go -for _, filter := range extpoints.RequestFilters.All() { - filter.FilterRequest(req) -} -``` - -#### Match and Use -```go -for _, handler := range extpoints.RequestHandlers.All() { - if handler.MatchRequest(req) { - handler.HandleRequest(req) - break - } -} -``` - -## Why the `extpoints` subpackage? - -Since we encourage the convention of a subpackage called `extpoints`, it makes it very easy to identify a package as having extension points from looking at the project tree. You then know where to look to find the interfaces that are exposed as extension points. - -Third-party packages have a well known package to import for registering. Whether you have extension points for a library package or a command with just a `main` package, there's always a definite `extpoints` package there to import. - -It also makes it clearer in your code when you're using extension points. You have to explicitly import the package, then call `extpoints.` when using them. This helps identify where extension points actually hook into your program. - -## Groundwork for Dynamic Extensions - -Although this only seems to allow for compile-time extensibility, this itself is quite a win. It means power users can build and compile in their own extensions that live outside your repository. - -However, it also lays the groundwork for other dynamic extensions. I've used this model to wrap extension points for components in embedded scripting languages, as hook scripts, as remote plugin daemons via RPC, or all of the above implemented as components themselves! - -No matter how you're thinking about dynamic extensions later on, using `go-extpoints` gives you a lot of options. Once Go supports dynamic libraries? This will work perfectly with that, too. - -## Inspiration - -This project and component model is a lightweight, Go idiomatic port of the [component architecture](http://trac.edgewall.org/wiki/TracDev/ComponentArchitecture) used in Trac, which is written in Python. It's taken about a year to get this right in Go. - -![Trac Component Architecture](http://trac.edgewall.org/raw-attachment/wiki/TracDev/ComponentArchitecture/xtnpt.png) - -## License - -BSD diff --git a/vendor/github.com/progrium/go-extpoints/main.go b/vendor/github.com/progrium/go-extpoints/main.go deleted file mode 100644 index c6a2cf25ec..0000000000 --- a/vendor/github.com/progrium/go-extpoints/main.go +++ /dev/null @@ -1,179 +0,0 @@ -package main - -import ( - "go/ast" - "go/parser" - "go/token" - "io/ioutil" - "log" - "os" - "path/filepath" - "strings" - "text/template" - - "github.com/gedex/inflector" -) - -func processFile(inputPath string) (string, []string) { - fset := token.NewFileSet() - f, err := parser.ParseFile(fset, inputPath, nil, parser.ParseComments) - if err != nil { - log.Fatalf("Could not parse file: %s", err) - } - - packageName := identifyPackage(f) - if packageName == "" { - log.Fatalf("Could not determine package name of %s", inputPath) - } - - var ifaces []string - for _, decl := range f.Decls { - if typeName, ok := identifyInterface(decl); ok { - ifaces = append(ifaces, typeName) - continue - } - if typeName, ok := identifyFuncType(decl); ok { - ifaces = append(ifaces, typeName) - continue - } - } - - return packageName, ifaces -} - -func identifyPackage(f *ast.File) string { - if f.Name == nil { - return "" - } - return f.Name.Name -} - -func identifyFuncType(decl ast.Decl) (typeName string, match bool) { - genDecl, ok := decl.(*ast.GenDecl) - if !ok { - return - } - for _, spec := range genDecl.Specs { - if typeSpec, ok := spec.(*ast.TypeSpec); ok { - if _, ok := typeSpec.Type.(*ast.FuncType); ok { - if typeSpec.Name != nil { - typeName = typeSpec.Name.Name - break - } - } - } - } - if typeName == "" { - return - } - match = true - return -} - -func identifyInterface(decl ast.Decl) (typeName string, match bool) { - genDecl, ok := decl.(*ast.GenDecl) - if !ok { - return - } - for _, spec := range genDecl.Specs { - if typeSpec, ok := spec.(*ast.TypeSpec); ok { - if _, ok := typeSpec.Type.(*ast.InterfaceType); ok { - if typeSpec.Name != nil { - typeName = typeSpec.Name.Name - break - } - } - } - } - if typeName == "" { - return - } - match = true - return -} - -type extensionPoint struct { - Name string -} - -func (i *extensionPoint) Var() string { - return inflector.Pluralize(i.Name) -} - -func (i *extensionPoint) Type() string { - return strings.ToLower(i.Name[0:1]) + i.Name[1:] + "Ext" -} - -func extensionPoints(ifaces []string) []extensionPoint { - var extpoints []extensionPoint - for _, iface := range ifaces { - extpoints = append(extpoints, extensionPoint{iface}) - } - return extpoints -} - -type templateData struct { - Package string - ExtensionPoints []extensionPoint -} - -func renderExtpoints(path, packageName string, ifaces []string) error { - output, err := os.Create(path) - if err != nil { - log.Fatalf("Could not open output file: %s", err) - } - defer output.Close() - outputTemplate := template.Must(template.New("render").Parse(extpointsTemplate)) - return outputTemplate.Execute(output, templateData{packageName, extensionPoints(ifaces)}) -} - -func main() { - log.SetFlags(0) - log.SetPrefix("extpoints: ") - - packagePath := "./extpoints" - if len(os.Args) > 1 { - packagePath = os.Args[1] - } - if _, err := os.Stat(packagePath); os.IsNotExist(err) { - log.Fatal("Unable to find package for extpoints, not found:", packagePath) - } - - var packageName string - var ifaces, ifacesAll []string - ifacesAllowed := make(map[string]struct{}) - - if len(os.Args) > 2 { - for _, iface := range os.Args[2:] { - ifacesAllowed[iface] = struct{}{} - } - } - - files, _ := ioutil.ReadDir(packagePath) - for _, file := range files { - if file.Name() != "extpoints.go" && !strings.HasSuffix(file.Name(), "_ext.go") { - path := filepath.Join(packagePath, file.Name()) - log.Printf("Processing file %s", path) - packageName, ifaces = processFile(path) - if len(ifacesAllowed) > 0 { - var ifacesFiltered []string - for _, iface := range ifaces { - _, allowed := ifacesAllowed[iface] - if allowed { - ifacesFiltered = append(ifacesFiltered, iface) - } - } - ifaces = ifacesFiltered - } - log.Printf("Found interfaces: %#v", ifaces) - ifacesAll = append(ifacesAll, ifaces...) - } - } - - path := filepath.Join(packagePath, "extpoints.go") - log.Printf("Writing file %s", path) - err := renderExtpoints(path, packageName, ifacesAll) - if err != nil { - log.Fatalf("Could not write extpoints.go file: %s", err) - } -} diff --git a/vendor/github.com/progrium/go-extpoints/template.go b/vendor/github.com/progrium/go-extpoints/template.go deleted file mode 100644 index 101af5f82d..0000000000 --- a/vendor/github.com/progrium/go-extpoints/template.go +++ /dev/null @@ -1,183 +0,0 @@ -package main - -var extpointsTemplate = `// generated by go-extpoints -- DO NOT EDIT -package {{.Package}} - -import ( - "reflect" - "runtime" - "strings" - "sync" -) - -var extRegistry = ®istryType{m: make(map[string]*extensionPoint)} - -type registryType struct { - sync.Mutex - m map[string]*extensionPoint -} - -// Top level registration - -func extensionTypes(extension interface{}) []string { - var ifaces []string - typ := reflect.TypeOf(extension) - for name, ep := range extRegistry.m { - if ep.iface.Kind() == reflect.Func && typ.AssignableTo(ep.iface) { - ifaces = append(ifaces, name) - } - if ep.iface.Kind() != reflect.Func && typ.Implements(ep.iface) { - ifaces = append(ifaces, name) - } - } - return ifaces -} - -func RegisterExtension(extension interface{}, name string) []string { - extRegistry.Lock() - defer extRegistry.Unlock() - var ifaces []string - for _, iface := range extensionTypes(extension) { - if extRegistry.m[iface].register(extension, name) { - ifaces = append(ifaces, iface) - } - } - return ifaces -} - -func UnregisterExtension(name string) []string { - extRegistry.Lock() - defer extRegistry.Unlock() - var ifaces []string - for iface, extpoint := range extRegistry.m { - if extpoint.unregister(name) { - ifaces = append(ifaces, iface) - } - } - return ifaces -} - - -// Base extension point - -type extensionPoint struct { - sync.Mutex - iface reflect.Type - extensions map[string]interface{} -} - -func newExtensionPoint(iface interface{}) *extensionPoint { - ep := &extensionPoint{ - iface: reflect.TypeOf(iface).Elem(), - extensions: make(map[string]interface{}), - } - extRegistry.Lock() - extRegistry.m[ep.iface.Name()] = ep - extRegistry.Unlock() - return ep -} - -func (ep *extensionPoint) lookup(name string) interface{} { - ep.Lock() - defer ep.Unlock() - ext, ok := ep.extensions[name] - if !ok { - return nil - } - return ext -} - -func (ep *extensionPoint) all() map[string]interface{} { - ep.Lock() - defer ep.Unlock() - all := make(map[string]interface{}) - for k, v := range ep.extensions { - all[k] = v - } - return all -} - -func (ep *extensionPoint) register(extension interface{}, name string) bool { - ep.Lock() - defer ep.Unlock() - if name == "" { - typ := reflect.TypeOf(extension) - if typ.Kind() == reflect.Func { - nameParts := strings.Split(runtime.FuncForPC( - reflect.ValueOf(extension).Pointer()).Name(), ".") - name = nameParts[len(nameParts)-1] - } else { - name = typ.Elem().Name() - } - } - _, exists := ep.extensions[name] - if exists { - return false - } - ep.extensions[name] = extension - return true -} - -func (ep *extensionPoint) unregister(name string) bool { - ep.Lock() - defer ep.Unlock() - _, exists := ep.extensions[name] - if !exists { - return false - } - delete(ep.extensions, name) - return true -} - -{{range .ExtensionPoints}}// {{.Name}} - -var {{.Var}} = &{{.Type}}{ - newExtensionPoint(new({{.Name}})), -} - -type {{.Type}} struct { - *extensionPoint -} - -func (ep *{{.Type}}) Unregister(name string) bool { - return ep.unregister(name) -} - -func (ep *{{.Type}}) Register(extension {{.Name}}, name string) bool { - return ep.register(extension, name) -} - -func (ep *{{.Type}}) Lookup(name string) {{.Name}} { - ext := ep.lookup(name) - if ext == nil { - return nil - } - return ext.({{.Name}}) -} - -func (ep *{{.Type}}) Select(names []string) []{{.Name}} { - var selected []{{.Name}} - for _, name := range names { - selected = append(selected, ep.Lookup(name)) - } - return selected -} - -func (ep *{{.Type}}) All() map[string]{{.Name}} { - all := make(map[string]{{.Name}}) - for k, v := range ep.all() { - all[k] = v.({{.Name}}) - } - return all -} - -func (ep *{{.Type}}) Names() []string { - var names []string - for k := range ep.all() { - names = append(names, k) - } - return names -} - - -{{end}}` From e6425d1f8b8a32543a8063bad07c8c87bd1eb6b1 Mon Sep 17 00:00:00 2001 From: AlmogBaku Date: Mon, 29 Aug 2016 10:30:26 +0300 Subject: [PATCH 5/5] fixes logs msgs and verbosity levels (due to @huangyuqi review) --- .gitignore | 1 + common/elasticsearch/elasticsearch.go | 8 +++----- events/sinks/elasticsearch/driver.go | 8 ++++---- metrics/sinks/elasticsearch/driver.go | 8 ++++---- 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index 39dd0018c8..1d369a2055 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ eventer *.un~ Session.vim .netrwhist +.idea \ No newline at end of file diff --git a/common/elasticsearch/elasticsearch.go b/common/elasticsearch/elasticsearch.go index c04ab7a1dc..b50dc4b074 100644 --- a/common/elasticsearch/elasticsearch.go +++ b/common/elasticsearch/elasticsearch.go @@ -55,7 +55,7 @@ func SaveDataIntoES(esClient *elastic.Client, indexName string, typeName string, return err } if !createIndex.Acknowledged { - return fmt.Errorf("failed to create Index in ES cluster: %s", err) + return fmt.Errorf("Failed to create Index in ES cluster: %s", err) } } indexID := uuid.NewUUID() @@ -78,7 +78,7 @@ func CreateElasticSearchConfig(uri *url.URL) (*ElasticSearchConfig, error) { var esConfig ElasticSearchConfig opts, err := url.ParseQuery(uri.RawQuery) if err != nil { - return nil, fmt.Errorf("fFailed to parser url's query string: %s", err) + return nil, fmt.Errorf("Failed to parser url's query string: %s", err) } // set the index for es,the default value is "heapster" @@ -130,7 +130,6 @@ func CreateElasticSearchConfig(uri *url.URL) (*ElasticSearchConfig, error) { if os.Getenv("AWS_ACCESS_KEY_ID") != "" || os.Getenv("AWS_ACCESS_KEY") != "" || os.Getenv("AWS_SECRET_ACCESS_KEY") != "" || os.Getenv("AWS_SECRET_KEY") != "" { - glog.Info("Configuring with AWS credentials..") awsClient, err := createAWSClient() @@ -138,8 +137,7 @@ func CreateElasticSearchConfig(uri *url.URL) (*ElasticSearchConfig, error) { return nil, err } - startupFns = append(startupFns, elastic.SetHttpClient(awsClient)) - startupFns = append(startupFns, elastic.SetSniff(false)) + startupFns = append(startupFns, elastic.SetHttpClient(awsClient), elastic.SetSniff(false)) } else { if len(opts["sniff"]) > 0 { sniff, err := strconv.ParseBool(opts["sniff"][0]) diff --git a/events/sinks/elasticsearch/driver.go b/events/sinks/elasticsearch/driver.go index de974fc81e..a05a063f62 100644 --- a/events/sinks/elasticsearch/driver.go +++ b/events/sinks/elasticsearch/driver.go @@ -84,11 +84,11 @@ func (sink *elasticSearchSink) ExportEvents(eventBatch *event_core.EventBatch) { for _, event := range eventBatch.Events { point, err := eventToPoint(event) if err != nil { - glog.Info("Failed to convert event to point: %v", err) + glog.Warningf("Failed to convert event to point: %v", err) } err = sink.saveDataFunc(sink.esConfig.EsClient, sink.esConfig.Index, typeName, point) if err != nil { - glog.V(3).Info("Failed to export data to ElasticSearch sink: ", err) + glog.Warningf("Failed to export data to ElasticSearch sink: %v", err) } } } @@ -105,12 +105,12 @@ func NewElasticSearchSink(uri *url.URL) (event_core.EventSink, error) { var esSink elasticSearchSink elasticsearchConfig, err := esCommon.CreateElasticSearchConfig(uri) if err != nil { - glog.V(2).Infof("Failed to config ElasticSearch") + glog.Warning("Failed to config ElasticSearch") return nil, err } esSink.esConfig = *elasticsearchConfig esSink.saveDataFunc = esCommon.SaveDataIntoES - glog.V(2).Infof("ElasticSearch sink setup successfully") + glog.V(2).Info("ElasticSearch sink setup successfully") return &esSink, nil } diff --git a/metrics/sinks/elasticsearch/driver.go b/metrics/sinks/elasticsearch/driver.go index 61c1e0d25d..7cc749186b 100644 --- a/metrics/sinks/elasticsearch/driver.go +++ b/metrics/sinks/elasticsearch/driver.go @@ -60,7 +60,7 @@ func (sink *elasticSearchSink) ExportData(dataBatch *core.DataBatch) { } err := sink.saveDataFunc(sink.esConfig.EsClient, sink.esConfig.Index, typeName, point) if err != nil { - glog.V(3).Info("Failed to export data to ElasticSearch sink: ", err) + glog.Warningf("Failed to export data to ElasticSearch sink: %v", err) } } for _, metric := range metricSet.LabeledMetrics { @@ -81,7 +81,7 @@ func (sink *elasticSearchSink) ExportData(dataBatch *core.DataBatch) { } err := sink.saveDataFunc(sink.esConfig.EsClient, sink.esConfig.Index, typeName, point) if err != nil { - glog.V(3).Info("Failed to export data to ElasticSearch sink: ", err) + glog.Warningf("Failed to export data to ElasticSearch sink: %v", err) } } } @@ -99,12 +99,12 @@ func NewElasticSearchSink(uri *url.URL) (core.DataSink, error) { var esSink elasticSearchSink elasticsearchConfig, err := esCommon.CreateElasticSearchConfig(uri) if err != nil { - glog.V(2).Infof("Failed to config ElasticSearch") + glog.Warningf("Failed to config ElasticSearch: %v", err) return nil, err } esSink.esConfig = *elasticsearchConfig esSink.saveDataFunc = esCommon.SaveDataIntoES - glog.V(2).Infof("ElasticSearch sink setup successfully") + glog.V(2).Info("ElasticSearch sink setup successfully") return &esSink, nil }