Upgrade to aws sdk v3

COMPATIBILITY.md

@@ -5,23 +5,25 @@
 |  logstash-output-opensearch | Logstash OSS|
 | ------------- | ------------- |
 | 1.0.0+  | 7.13.2  |
+| 2.0.0+  | 7.13.2  |
 
 ### Matrix for OpenSearch
 
 |  logstash-output-opensearch | OpenSearch  |
 | ------------- | ------------- |
-| 1.0.0+  | 1.0.0  |  
-
+| 1.0.0+  | 1.0.0  |
+| 2.0.0+  | 1.0.0  |
 
 ### Matrix for ODFE
 
 |  logstash-output-opensearch |  ODFE |
 | ------------- | ------------- |
 | 1.0.0+   |   1.x - 1.13.2 |
-
+| 2.0.0+   |   1.x - 1.13.2 |
 
 ### Matrix for Elasticsearch OSS
 
 |  logstash-output-opensearch |  Elasticsearch OSS|
 | ------------- | ------------- |
 | 1.0.0+   |  7.x - 7.10.2      |
+| 2.0.0+   |  7.x - 7.10.2      |

README.md

@@ -44,52 +44,67 @@ output {
 
 To run the Logstash Output Opensearch plugin using aws_iam authentication, refer to the sample configuration shown below:
 ```
-output {        
-   opensearch {     
-          hosts => ["hostname:port"]              
-          auth_type => {    
-              type => 'aws_iam'     
-              aws_access_key_id => 'ACCESS_KEY'     
-              aws_secret_access_key => 'SECRET_KEY'     
-              region => 'us-west-2'         
-          }         
-          index  => "logstash-logs-%{+YYYY.MM.dd}"      
-   }            
+output {
+   opensearch {
+          hosts => ["hostname:port"]
+          auth_type => {
+              type => 'aws_iam'
+              aws_access_key_id => 'ACCESS_KEY'
+              aws_secret_access_key => 'SECRET_KEY'
+              region => 'us-west-2'
+          }
+          index  => "logstash-logs-%{+YYYY.MM.dd}"
+   }
 }
 ```
 
 In addition to the existing authentication mechanisms, if we want to add new authentication then we will be adding them in the configuration by using auth_type.
 
 Example Configuration for basic authentication:
 ```
-output {    
-    opensearch {        
-          hosts  => ["hostname:port"]     
-          auth_type => {            
-              type => 'basic'           
-              user => 'admin'           
-              password => 'admin'           
-          }             
-          index => "logstash-logs-%{+YYYY.MM.dd}"       
-   }            
-}  
+output {
+    opensearch {
+          hosts  => ["hostname:port"]
+          auth_type => {
+              type => 'basic'
+              user => 'admin'
+              password => 'admin'
+          }
+          index => "logstash-logs-%{+YYYY.MM.dd}"
+   }
+}
 ```
 
 To ingest data into a `data stream` through logstash, we need to create the data stream and specify the name of data stream and the `op_type` of `create` in the output configuration. The sample configuration is shown below:
 
 ```yml
-output {    
-    opensearch {        
-          hosts  => ["https://hostname:port"]     
-          auth_type => {            
-              type => 'basic'           
-              user => 'admin'           
-              password => 'admin'           
+output {
+    opensearch {
+          hosts  => ["https://hostname:port"]
+          auth_type => {
+              type => 'basic'
+              user => 'admin'
+              password => 'admin'
           }
           index => "my-data-stream"
           action => "create"
-   }            
-}               
+   }
+}
+```
+
+Starting in 2.0.0, the aws sdk version is bumped to v3. In order for all other AWS plugins to work together, please remove pre-installed aws plugins and install logstash-integration-aws plugin as follows. See also https://github.com/logstash-plugins/logstash-mixin-aws/issues/38
+```
+# Remove existing logstash aws plugins and install logstash-integration-aws to keep sdk dependency the same
+# https://github.com/logstash-plugins/logstash-mixin-aws/issues/38
+/usr/share/logstash/bin/logstash-plugin remove logstash-input-s3
+/usr/share/logstash/bin/logstash-plugin remove logstash-input-sqs
+/usr/share/logstash/bin/logstash-plugin remove logstash-output-s3
+/usr/share/logstash/bin/logstash-plugin remove logstash-output-sns
+/usr/share/logstash/bin/logstash-plugin remove logstash-output-sqs
+/usr/share/logstash/bin/logstash-plugin remove logstash-output-cloudwatch
+
+/usr/share/logstash/bin/logstash-plugin install --version 0.1.0.pre logstash-integration-aws
+bin/logstash-plugin install --version 2.0.0 logstash-output-opensearch
 ```
 
 For more details refer to this [documentation](https://opensearch.org/docs/latest/clients/logstash/ship-to-opensearch/#opensearch-output-plugin)
@@ -104,4 +119,4 @@ This project is licensed under the [Apache v2.0 License](LICENSE).
 
 ## Copyright
 
-Copyright OpenSearch Contributors. See [NOTICE](NOTICE) for details.
+Copyright OpenSearch Contributors. See [NOTICE](NOTICE) for details.

lib/logstash/outputs/opensearch/http_client/manticore_adapter.rb

@@ -57,7 +57,7 @@ def initialize(logger, options={})
       if options[:proxy]
         options[:proxy] = manticore_proxy_hash(options[:proxy])
       end
-      
+
       @manticore = ::Manticore::Client.new(options)
     end
 
@@ -105,7 +105,7 @@ def get_user()
     def get_password()
       @password
     end
-    
+
     # Transform the proxy option to a hash. Manticore's support for non-hash
     # proxy options is broken. This was fixed in https://github.com/cheald/manticore/commit/34a00cee57a56148629ed0a47c329181e7319af5
     # but this is not yet released
@@ -137,12 +137,12 @@ def perform_request(url, method, path, params={}, body=nil)
       params[:body] = body if body
 
       if url.user
-        params[:auth] = { 
+        params[:auth] = {
           :user => CGI.unescape(url.user),
           # We have to unescape the password here since manticore won't do it
           # for us unless its part of the URL
-          :password => CGI.unescape(url.password), 
-          :eager => true 
+          :password => CGI.unescape(url.password),
+          :eager => true
         }
       elsif @type == BASIC_AUTH_TYPE
         add_basic_auth_to_params(params)
@@ -174,11 +174,17 @@ def perform_request(url, method, path, params={}, body=nil)
 
     def sign_aws_request(request_uri, path, method, params)
       url = URI::HTTPS.build({:host=>URI(request_uri.to_s).host, :port=>AWS_DEFAULT_PORT.to_s, :path=>path})
-      key = Seahorse::Client::Http::Request.new(options={:endpoint=>url, :http_method => method.to_s.upcase,
-                                                         :headers => params[:headers],:body => params[:body]})
-      aws_signer = Aws::Signers::V4.new(@credentials,  AWS_SERVICE, get_aws_region )
-      signed_key =  aws_signer.sign(key)
-      params[:headers] =  params[:headers].merge(signed_key.headers)
+      request = Seahorse::Client::Http::Request.new(options={:endpoint=>url, :http_method => method.to_s.upcase,
+        :headers => params[:headers],:body => params[:body]})
+
+      aws_signer = Aws::Sigv4::Signer.new(service: @service_name, region: @region, credentials_provider: @credentials)
+      signed_key = aws_signer.sign_request(
+        http_method: request.http_method,
+        url: url,
+        headers: params[:headers],
+        body: params[:body]
+      )
+      params[:headers] = params[:headers].merge(signed_key.headers)
     end
 
     def add_basic_auth_to_params(params)
@@ -192,7 +198,7 @@ def add_basic_auth_to_params(params)
     # Returned urls from this method should be checked for double escaping.
     def format_url(url, path_and_query=nil)
       request_uri = url.clone
-      
+
       # We excise auth info from the URL in case manticore itself tries to stick
       # sensitive data in a thrown exception or log data
       request_uri.user = nil
@@ -208,7 +214,7 @@ def format_url(url, path_and_query=nil)
       new_query_parts = [request_uri.query, parsed_path_and_query.query].select do |part|
         part && !part.empty? # Skip empty nil and ""
       end
-      
+
       request_uri.query = new_query_parts.join("&") unless new_query_parts.empty?
 
       # use `raw_path`` as `path` will unescape any escaped '/' in the path

logstash-output-opensearch.gemspec

@@ -11,7 +11,7 @@ signing_key_path = "gem-private_key.pem"
 
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-opensearch'
-  s.version         = '1.3.0'
+  s.version         = '2.0.0'
 
   s.licenses        = ['Apache-2.0']
   s.summary         = "Stores logs in OpenSearch"
@@ -45,7 +45,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'stud', ['>= 0.0.17', '~> 0.0']
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.0'
-  s.add_runtime_dependency 'aws-sdk',  '>= 2.11.632', '~> 2'
+  s.add_runtime_dependency 'aws-sdk', '~> 3'
   s.add_runtime_dependency 'json', '>= 2.3.0', '~> 2'
 
   s.add_development_dependency 'logstash-codec-plain'