Add commands to switch client media ingest bucket to public/private

CHANGELOG.adoc

@@ -27,6 +27,14 @@ These changes are merged into the `main` branch, but have not been released. Aft
 
 === Deprecated/Will break in a future version
 
+== 2.2.0 (2023-04-24)
+=== Added
+* Optional `aws_media_ingest_profile` system config setting, specifying AWS profile through which to access client media ingest bucket
+* Optional `media_bucket` client config setting, specifying name of client media ingest bucket
+* Two new commands to control the access policy of client media ingest bucket:
+** thor bucket:public (makes bucket public, so media can be ingested)
+** thor bucket:private (makes bucket private)
+
 == 2.1.0 (2023-04-19)
 === Added
 * Ability to ingest structured date details. See https://github.com/lyrasis/collectionspace_migration_tools/blob/main/doc/dates.adoc[Dates workflows documentation] for details.

README.adoc

@@ -61,7 +61,7 @@ If you want to change the Redis ports, you need to update them in two places:
 Nothing in `./redis.yml` is sensitive, as it's all just on your local machine.
 
 === AWS config
-You will need to create an AWS profile that this tool will use to interact with S3 Fast Import Buckets (and eventually CloudWatchLogs).
+You will need to create an AWS profile that this tool will use to interact with S3 Fast Import Buckets (and eventually CloudWatchLogs) and S3 buckets set up to hold client media files for ingest.
 
 - You will need personal user credentials to the CollectionSpace AWS account. Mark (or whoever admins that account) will need to set that up and ensure your user has the "MigrationsRole"
 - If you do not already have an `~/.aws/credentials` file, create that file.
@@ -76,8 +76,15 @@ aws_secret_access_key = {your user secret access key}
 role_arn = arn:aws:iam::163800758780:role/MigrationsRole
 source_profile = csdefault
 region = us-west-2
+
+[cs-media]
+role_arn = arn:aws:iam::005622933699:role/MigrationsRole
+source_profile = csdefault
+region = us-west-2
 ....
 
+Your personal user credentials go in the `[csdefault]` entry. The `[collectionspacemigrationtools]` entry uses those credentials to access the account where the fast import buckets are created. The `[cs-media]` uses those credentials to access the account where client media buckets are created.
+
 You can change the profile names in brackets as you wish. Just note that the first profile name is the value of `source_profile` in the second profile.
 
 === System config

lib/collectionspace_migration_tools/build/s3_media_client.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module CollectionspaceMigrationTools
+  module Build
+  # Returns AWS S3 client for interacting with media ingest bucket
+    class S3MediaClient < S3Client
+      def initialize
+        @profile = CMT.config.system.aws_media_ingest_profile
+        @bucket = CMT.config.client.media_bucket
+      end
+    end
+  end
+end

lib/collectionspace_migration_tools/s3/bucket_policy_setter.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module CollectionspaceMigrationTools
+  module S3
+    class BucketPolicySetter
+      include Dry::Monads[:result]
+      include Dry::Monads::Do.for(:call)
+
+      class << self
+        def call(...)
+          self.new(...).call
+        end
+      end
+
+      # @param policy [:private, :public]
+      def initialize(policy:)
+        @policy = policy
+        @public_access_block = policy == :private ? true : false
+      end
+
+      def call
+        @bucket = yield get_bucket
+        @client = yield CMT::Build::S3MediaClient.call
+        _block_set = yield set_public_access_block
+        _policy_set = yield set_policy
+
+        Success()
+      end
+
+      def to_monad
+        Success(self)
+      end
+
+      private
+
+      attr_reader :bucket, :client, :policy, :public_access_block
+
+      def get_bucket
+        return Failure(CMT::Failure.new(
+          context: "#{self.class.name}.#{__callee__}",
+          message: "No client media_bucket setting specified"
+        )) unless CMT.config.client.respond_to?(:media_bucket)
+
+        Success(CMT.config.client.media_bucket)
+      end
+
+      def set_public_access_block
+        resp = client.put_public_access_block({
+          bucket: bucket,
+          public_access_block_configuration: public_access_block_config
+        })
+      rescue => err
+        msg = "#{err.message} IN #{err.backtrace[0]}"
+        Failure(
+          CMT::Failure.new(
+            context: "#{self.class.name}.#{__callee__}",
+            message: msg
+          )
+        )
+      else
+        Success(resp)
+      end
+
+      def public_access_block_config
+        {
+          block_public_acls: public_access_block,
+          ignore_public_acls: public_access_block,
+          block_public_policy: public_access_block,
+          restrict_public_buckets: public_access_block,
+        }
+      end
+
+      def set_policy
+        if policy == :public
+          resp = client.put_bucket_policy({
+            bucket: bucket,
+            policy: public_policy
+          })
+        else
+          resp = client.delete_bucket_policy({
+            bucket: bucket
+          })
+        end
+      rescue => err
+        msg = "#{err.message} IN #{err.backtrace[0]}"
+        Failure(
+          CMT::Failure.new(
+            context: "#{self.class.name}.#{__callee__}",
+            message: msg
+          )
+        )
+      else
+        Success(resp)
+      end
+
+      def public_policy
+        "{\"Version\": \"2008-10-17\", \"Statement\": [ { \"Sid\": \"AllowPublicRead\", \"Effect\": \"Allow\", \"Principal\": { \"AWS\": \"*\" }, \"Action\": \"s3:GetObject\", \"Resource\": \"arn:aws:s3:::#{bucket}/*\" }, { \"Sid\": \"Stmt1546414471931\", \"Effect\": \"Allow\", \"Principal\": { \"AWS\": \"*\" }, \"Action\": \"s3:ListBucket\", \"Resource\": \"arn:aws:s3:::#{bucket}\" } ]}"
+      end
+    end
+  end
+end

lib/collectionspace_migration_tools/validate/config_client_contract.rb

@@ -26,6 +26,7 @@ class ConfigClientContract < CMT::Validate::ApplicationContract
         required(:s3_delimiter).filled(:string)
         required(:max_media_upload_threads).filled(:integer)
         required(:media_with_blob_upload_delay).filled(:integer)
+        optional(:media_bucket).maybe(:string)
       end
 
       rule(:base_dir) do

lib/collectionspace_migration_tools/validate/config_system_contract.rb

@@ -9,6 +9,7 @@ class ConfigSystemContract < CMT::Validate::ApplicationContract
         required(:csv_chunk_size).filled(:integer)
         required(:max_threads).filled(:integer)
         required(:aws_profile).filled(:string)
+        optional(:aws_media_ingest_profile).maybe(:string)
       end
     end
   end

lib/collectionspace_migration_tools/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module CollectionspaceMigrationTools
-  VERSION = '2.1.0'
+  VERSION = '2.2.0'
 end

lib/tasks/bucket.thor

@@ -14,12 +14,28 @@ class Bucket < Thor
       ->(failure){ puts failure.to_s; exit(1) }
     )
   end
-  
+
   desc 'objs', 'returns keys of objects in bucket'
   def objs
     CMT::S3::Bucket.objects.either(
       ->(list){ handle_list(list) },
-      ->(failure){ puts failure.to_s, exit(1) }
+      ->(failure){ puts failure.to_s; exit(1) }
+    )
+  end
+
+  desc 'private', 'sets policy of MEDIA INGEST bucket to private'
+  def private
+    CMT::S3::BucketPolicySetter.call(policy: :private).either(
+      ->(success){ puts "Success"; exit(0) },
+      ->(failure){ puts failure.to_s; exit(1) }
+    )
+  end
+
+  desc 'public', 'sets policy of MEDIA INGEST bucket to public for ingest'
+  def public
+    CMT::S3::BucketPolicySetter.call(policy: :public).either(
+      ->(success){ puts "Success"; exit(0) },
+      ->(failure){ puts failure.to_s; exit(1) }
     )
   end
 

sample_client_config.yml

@@ -142,6 +142,9 @@ client:
   # This is the bucket we upload CSXML objects into, which uses AWS Lambda to call ingest for
   #   each object. It is NOT the bucket where client media files are uploaded.
   s3_bucket: "bucket_name"
+  # The name of the S3 bucket from which client media files will be ingested. This is optional, since some migrations
+  #   do not include media files.
+  media_bucket: "other_bucket_name"
 database:
   port: 5432
   db_password: dbpassword

sample_system_config.yml

@@ -3,3 +3,4 @@ system:
   max_threads: 10
   max_processes: 6
   aws_profile: collectionspacemigrationtools
+  aws_media_ingest_profile: cs-media