17827 - initial TDI17 file retrieval setup (#1274)

* 17827 - initial TDI17 file retrieval setup

* lint fix

jobs/ftp-poller/config.py

@@ -73,6 +73,16 @@ class _Config(object):  # pylint: disable=too-few-public-methods
     CGI_SFTP_BACKUP_DIRECTORY = os.getenv('CGI_SFTP_BACKUP_DIRECTORY', '/backup')
     CGI_SFTP_DIRECTORY = os.getenv('CGI_SFTP_DIRECTORY', '/data')
 
+    # EFT FTP CONFIG
+    EFT_SFTP_HOST = os.getenv('EFT_SFTP_HOST', 'localhost')
+    EFT_SFTP_USER_NAME = os.getenv('EFT_SFTP_USER_NAME', 'foo')
+    EFT_SFTP_PASSWORD = os.getenv('EFT_SFTP_PASSWORD', '')
+    EFT_SFTP_DIRECTORY = os.getenv('EFT_SFTP_DIRECTORY', '/eft')
+    EFT_SFTP_BACKUP_DIRECTORY = os.getenv('EFT_SFTP_BACKUP_DIRECTORY', '/backup')
+    EFT_SFTP_VERIFY_HOST = os.getenv('EFT_SFTP_VERIFY_HOST', 'True')
+    EFT_SFTP_PORT = os.getenv('EFT_SFTP_PORT', 22)
+    EFT_SFTP_HOST_KEY = os.getenv('EFT_SFTP_HOST_KEY', '')
+
     # CGI File specific configs
     CGI_TRIGGER_FILE_SUFFIX = os.getenv('CGI_TRIGGER_FILE_SUFFIX', '.TRG')
     CGI_ACK_FILE_PREFIX = os.getenv('CGI_ACK_FILE_PREFIX', 'ACK')
@@ -110,6 +120,17 @@ class _Config(object):  # pylint: disable=too-few-public-methods
             'SFTP_PORT': CAS_SFTP_PORT,  # same as CAS
             'FTP_PRIVATE_KEY_LOCATION': BCREG_CGI_FTP_PRIVATE_KEY_LOCATION,  # different user.so not same as CAS
             'BCREG_FTP_PRIVATE_KEY_PASSPHRASE': BCREG_CGI_FTP_PRIVATE_KEY_PASSPHRASE
+        },
+        # FUTURE - specific configuration values TBD - initial set up code
+        'EFT': {
+            'SFTP_HOST': EFT_SFTP_HOST,
+            'SFTP_USERNAME': EFT_SFTP_USER_NAME,
+            'SFTP_PASSWORD': EFT_SFTP_PASSWORD,
+            'SFTP_VERIFY_HOST': EFT_SFTP_VERIFY_HOST,
+            'SFTP_HOST_KEY': EFT_SFTP_HOST_KEY,
+            'SFTP_PORT': EFT_SFTP_PORT,
+            'FTP_PRIVATE_KEY_LOCATION': BCREG_FTP_PRIVATE_KEY_LOCATION,
+            'BCREG_FTP_PRIVATE_KEY_PASSPHRASE': BCREG_FTP_PRIVATE_KEY_PASSPHRASE
         }
     }
 
@@ -119,6 +140,7 @@ class _Config(object):  # pylint: disable=too-few-public-methods
     MINIO_ACCESS_SECRET = os.getenv('MINIO_ACCESS_SECRET')
     MINIO_BUCKET_NAME = os.getenv('MINIO_BUCKET_NAME', 'payment-sftp')
     MINIO_CGI_BUCKET_NAME = os.getenv('MINIO_CGI_BUCKET_NAME', 'cgi-ejv')
+    MINIO_EFT_BUCKET_NAME = os.getenv('MINIO_EFT_BUCKET_NAME', 'eft-sftp')
     MINIO_SECURE = True
 
     SENTRY_ENABLE = os.getenv('SENTRY_ENABLE', 'False')

jobs/ftp-poller/invoke_jobs.py

@@ -67,6 +67,7 @@ def shell_context():
 def run(job_name):
     from tasks.cas_poller_ftp import CASPollerFtpTask
     from tasks.cgi_feeder_poller_task import CGIFeederPollerTask
+    from tasks.eft_poller_ftp import EFTPollerFtpTask
 
     application = create_app()
 
@@ -77,6 +78,9 @@ def run(job_name):
     elif job_name == 'CGI_FTP_POLLER':
         CGIFeederPollerTask.poll_ftp()
         application.logger.info(f'<<<< Completed Polling CGI FTP >>>>')
+    elif job_name == 'EFT_FTP_POLLER':
+        EFTPollerFtpTask.poll_ftp()
+        application.logger.info(f'<<<< Completed Polling EFT FTP >>>>')
     else:
         application.logger.debug('No valid args passed.Exiting job without running any ***************')
 

jobs/ftp-poller/run_eft_ftp_poller.sh

@@ -0,0 +1,3 @@
+#! /bin/sh
+echo 'run invoke_jobs.py EFT_FTP_POLLER'
+python3 invoke_jobs.py EFT_FTP_POLLER

jobs/ftp-poller/tasks/eft_poller_ftp.py

@@ -0,0 +1,80 @@
+# Copyright © 2023 Province of British Columbia
+#
+# Licensed under the Apache License, Version 2.0 (the 'License');
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an 'AS IS' BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Service to manage EFT TDI17 files."""
+from typing import List
+
+from flask import current_app
+from paramiko.sftp_attr import SFTPAttributes
+
+from services.sftp import SFTPService
+from utils.constants import EFT_MESSAGE_TYPE
+from utils.utils import publish_to_queue, upload_to_minio
+
+
+class EFTPollerFtpTask:  # pylint:disable=too-few-public-methods
+    """Task to Poll EFT FTP."""
+
+    @classmethod
+    def poll_ftp(cls):
+        """Poll SFTP.
+
+        Steps:
+        1. List Files.
+        2. If file exists ,
+                copy to minio
+                archive to back up folder
+                send jms message
+        """
+        payment_file_list: List[str] = []
+        with SFTPService.get_connection('EFT') as sftp_client:
+            try:
+                ftp_dir: str = current_app.config.get('EFT_SFTP_DIRECTORY')
+                file_list: List[SFTPAttributes] = sftp_client.listdir_attr(ftp_dir)
+                current_app.logger.info(f'Found {len(file_list)} to be copied.')
+                for file in file_list:
+                    file_name = file.filename
+                    file_full_name = ftp_dir + '/' + file_name
+                    current_app.logger.info(f'Processing file  {file_full_name} started-----.')
+                    if EFTPollerFtpTask._is_valid_payment_file(sftp_client, file_full_name):
+                        upload_to_minio(file, file_full_name, sftp_client, current_app.config['MINIO_EFT_BUCKET_NAME'])
+                        payment_file_list.append(file_name)
+
+                if len(payment_file_list) > 0:
+                    EFTPollerFtpTask._post_process(sftp_client, payment_file_list)
+
+            except Exception as e:  # NOQA # pylint: disable=broad-except
+                current_app.logger.error(e)
+        return payment_file_list
+
+    @classmethod
+    def _post_process(cls, sftp_client, payment_file_list: List[str]):
+        """
+        Post processing of the file.
+
+        1.Move the file to backup folder
+        2.Send a message to queue
+        """
+        cls._move_file_to_backup(sftp_client, payment_file_list)
+        publish_to_queue(payment_file_list, EFT_MESSAGE_TYPE, location=current_app.config.get('MINIO_EFT_BUCKET_NAME'))
+
+    @classmethod
+    def _move_file_to_backup(cls, sftp_client, payment_file_list):
+        ftp_backup_dir: str = current_app.config.get('EFT_SFTP_BACKUP_DIRECTORY')
+        ftp_dir: str = current_app.config.get('EFT_SFTP_DIRECTORY')
+        for file_name in payment_file_list:
+            sftp_client.rename(ftp_dir + '/' + file_name, ftp_backup_dir + '/' + file_name)
+
+    @classmethod
+    def _is_valid_payment_file(cls, sftp_client, file_name):
+        return sftp_client.isfile(file_name)

jobs/ftp-poller/utils/constants.py

@@ -16,3 +16,4 @@
 CAS_MESSAGE_TYPE = 'bc.registry.payment.casSettlementUploaded'
 CGI_ACK_MESSAGE_TYPE = 'bc.registry.payment.cgi.ACKReceived'
 CGI_FEEDBACK_MESSAGE_TYPE = 'bc.registry.payment.cgi.FEEDBACKReceived'
+EFT_MESSAGE_TYPE = 'bc.registry.payment.eft.fileUploaded'