Merge branch 'main' into feature/17534

.gitignore

@@ -126,3 +126,4 @@ jobs/payment-jobs/tests/docker/ftp
 ACK.INBOX.F12022020202
 queue_services/payment-reconciliations/ACK.*
 queue_services/payment-reconciliations/FEEDBACK.*
+jobs/notebook-report/data/

jobs/notebook-report/Dockerfile

@@ -19,8 +19,7 @@ WORKDIR /opt/app-root
 # Install the requirements
 COPY ./requirements.txt .
 
-#RUN pip install --upgrade pip
-RUN pip install pip==20.3.3
+RUN pip install --upgrade pip
 RUN pip install --no-cache-dir -r requirements.txt
 
 COPY . .

jobs/notebook-report/Makefile

@@ -45,14 +45,14 @@ clean-test: ## clean test files
 build-req: clean ## Upgrade requirements
 	test -f venv/bin/activate || python3.8 -m venv  $(CURRENT_ABS_DIR)/venv ;\
 	. venv/bin/activate ;\
-	pip install pip==21.1.2 ;\
+	pip install --upgrade pip ;\
 	pip install -Ur requirements/prod.txt ;\
 	pip freeze | sort > requirements.txt
 
 install: clean ## Install python virtrual environment
 	test -f venv/bin/activate || python3.8 -m venv  $(CURRENT_ABS_DIR)/venv ;\
 	. venv/bin/activate ;\
-	pip install pip==21.1.2 ;\
+	pip install --upgrade pip ;\
 	pip install -Ur requirements.txt
 
 install-dev: ## Install local application

jobs/notebook-report/README.md

@@ -1,5 +1,8 @@
 # Notebook Report
 
+### Important: 1Pass vault does not exist, and secrets are to be updated in OCP, cronjob.yaml will need to be updated
+### to include the new key/value pairs. 
+
 Generate notebook report
 
 ## Development Environment

jobs/notebook-report/config.py

@@ -11,12 +11,17 @@ class Config(object):
     SENDER_EMAIL = os.getenv('SENDER_EMAIL', '')
     ERROR_EMAIL_RECIPIENTS = os.getenv('ERROR_EMAIL_RECIPIENTS', '')
     DAILY_RECONCILIATION_RECIPIENTS = os.getenv('DAILY_RECONCILIATION_RECIPIENTS', '')
+    VS_DAILY_RECONCILIATION_RECIPIENTS = os.getenv('VS_DAILY_RECONCILIATION_RECIPIENTS', '')
+    CSO_DAILY_RECONCILIATION_RECIPIENTS = os.getenv('CSO_DAILY_RECONCILIATION_RECIPIENTS', '')
     WEEKLY_PAY_RECIPIENTS = os.getenv('WEEKLY_PAY_RECIPIENTS', '')
     MONTHLY_RECONCILIATION_RECIPIENTS = os.getenv('MONTHLY_RECONCILIATION_RECIPIENTS', '')
+    VS_MONTHLY_RECONCILIATION_RECIPIENTS = os.getenv('VS_MONTHLY_RECONCILIATION_RECIPIENTS', '')
+    CSO_MONTHLY_RECONCILIATION_RECIPIENTS = os.getenv('CSO_MONTHLY_RECONCILIATION_RECIPIENTS', '')
     EMAIL_SMTP = os.getenv('EMAIL_SMTP', '')
     ENVIRONMENT = os.getenv('ENVIRONMENT', '')
-    WEEKLY_REPORT_DATES = os.getenv('WEEKLY_REPORT_DATES', '1')
-    MONTHLY_REPORT_DATES = os.getenv('MONTHLY_REPORT_DATES', '1')
+    WEEKLY_REPORT_DATES = os.getenv('WEEKLY_REPORT_DATES', '[1]')
+    MONTHLY_REPORT_DATES = os.getenv('MONTHLY_REPORT_DATES', '[1]')
+    PARTNER_CODES = os.getenv('PARTNER_CODES', 'CSO,VS')
 
     # POSTGRESQL
     PAY_USER = os.getenv('PAY_USER', '')

jobs/notebook-report/daily/reconciliation_details.ipynb → jobs/notebook-report/daily/cso_reconciliation_details.ipynb

@@ -3,7 +3,15 @@
   {
    "cell_type": "markdown",
    "metadata": {
-    "collapsed": false
+    "collapsed": false,
+    "editable": true,
+    "jupyter": {
+     "outputs_hidden": false
+    },
+    "slideshow": {
+     "slide_type": ""
+    },
+    "tags": []
    },
    "source": [
     "# Invoice Reconciliation Daily Stats"
@@ -14,10 +22,18 @@
    "execution_count": null,
    "metadata": {
     "collapsed": false,
+    "editable": true,
+    "jupyter": {
+     "outputs_hidden": false
+    },
     "pycharm": {
      "is_executing": false,
      "name": "#%%\n"
-    }
+    },
+    "slideshow": {
+     "slide_type": ""
+    },
+    "tags": []
    },
    "outputs": [],
    "source": [
@@ -32,7 +48,33 @@
   {
    "cell_type": "markdown",
    "metadata": {
-    "collapsed": false
+    "editable": true,
+    "slideshow": {
+     "slide_type": ""
+    },
+    "tags": [
+     "parameters"
+    ]
+   },
+   "source": [
+    "# Parameters cell for external parameters via papermill (job running this notebook will insert a parameter cell below this). This cell has a tag of with the name \"parameters\" that is used by papermill\n",
+    "\n",
+    "e.g.\n",
+    "param1 = \"some_value\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "collapsed": false,
+    "editable": true,
+    "jupyter": {
+     "outputs_hidden": false
+    },
+    "slideshow": {
+     "slide_type": ""
+    },
+    "tags": []
    },
    "source": [
     "This will create the connection to the database and prep the jupyter magic for SQL"
@@ -43,10 +85,18 @@
    "execution_count": null,
    "metadata": {
     "collapsed": false,
+    "editable": true,
+    "jupyter": {
+     "outputs_hidden": false
+    },
     "pycharm": {
      "is_executing": false,
      "name": "#%%\n"
-    }
+    },
+    "slideshow": {
+     "slide_type": ""
+    },
+    "tags": []
    },
    "outputs": [],
    "source": [
@@ -56,7 +106,10 @@
   {
    "cell_type": "markdown",
    "metadata": {
-    "collapsed": false
+    "collapsed": false,
+    "jupyter": {
+     "outputs_hidden": false
+    }
    },
    "source": [
     "Simplest query to run to ensure our libraries are loaded and our DB connection is working"
@@ -67,21 +120,27 @@
    "execution_count": null,
    "metadata": {
     "collapsed": false,
+    "jupyter": {
+     "outputs_hidden": false
+    },
     "pycharm": {
      "is_executing": false,
      "name": "#%%\n"
     }
    },
    "outputs": [],
    "source": [
-    "%%sql \n",
+    "%%sql\n",
     "select now() AT TIME ZONE 'PST' as current_date"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {
-    "collapsed": false
+    "collapsed": false,
+    "jupyter": {
+     "outputs_hidden": false
+    }
    },
    "source": [
     "Daily query for details data."
@@ -92,19 +151,27 @@
    "execution_count": null,
    "metadata": {
     "collapsed": false,
+    "editable": true,
+    "jupyter": {
+     "outputs_hidden": false
+    },
     "pycharm": {
      "is_executing": false,
      "name": "#%%\n"
-    }
+    },
+    "slideshow": {
+     "slide_type": ""
+    },
+    "tags": []
    },
    "outputs": [],
    "source": [
     "%%sql daily_reconciliation_details  <<\n",
-    "SELECT i.created_on, i.updated_on, i.id, ir.invoice_number, (i.total - i.service_fees) AS subtotal, i.service_fees, i.total, i.disbursement_status_code, i.payment_method_code, i.invoice_status_code, i.corp_type_code \n",
+    "SELECT i.created_on, i.updated_on, i.id, ir.invoice_number, (i.total - i.service_fees) AS subtotal, i.service_fees, i.total, i.disbursement_status_code, i.payment_method_code, i.invoice_status_code, i.corp_type_code\n",
     "FROM invoices i join invoice_references ir on ir.invoice_id = i.id\n",
-    "WHERE i.corp_type_code = 'CSO' \n",
-    "AND i.invoice_status_code IN ('PAID', 'REFUNDED', 'CANCELLED', 'CREDITED') \n",
-    "AND i.payment_method_code IN ('PAD','EJV', 'DRAWDOWN') \n",
+    "WHERE i.corp_type_code = :partner_code\n",
+    "AND i.invoice_status_code IN ('PAID', 'REFUNDED', 'CANCELLED', 'CREDITED')\n",
+    "AND i.payment_method_code IN ('PAD','EJV', 'DRAWDOWN')\n",
     "AND date(i.created_on)  > date(current_date - 1 - interval '1 days')\n",
     "AND date(i.created_on)  <= date(current_date - 1)\n",
     "ORDER BY 1;"
@@ -113,7 +180,10 @@
   {
    "cell_type": "markdown",
    "metadata": {
-    "collapsed": false
+    "collapsed": false,
+    "jupyter": {
+     "outputs_hidden": false
+    }
    },
    "source": [
     "Save to CSV"
@@ -124,29 +194,37 @@
    "execution_count": null,
    "metadata": {
     "collapsed": false,
+    "editable": true,
+    "jupyter": {
+     "outputs_hidden": false
+    },
     "pycharm": {
      "is_executing": false,
      "name": "#%%\n"
-    }
+    },
+    "slideshow": {
+     "slide_type": ""
+    },
+    "tags": []
    },
    "outputs": [],
    "source": [
-    "filename = os.path.join(os.getcwd(), r'data/')+'daily_reconciliation_' + datetime.strftime(datetime.now()-timedelta(1), '%Y-%m-%d') +'.csv'\n",
+    "filename = os.path.join(os.getcwd(), r'data/')+partner_code+'_daily_reconciliation_' + datetime.strftime(datetime.now()-timedelta(1), '%Y-%m-%d') +'.csv'\n",
     "df = daily_reconciliation_details.DataFrame()\n",
     "\n",
-    "with open(filename, 'w') as f:   \n",
+    "with open(filename, 'w') as f:\n",
     "    f.write('Daily Reconciliation Details:\\n\\n')\n",
     "    if df.empty:\n",
     "        f.write('No Data Retrieved')\n",
-    "    else:        \n",
-    "        df.to_csv(f, sep=',', encoding='utf-8', index=False)"
+    "    else:\n",
+    "        df.to_csv(f, sep=',', encoding='utf-8', index=False)\n"
    ]
   }
  ],
  "metadata": {
   "celltoolbar": "Tags",
   "kernelspec": {
-   "display_name": "Python 3.10.4 64-bit",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },
@@ -160,7 +238,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.4"
+   "version": "3.11.4"
   },
   "pycharm": {
    "stem_cell": {
@@ -178,5 +256,5 @@
   }
  },
  "nbformat": 4,
- "nbformat_minor": 2
+ "nbformat_minor": 4
 }