18758 - split notebooks for different partners, update vs queries

.gitignore

@@ -126,3 +126,4 @@ jobs/payment-jobs/tests/docker/ftp
 ACK.INBOX.F12022020202
 queue_services/payment-reconciliations/ACK.*
 queue_services/payment-reconciliations/FEEDBACK.*
+jobs/notebook-report/data/

jobs/notebook-report/config.py

@@ -19,8 +19,8 @@ class Config(object):
     CSO_MONTHLY_RECONCILIATION_RECIPIENTS = os.getenv('CSO_MONTHLY_RECONCILIATION_RECIPIENTS', '')
     EMAIL_SMTP = os.getenv('EMAIL_SMTP', '')
     ENVIRONMENT = os.getenv('ENVIRONMENT', '')
-    WEEKLY_REPORT_DATES = os.getenv('WEEKLY_REPORT_DATES', '1')
-    MONTHLY_REPORT_DATES = os.getenv('MONTHLY_REPORT_DATES', '1')
+    WEEKLY_REPORT_DATES = os.getenv('WEEKLY_REPORT_DATES', '[1]')
+    MONTHLY_REPORT_DATES = os.getenv('MONTHLY_REPORT_DATES', '[1]')
     PARTNER_CODES = os.getenv('PARTNER_CODES', 'CSO,VS')
 
     # POSTGRESQL

jobs/notebook-report/daily/vs_reconciliation_details.ipynb

@@ -0,0 +1,260 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "collapsed": false,
+    "editable": true,
+    "jupyter": {
+     "outputs_hidden": false
+    },
+    "slideshow": {
+     "slide_type": ""
+    },
+    "tags": []
+   },
+   "source": [
+    "# Invoice Reconciliation Daily Stats"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": false,
+    "editable": true,
+    "jupyter": {
+     "outputs_hidden": false
+    },
+    "pycharm": {
+     "is_executing": false,
+     "name": "#%%\n"
+    },
+    "slideshow": {
+     "slide_type": ""
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "from datetime import datetime, timedelta\n",
+    "from config import Config\n",
+    "\n",
+    "%load_ext sql\n",
+    "%config SqlMagic.displaylimit = 5"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "editable": true,
+    "slideshow": {
+     "slide_type": ""
+    },
+    "tags": [
+     "parameters"
+    ]
+   },
+   "source": [
+    "# Parameters cell for external parameters via papermill (job running this notebook will insert a parameter cell below this). This cell has a tag of with the name \"parameters\" that is used by papermill\n",
+    "\n",
+    "e.g.\n",
+    "param1 = \"some_value\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "collapsed": false,
+    "editable": true,
+    "jupyter": {
+     "outputs_hidden": false
+    },
+    "slideshow": {
+     "slide_type": ""
+    },
+    "tags": []
+   },
+   "source": [
+    "This will create the connection to the database and prep the jupyter magic for SQL"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": false,
+    "editable": true,
+    "jupyter": {
+     "outputs_hidden": false
+    },
+    "pycharm": {
+     "is_executing": false,
+     "name": "#%%\n"
+    },
+    "slideshow": {
+     "slide_type": ""
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "%sql $Config.SQLALCHEMY_DATABASE_URI"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "collapsed": false,
+    "jupyter": {
+     "outputs_hidden": false
+    }
+   },
+   "source": [
+    "Simplest query to run to ensure our libraries are loaded and our DB connection is working"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": false,
+    "jupyter": {
+     "outputs_hidden": false
+    },
+    "pycharm": {
+     "is_executing": false,
+     "name": "#%%\n"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "%%sql\n",
+    "select now() AT TIME ZONE 'PST' as current_date"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "collapsed": false,
+    "jupyter": {
+     "outputs_hidden": false
+    }
+   },
+   "source": [
+    "Daily query for details data."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": false,
+    "editable": true,
+    "jupyter": {
+     "outputs_hidden": false
+    },
+    "pycharm": {
+     "is_executing": false,
+     "name": "#%%\n"
+    },
+    "slideshow": {
+     "slide_type": ""
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "%%sql daily_reconciliation_details  <<\n",
+    "SELECT i.created_on, i.updated_on, i.id, ir.invoice_number, (i.total - i.service_fees) AS subtotal, i.service_fees, i.total, i.disbursement_status_code, i.payment_method_code, i.invoice_status_code, i.corp_type_code\n",
+    "FROM invoices i join invoice_references ir on ir.invoice_id = i.id\n",
+    "WHERE i.corp_type_code = :partner_code\n",
+    "AND i.invoice_status_code IN ('PAID', 'REFUNDED', 'CANCELLED', 'CREDITED')\n",
+    "AND i.payment_method_code IN ('PAD','EJV', 'DRAWDOWN')\n",
+    "AND date(i.created_on)  > date(current_date - 1 - interval '1 days')\n",
+    "AND date(i.created_on)  <= date(current_date - 1)\n",
+    "ORDER BY 1;"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "collapsed": false,
+    "jupyter": {
+     "outputs_hidden": false
+    }
+   },
+   "source": [
+    "Save to CSV"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": false,
+    "editable": true,
+    "jupyter": {
+     "outputs_hidden": false
+    },
+    "pycharm": {
+     "is_executing": false,
+     "name": "#%%\n"
+    },
+    "slideshow": {
+     "slide_type": ""
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "filename = os.path.join(os.getcwd(), r'data/')+partner_code+'_daily_reconciliation_' + datetime.strftime(datetime.now()-timedelta(1), '%Y-%m-%d') +'.csv'\n",
+    "df = daily_reconciliation_details.DataFrame()\n",
+    "\n",
+    "with open(filename, 'w') as f:\n",
+    "    f.write('Daily Reconciliation Details:\\n\\n')\n",
+    "    if df.empty:\n",
+    "        f.write('No Data Retrieved')\n",
+    "    else:\n",
+    "        df.to_csv(f, sep=',', encoding='utf-8', index=False)\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "celltoolbar": "Tags",
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.4"
+  },
+  "pycharm": {
+   "stem_cell": {
+    "cell_type": "raw",
+    "metadata": {
+     "collapsed": false
+    },
+    "source": []
+   }
+  },
+  "vscode": {
+   "interpreter": {
+    "hash": "fcb35bce15c55b4cacb5112e543368f86c7f98ed17acd45e6841ee83ed1df6e3"
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}