From b8e6a0ac28f64473062cf04dfbc22aa0f1e85b00 Mon Sep 17 00:00:00 2001 From: Ratheesh kumar R Date: Mon, 23 Sep 2024 16:29:25 -0700 Subject: [PATCH 01/22] DF-3081 - Created Dashboard to show From count statistics --- .../prohibition_web_svc/blueprints/forms.py | 16 + python/prohibition_web_svc/http_responses.py | 5 + .../middleware/form_middleware.py | 55 ++ .../frontend_web_app/package-lock.json | 472 +++++++++++++++++- .../frontend_web_app/package.json | 1 + .../src/api/formStatisticsApi.js | 28 ++ .../src/components/common/Header/Header.js | 70 ++- .../src/components/common/Header/header.scss | 26 + .../userAdminDashboard/formStatistics.js | 88 ++++ .../userAdminDashboard/formStatistics.scss | 79 +++ .../frontend_web_app/src/routes/appRouter.js | 2 + 11 files changed, 800 insertions(+), 42 deletions(-) create mode 100644 roadside-forms-frontend/frontend_web_app/src/api/formStatisticsApi.js create mode 100644 roadside-forms-frontend/frontend_web_app/src/components/userAdminDashboard/formStatistics.js create mode 100644 roadside-forms-frontend/frontend_web_app/src/components/userAdminDashboard/formStatistics.scss diff --git a/python/prohibition_web_svc/blueprints/forms.py b/python/prohibition_web_svc/blueprints/forms.py index 5f5255c63..94564417f 100644 --- a/python/prohibition_web_svc/blueprints/forms.py +++ b/python/prohibition_web_svc/blueprints/forms.py @@ -112,3 +112,19 @@ def delete(form_type, form_id): if request.method == 'DELETE': return make_response({'error': 'method not implemented'}, 405) +@bp.route('/forms/statistics', methods=['GET']) +def get_statistics(): + """ + Get statistics about form usage (public endpoint) + """ + if request.method == 'GET': + kwargs = helper.middle_logic([ + {"try": form_middleware.get_form_statistics, "fail": [ + # {"try": form_middleware.record_form_error, "fail": []}, + {"try": http_responses.server_error_response, "fail": []}, + ]}, + {"try": http_responses.successful_get_response, "fail": []}, + ], + request=request, + config=Config) + return kwargs.get('response') \ No newline at end of file diff --git a/python/prohibition_web_svc/http_responses.py b/python/prohibition_web_svc/http_responses.py index 1edaa349c..3496124b1 100644 --- a/python/prohibition_web_svc/http_responses.py +++ b/python/prohibition_web_svc/http_responses.py @@ -1,5 +1,6 @@ from flask import make_response import logging +from flask import jsonify def successful_create_response(**kwargs) -> tuple: @@ -86,3 +87,7 @@ def failed_validation(**kwargs) -> tuple: def no_payload(**kwargs) -> tuple: kwargs['response'] = make_response({'error': 'no payload'}, 400) return True, kwargs + +def successful_get_response(**kwargs) -> tuple: + response = make_response(jsonify(kwargs.get('response_dict')), 200) + return True, {'response': response} diff --git a/python/prohibition_web_svc/middleware/form_middleware.py b/python/prohibition_web_svc/middleware/form_middleware.py index 736329333..8ae348dce 100644 --- a/python/prohibition_web_svc/middleware/form_middleware.py +++ b/python/prohibition_web_svc/middleware/form_middleware.py @@ -5,6 +5,8 @@ from datetime import datetime from cerberus import Validator from dataclasses import asdict +from sqlalchemy import func, case +from sqlalchemy.sql import expression from flask import jsonify, make_response from python.prohibition_web_svc.models import db, Form from python.prohibition_web_svc.config import Config @@ -245,3 +247,56 @@ def convert_vancouver_to_utc(iso_datetime_string: str) -> datetime: utc_timezone = pytz.timezone("UTC") printed = iso8601.parse_date(iso_datetime_string) return printed.astimezone(utc_timezone).replace(tzinfo=None) + +def get_form_statistics(**kwargs) -> tuple: + try: + results = db.session.query( + Form.form_type, + func.count().label('total_forms'), + func.sum(case( + (expression.and_( + Form.printed_timestamp.is_(None), + Form.spoiled_timestamp.is_(None), + expression.or_(Form.user_guid.isnot(None), Form.lease_expiry.isnot(None)) + ), 1), + else_=0 + )).label('leased_forms'), + func.sum(case( + (expression.or_( + Form.printed_timestamp.isnot(None), + Form.spoiled_timestamp.isnot(None) + ), 1), + else_=0 + )).label('total_used_forms'), + func.sum(case( + (expression.and_( + Form.printed_timestamp.is_(None), + Form.spoiled_timestamp.is_(None), + Form.user_guid.is_(None), + Form.lease_expiry.is_(None) + ), 1), + else_=0 + )).label('available_forms') + ).group_by(Form.form_type).order_by(Form.form_type).all() + + stats = [ + { + 'form_type': r.form_type, + 'total_forms': r.total_forms, + 'leased_forms': r.leased_forms, + 'total_used_forms': r.total_used_forms, + 'available_forms': r.available_forms + } for r in results + ] + + kwargs['response_dict'] = stats + return True, kwargs + except Exception as e: + logging.error(f"Error in get_form_statistics: {str(e)}") + # kwargs['error'] = { + # 'error_code': ErrorCode.F03, + # 'error_details': str(e), + # 'event_type': 'form_statistics', + # 'func': get_form_statistics, + # } + return False, kwargs \ No newline at end of file diff --git a/roadside-forms-frontend/frontend_web_app/package-lock.json b/roadside-forms-frontend/frontend_web_app/package-lock.json index e1d7fdbd0..399a132e8 100644 --- a/roadside-forms-frontend/frontend_web_app/package-lock.json +++ b/roadside-forms-frontend/frontend_web_app/package-lock.json @@ -36,6 +36,7 @@ "react-router-dom": "6.19.0", "react-scripts": "5.0.1", "react-toastify": "^9.1.3", + "recharts": "^2.12.7", "recoil": "^0.7.7", "recoil-nexus": "^0.5.0", "use-between": "^1.3.5", @@ -9389,6 +9390,60 @@ "@types/node": "*" } }, + "node_modules/@types/d3-array": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/@types/d3-array/-/d3-array-3.2.1.tgz", + "integrity": "sha512-Y2Jn2idRrLzUfAKV2LyRImR+y4oa2AntrgID95SHJxuMUrkNXmanDSed71sRNZysveJVt1hLLemQZIady0FpEg==" + }, + "node_modules/@types/d3-color": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/@types/d3-color/-/d3-color-3.1.3.tgz", + "integrity": "sha512-iO90scth9WAbmgv7ogoq57O9YpKmFBbmoEoCHDB2xMBY0+/KVrqAaCDyCE16dUspeOvIxFFRI+0sEtqDqy2b4A==" + }, + "node_modules/@types/d3-ease": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/d3-ease/-/d3-ease-3.0.2.tgz", + "integrity": "sha512-NcV1JjO5oDzoK26oMzbILE6HW7uVXOHLQvHshBUW4UMdZGfiY6v5BeQwh9a9tCzv+CeefZQHJt5SRgK154RtiA==" + }, + "node_modules/@types/d3-interpolate": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/d3-interpolate/-/d3-interpolate-3.0.4.tgz", + "integrity": "sha512-mgLPETlrpVV1YRJIglr4Ez47g7Yxjl1lj7YKsiMCb27VJH9W8NVM6Bb9d8kkpG/uAQS5AmbA48q2IAolKKo1MA==", + "dependencies": { + "@types/d3-color": "*" + } + }, + "node_modules/@types/d3-path": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@types/d3-path/-/d3-path-3.1.0.tgz", + "integrity": "sha512-P2dlU/q51fkOc/Gfl3Ul9kicV7l+ra934qBFXCFhrZMOL6du1TM0pm1ThYvENukyOn5h9v+yMJ9Fn5JK4QozrQ==" + }, + "node_modules/@types/d3-scale": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/@types/d3-scale/-/d3-scale-4.0.8.tgz", + "integrity": "sha512-gkK1VVTr5iNiYJ7vWDI+yUFFlszhNMtVeneJ6lUTKPjprsvLLI9/tgEGiXJOnlINJA8FyA88gfnQsHbybVZrYQ==", + "dependencies": { + "@types/d3-time": "*" + } + }, + "node_modules/@types/d3-shape": { + "version": "3.1.6", + "resolved": "https://registry.npmjs.org/@types/d3-shape/-/d3-shape-3.1.6.tgz", + "integrity": "sha512-5KKk5aKGu2I+O6SONMYSNflgiP0WfZIQvVUMan50wHsLG1G94JlxEVnCpQARfTtzytuY0p/9PXXZb3I7giofIA==", + "dependencies": { + "@types/d3-path": "*" + } + }, + "node_modules/@types/d3-time": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/d3-time/-/d3-time-3.0.3.tgz", + "integrity": "sha512-2p6olUZ4w3s+07q3Tm2dbiMZy5pCDfYwtLXXHUnVzXgQlZ/OyPtUz6OL382BkOuGlLXqfT+wqv8Fw2v8/0geBw==" + }, + "node_modules/@types/d3-timer": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/d3-timer/-/d3-timer-3.0.2.tgz", + "integrity": "sha512-Ps3T8E8dZDam6fUyNiMkekK3XUsaUEik+idO9/YjPtfj2qruF8tFBXS7XhtE4iIXBLxhmLjP3SXpLhVf21I9Lw==" + }, "node_modules/@types/doctrine": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/@types/doctrine/-/doctrine-0.0.3.tgz", @@ -14145,6 +14200,116 @@ "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", "dev": true }, + "node_modules/d3-array": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-3.2.4.tgz", + "integrity": "sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg==", + "dependencies": { + "internmap": "1 - 2" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-color": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-color/-/d3-color-3.1.0.tgz", + "integrity": "sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-ease": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-ease/-/d3-ease-3.0.1.tgz", + "integrity": "sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w==", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-format": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-format/-/d3-format-3.1.0.tgz", + "integrity": "sha512-YyUI6AEuY/Wpt8KWLgZHsIU86atmikuoOmCfommt0LYHiQSPjvX2AcFc38PX0CBpr2RCyZhjex+NS/LPOv6YqA==", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-interpolate": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-interpolate/-/d3-interpolate-3.0.1.tgz", + "integrity": "sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==", + "dependencies": { + "d3-color": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-path": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-path/-/d3-path-3.1.0.tgz", + "integrity": "sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ==", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-scale": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/d3-scale/-/d3-scale-4.0.2.tgz", + "integrity": "sha512-GZW464g1SH7ag3Y7hXjf8RoUuAFIqklOAq3MRl4OaWabTFJY9PN/E1YklhXLh+OQ3fM9yS2nOkCoS+WLZ6kvxQ==", + "dependencies": { + "d3-array": "2.10.0 - 3", + "d3-format": "1 - 3", + "d3-interpolate": "1.2.0 - 3", + "d3-time": "2.1.1 - 3", + "d3-time-format": "2 - 4" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-shape": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-3.2.0.tgz", + "integrity": "sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA==", + "dependencies": { + "d3-path": "^3.1.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-time": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-time/-/d3-time-3.1.0.tgz", + "integrity": "sha512-VqKjzBLejbSMT4IgbmVgDjpkYrNWUYJnbCGo874u7MMKIWsILRX+OpX/gTk8MqjpT1A/c6HY2dCA77ZN0lkQ2Q==", + "dependencies": { + "d3-array": "2 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-time-format": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/d3-time-format/-/d3-time-format-4.1.0.tgz", + "integrity": "sha512-dJxPBlzC7NugB2PDLwo9Q8JiTR3M3e4/XANkreKSUxF8vvXKqm1Yfq4Q5dl8budlunRVlUUaDUgFt7eA8D6NLg==", + "dependencies": { + "d3-time": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-timer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-timer/-/d3-timer-3.0.1.tgz", + "integrity": "sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA==", + "engines": { + "node": ">=12" + } + }, "node_modules/damerau-levenshtein": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/damerau-levenshtein/-/damerau-levenshtein-1.0.8.tgz", @@ -14248,6 +14413,11 @@ "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.4.3.tgz", "integrity": "sha512-VBBaLc1MgL5XpzgIP7ny5Z6Nx3UrRkIViUkPUdtl9aya5amy3De1gsUUSB1g3+3sExYNjCAsAznmukyxCb1GRA==" }, + "node_modules/decimal.js-light": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/decimal.js-light/-/decimal.js-light-2.5.1.tgz", + "integrity": "sha512-qIMFpTMZmny+MMIitAB6D7iVPEorVw6YQRWkvarTkT4tBeSLLiHzcwj6q0MmYSFCiVpiqPJTJEYIrpcPzVEIvg==" + }, "node_modules/decode-uri-component": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.2.tgz", @@ -16226,6 +16396,14 @@ "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" }, + "node_modules/fast-equals": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/fast-equals/-/fast-equals-5.0.1.tgz", + "integrity": "sha512-WF1Wi8PwwSY7/6Kx0vKXtw8RwuSGoM1bvDaJbu7MxDlR1vovZjIAKrnzyrThgAjm6JDTu0fVgWXDlMGspodfoQ==", + "engines": { + "node": ">=6.0.0" + } + }, "node_modules/fast-glob": { "version": "3.3.2", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.2.tgz", @@ -17993,6 +18171,14 @@ "node": ">= 0.4" } }, + "node_modules/internmap": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/internmap/-/internmap-2.0.3.tgz", + "integrity": "sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg==", + "engines": { + "node": ">=12" + } + }, "node_modules/interpret": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/interpret/-/interpret-2.2.0.tgz", @@ -26073,6 +26259,20 @@ "use-isomorphic-layout-effect": "^1.1.2" } }, + "node_modules/react-smooth": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/react-smooth/-/react-smooth-4.0.1.tgz", + "integrity": "sha512-OE4hm7XqR0jNOq3Qmk9mFLyd6p2+j6bvbPJ7qlB7+oo0eNcL2l7WQzG6MBnT3EXY6xzkLMUBec3AfewJdA0J8w==", + "dependencies": { + "fast-equals": "^5.0.1", + "prop-types": "^15.8.1", + "react-transition-group": "^4.4.5" + }, + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0", + "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0" + } + }, "node_modules/react-style-singleton": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/react-style-singleton/-/react-style-singleton-2.2.1.tgz", @@ -26236,6 +26436,44 @@ "node": ">=0.10.0" } }, + "node_modules/recharts": { + "version": "2.12.7", + "resolved": "https://registry.npmjs.org/recharts/-/recharts-2.12.7.tgz", + "integrity": "sha512-hlLJMhPQfv4/3NBSAyq3gzGg4h2v69RJh6KU7b3pXYNNAELs9kEoXOjbkxdXpALqKBoVmVptGfLpxdaVYqjmXQ==", + "dependencies": { + "clsx": "^2.0.0", + "eventemitter3": "^4.0.1", + "lodash": "^4.17.21", + "react-is": "^16.10.2", + "react-smooth": "^4.0.0", + "recharts-scale": "^0.4.4", + "tiny-invariant": "^1.3.1", + "victory-vendor": "^36.6.8" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "react": "^16.0.0 || ^17.0.0 || ^18.0.0", + "react-dom": "^16.0.0 || ^17.0.0 || ^18.0.0" + } + }, + "node_modules/recharts-scale": { + "version": "0.4.5", + "resolved": "https://registry.npmjs.org/recharts-scale/-/recharts-scale-0.4.5.tgz", + "integrity": "sha512-kivNFO+0OcUNu7jQquLXAxz1FIwZj8nrj+YkOKc5694NbjCvcT6aSZiIzNzd2Kul4o4rTto8QVR9lMNtxD4G1w==", + "dependencies": { + "decimal.js-light": "^2.4.1" + } + }, + "node_modules/recharts/node_modules/clsx": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz", + "integrity": "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==", + "engines": { + "node": ">=6" + } + }, "node_modules/recoil": { "version": "0.7.7", "resolved": "https://registry.npmjs.org/recoil/-/recoil-0.7.7.tgz", @@ -28679,8 +28917,7 @@ "node_modules/tiny-invariant": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/tiny-invariant/-/tiny-invariant-1.3.3.tgz", - "integrity": "sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg==", - "dev": true + "integrity": "sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg==" }, "node_modules/tiny-warning": { "version": "1.0.3", @@ -29615,6 +29852,27 @@ "integrity": "sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ==", "dev": true }, + "node_modules/victory-vendor": { + "version": "36.9.2", + "resolved": "https://registry.npmjs.org/victory-vendor/-/victory-vendor-36.9.2.tgz", + "integrity": "sha512-PnpQQMuxlwYdocC8fIJqVXvkeViHYzotI+NJrCuav0ZYFoq912ZHBk3mCeuj+5/VpodOjPe1z0Fk2ihgzlXqjQ==", + "dependencies": { + "@types/d3-array": "^3.0.3", + "@types/d3-ease": "^3.0.0", + "@types/d3-interpolate": "^3.0.1", + "@types/d3-scale": "^4.0.2", + "@types/d3-shape": "^3.1.0", + "@types/d3-time": "^3.0.0", + "@types/d3-timer": "^3.0.0", + "d3-array": "^3.1.6", + "d3-ease": "^3.0.1", + "d3-interpolate": "^3.0.1", + "d3-scale": "^4.0.2", + "d3-shape": "^3.1.0", + "d3-time": "^3.0.0", + "d3-timer": "^3.0.1" + } + }, "node_modules/vm-browserify": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vm-browserify/-/vm-browserify-1.1.2.tgz", @@ -38781,6 +39039,60 @@ "@types/node": "*" } }, + "@types/d3-array": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/@types/d3-array/-/d3-array-3.2.1.tgz", + "integrity": "sha512-Y2Jn2idRrLzUfAKV2LyRImR+y4oa2AntrgID95SHJxuMUrkNXmanDSed71sRNZysveJVt1hLLemQZIady0FpEg==" + }, + "@types/d3-color": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/@types/d3-color/-/d3-color-3.1.3.tgz", + "integrity": "sha512-iO90scth9WAbmgv7ogoq57O9YpKmFBbmoEoCHDB2xMBY0+/KVrqAaCDyCE16dUspeOvIxFFRI+0sEtqDqy2b4A==" + }, + "@types/d3-ease": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/d3-ease/-/d3-ease-3.0.2.tgz", + "integrity": "sha512-NcV1JjO5oDzoK26oMzbILE6HW7uVXOHLQvHshBUW4UMdZGfiY6v5BeQwh9a9tCzv+CeefZQHJt5SRgK154RtiA==" + }, + "@types/d3-interpolate": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/d3-interpolate/-/d3-interpolate-3.0.4.tgz", + "integrity": "sha512-mgLPETlrpVV1YRJIglr4Ez47g7Yxjl1lj7YKsiMCb27VJH9W8NVM6Bb9d8kkpG/uAQS5AmbA48q2IAolKKo1MA==", + "requires": { + "@types/d3-color": "*" + } + }, + "@types/d3-path": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@types/d3-path/-/d3-path-3.1.0.tgz", + "integrity": "sha512-P2dlU/q51fkOc/Gfl3Ul9kicV7l+ra934qBFXCFhrZMOL6du1TM0pm1ThYvENukyOn5h9v+yMJ9Fn5JK4QozrQ==" + }, + "@types/d3-scale": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/@types/d3-scale/-/d3-scale-4.0.8.tgz", + "integrity": "sha512-gkK1VVTr5iNiYJ7vWDI+yUFFlszhNMtVeneJ6lUTKPjprsvLLI9/tgEGiXJOnlINJA8FyA88gfnQsHbybVZrYQ==", + "requires": { + "@types/d3-time": "*" + } + }, + "@types/d3-shape": { + "version": "3.1.6", + "resolved": "https://registry.npmjs.org/@types/d3-shape/-/d3-shape-3.1.6.tgz", + "integrity": "sha512-5KKk5aKGu2I+O6SONMYSNflgiP0WfZIQvVUMan50wHsLG1G94JlxEVnCpQARfTtzytuY0p/9PXXZb3I7giofIA==", + "requires": { + "@types/d3-path": "*" + } + }, + "@types/d3-time": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/d3-time/-/d3-time-3.0.3.tgz", + "integrity": "sha512-2p6olUZ4w3s+07q3Tm2dbiMZy5pCDfYwtLXXHUnVzXgQlZ/OyPtUz6OL382BkOuGlLXqfT+wqv8Fw2v8/0geBw==" + }, + "@types/d3-timer": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/d3-timer/-/d3-timer-3.0.2.tgz", + "integrity": "sha512-Ps3T8E8dZDam6fUyNiMkekK3XUsaUEik+idO9/YjPtfj2qruF8tFBXS7XhtE4iIXBLxhmLjP3SXpLhVf21I9Lw==" + }, "@types/doctrine": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/@types/doctrine/-/doctrine-0.0.3.tgz", @@ -42857,6 +43169,83 @@ } } }, + "d3-array": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-3.2.4.tgz", + "integrity": "sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg==", + "requires": { + "internmap": "1 - 2" + } + }, + "d3-color": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-color/-/d3-color-3.1.0.tgz", + "integrity": "sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==" + }, + "d3-ease": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-ease/-/d3-ease-3.0.1.tgz", + "integrity": "sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w==" + }, + "d3-format": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-format/-/d3-format-3.1.0.tgz", + "integrity": "sha512-YyUI6AEuY/Wpt8KWLgZHsIU86atmikuoOmCfommt0LYHiQSPjvX2AcFc38PX0CBpr2RCyZhjex+NS/LPOv6YqA==" + }, + "d3-interpolate": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-interpolate/-/d3-interpolate-3.0.1.tgz", + "integrity": "sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==", + "requires": { + "d3-color": "1 - 3" + } + }, + "d3-path": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-path/-/d3-path-3.1.0.tgz", + "integrity": "sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ==" + }, + "d3-scale": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/d3-scale/-/d3-scale-4.0.2.tgz", + "integrity": "sha512-GZW464g1SH7ag3Y7hXjf8RoUuAFIqklOAq3MRl4OaWabTFJY9PN/E1YklhXLh+OQ3fM9yS2nOkCoS+WLZ6kvxQ==", + "requires": { + "d3-array": "2.10.0 - 3", + "d3-format": "1 - 3", + "d3-interpolate": "1.2.0 - 3", + "d3-time": "2.1.1 - 3", + "d3-time-format": "2 - 4" + } + }, + "d3-shape": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-3.2.0.tgz", + "integrity": "sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA==", + "requires": { + "d3-path": "^3.1.0" + } + }, + "d3-time": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-time/-/d3-time-3.1.0.tgz", + "integrity": "sha512-VqKjzBLejbSMT4IgbmVgDjpkYrNWUYJnbCGo874u7MMKIWsILRX+OpX/gTk8MqjpT1A/c6HY2dCA77ZN0lkQ2Q==", + "requires": { + "d3-array": "2 - 3" + } + }, + "d3-time-format": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/d3-time-format/-/d3-time-format-4.1.0.tgz", + "integrity": "sha512-dJxPBlzC7NugB2PDLwo9Q8JiTR3M3e4/XANkreKSUxF8vvXKqm1Yfq4Q5dl8budlunRVlUUaDUgFt7eA8D6NLg==", + "requires": { + "d3-time": "1 - 3" + } + }, + "d3-timer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-timer/-/d3-timer-3.0.1.tgz", + "integrity": "sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA==" + }, "damerau-levenshtein": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/damerau-levenshtein/-/damerau-levenshtein-1.0.8.tgz", @@ -42939,6 +43328,11 @@ "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.4.3.tgz", "integrity": "sha512-VBBaLc1MgL5XpzgIP7ny5Z6Nx3UrRkIViUkPUdtl9aya5amy3De1gsUUSB1g3+3sExYNjCAsAznmukyxCb1GRA==" }, + "decimal.js-light": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/decimal.js-light/-/decimal.js-light-2.5.1.tgz", + "integrity": "sha512-qIMFpTMZmny+MMIitAB6D7iVPEorVw6YQRWkvarTkT4tBeSLLiHzcwj6q0MmYSFCiVpiqPJTJEYIrpcPzVEIvg==" + }, "decode-uri-component": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.2.tgz", @@ -44608,6 +45002,11 @@ "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" }, + "fast-equals": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/fast-equals/-/fast-equals-5.0.1.tgz", + "integrity": "sha512-WF1Wi8PwwSY7/6Kx0vKXtw8RwuSGoM1bvDaJbu7MxDlR1vovZjIAKrnzyrThgAjm6JDTu0fVgWXDlMGspodfoQ==" + }, "fast-glob": { "version": "3.3.2", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.2.tgz", @@ -46100,6 +46499,11 @@ "side-channel": "^1.0.4" } }, + "internmap": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/internmap/-/internmap-2.0.3.tgz", + "integrity": "sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg==" + }, "interpret": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/interpret/-/interpret-2.2.0.tgz", @@ -52711,6 +53115,16 @@ "use-isomorphic-layout-effect": "^1.1.2" } }, + "react-smooth": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/react-smooth/-/react-smooth-4.0.1.tgz", + "integrity": "sha512-OE4hm7XqR0jNOq3Qmk9mFLyd6p2+j6bvbPJ7qlB7+oo0eNcL2l7WQzG6MBnT3EXY6xzkLMUBec3AfewJdA0J8w==", + "requires": { + "fast-equals": "^5.0.1", + "prop-types": "^15.8.1", + "react-transition-group": "^4.4.5" + } + }, "react-style-singleton": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/react-style-singleton/-/react-style-singleton-2.2.1.tgz", @@ -52855,6 +53269,36 @@ } } }, + "recharts": { + "version": "2.12.7", + "resolved": "https://registry.npmjs.org/recharts/-/recharts-2.12.7.tgz", + "integrity": "sha512-hlLJMhPQfv4/3NBSAyq3gzGg4h2v69RJh6KU7b3pXYNNAELs9kEoXOjbkxdXpALqKBoVmVptGfLpxdaVYqjmXQ==", + "requires": { + "clsx": "^2.0.0", + "eventemitter3": "^4.0.1", + "lodash": "^4.17.21", + "react-is": "^16.10.2", + "react-smooth": "^4.0.0", + "recharts-scale": "^0.4.4", + "tiny-invariant": "^1.3.1", + "victory-vendor": "^36.6.8" + }, + "dependencies": { + "clsx": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz", + "integrity": "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==" + } + } + }, + "recharts-scale": { + "version": "0.4.5", + "resolved": "https://registry.npmjs.org/recharts-scale/-/recharts-scale-0.4.5.tgz", + "integrity": "sha512-kivNFO+0OcUNu7jQquLXAxz1FIwZj8nrj+YkOKc5694NbjCvcT6aSZiIzNzd2Kul4o4rTto8QVR9lMNtxD4G1w==", + "requires": { + "decimal.js-light": "^2.4.1" + } + }, "recoil": { "version": "0.7.7", "resolved": "https://registry.npmjs.org/recoil/-/recoil-0.7.7.tgz", @@ -54957,8 +55401,7 @@ "tiny-invariant": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/tiny-invariant/-/tiny-invariant-1.3.3.tgz", - "integrity": "sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg==", - "dev": true + "integrity": "sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg==" }, "tiny-warning": { "version": "1.0.3", @@ -55747,6 +56190,27 @@ } } }, + "victory-vendor": { + "version": "36.9.2", + "resolved": "https://registry.npmjs.org/victory-vendor/-/victory-vendor-36.9.2.tgz", + "integrity": "sha512-PnpQQMuxlwYdocC8fIJqVXvkeViHYzotI+NJrCuav0ZYFoq912ZHBk3mCeuj+5/VpodOjPe1z0Fk2ihgzlXqjQ==", + "requires": { + "@types/d3-array": "^3.0.3", + "@types/d3-ease": "^3.0.0", + "@types/d3-interpolate": "^3.0.1", + "@types/d3-scale": "^4.0.2", + "@types/d3-shape": "^3.1.0", + "@types/d3-time": "^3.0.0", + "@types/d3-timer": "^3.0.0", + "d3-array": "^3.1.6", + "d3-ease": "^3.0.1", + "d3-interpolate": "^3.0.1", + "d3-scale": "^4.0.2", + "d3-shape": "^3.1.0", + "d3-time": "^3.0.0", + "d3-timer": "^3.0.1" + } + }, "vm-browserify": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vm-browserify/-/vm-browserify-1.1.2.tgz", diff --git a/roadside-forms-frontend/frontend_web_app/package.json b/roadside-forms-frontend/frontend_web_app/package.json index 1d52fa653..360ceeef6 100644 --- a/roadside-forms-frontend/frontend_web_app/package.json +++ b/roadside-forms-frontend/frontend_web_app/package.json @@ -32,6 +32,7 @@ "react-router-dom": "6.19.0", "react-scripts": "5.0.1", "react-toastify": "^9.1.3", + "recharts": "^2.12.7", "recoil": "^0.7.7", "recoil-nexus": "^0.5.0", "use-between": "^1.3.5", diff --git a/roadside-forms-frontend/frontend_web_app/src/api/formStatisticsApi.js b/roadside-forms-frontend/frontend_web_app/src/api/formStatisticsApi.js new file mode 100644 index 000000000..ec2225e1b --- /dev/null +++ b/roadside-forms-frontend/frontend_web_app/src/api/formStatisticsApi.js @@ -0,0 +1,28 @@ +import { api } from "./config/axiosConfig"; +import { createRequestHeader } from "../utils/requestHeaders"; + +export const FormStatisticsApi = { + getStatistics: async function () { + const headers = { + ...await createRequestHeader(), + }; + return await api + .request({ + url: "/api/v1/forms/statistics", // Adjust this to match your API endpoint + method: "GET", + headers: { ...headers }, + }) + .then((response) => { + return { + status: response.status, + data: response.data, + }; + }) + .catch((error) => { + return { + status: error.status, + data: error.response, + }; + }); + }, +}; \ No newline at end of file diff --git a/roadside-forms-frontend/frontend_web_app/src/components/common/Header/Header.js b/roadside-forms-frontend/frontend_web_app/src/components/common/Header/Header.js index 8692b319e..458f121d1 100644 --- a/roadside-forms-frontend/frontend_web_app/src/components/common/Header/Header.js +++ b/roadside-forms-frontend/frontend_web_app/src/components/common/Header/Header.js @@ -2,8 +2,11 @@ import React, { useState, useEffect } from "react"; import { useKeycloak } from "@react-keycloak/web"; import CloudOutlinedIcon from "@mui/icons-material/CloudOutlined"; import CloudOffOutlinedIcon from "@mui/icons-material/CloudOffOutlined"; +import AccountCircle from "@mui/icons-material/AccountCircle"; +import ArrowDropDownIcon from "@mui/icons-material/ArrowDropDown"; import { useSetRecoilState } from "recoil"; import { Link } from "react-router-dom"; +import { Dropdown } from "react-bootstrap"; import { userAtom } from "../../../atoms/users"; import { userRolesAtom } from "../../../atoms/userRoles"; @@ -18,7 +21,6 @@ import { Col, Row } from "react-bootstrap"; export const Header = () => { const { isConnected } = useSharedIsOnline(); - // const [isConnected, setIsConnected] = useState(navigator.onLine); const [isLoading, setIsLoading] = useState(true); const [userInfo, setUserInfo] = useState({ username: null, agency: null }); const [userAdminInfo, setuserAdminInfo] = useState(false); @@ -82,12 +84,6 @@ export const Header = () => { }); } - // const handleOnline = () => setIsConnected(true); - // const handleOffline = () => setIsConnected(false); - - // window.addEventListener("online", handleOnline); - // window.addEventListener("offline", handleOffline); - const interval = setInterval(() => { const { dateString, dayString, timeString } = getCurrentDateTime(); setDate(dateString); @@ -96,7 +92,6 @@ export const Header = () => { setIsLoading(false); }, 1000); - // Function to update last active time const updateLastActive = async () => { if (userId) { @@ -119,13 +114,9 @@ export const Header = () => { } }, 5 * 60 * 1000); // 5 minutes in milliseconds - - return () => { clearInterval(interval); clearInterval(lastActiveInterval); - // window.removeEventListener("online", handleOnline); - // window.removeEventListener("offline", handleOffline); }; }, [ keycloak.authenticated, @@ -170,31 +161,33 @@ export const Header = () => { > )} - -   - - {userInfo.username} - - - {userInfo.agency} - - - -   - {userAdminInfo && ( -
- - Admin - -
- )} - keycloak.logout()} - > - Logout - + + + +
+ +
+
{userInfo.username}
+
{userInfo.agency}
+
+ +
+ + + + {userAdminInfo && ( + <> + + Admin Console + + + Form Inventory + + + )} + keycloak.logout()}>Logout + + @@ -203,8 +196,9 @@ export const Header = () => { ); + }; Header.propTypes = {}; -Header.defaultProps = {}; +Header.defaultProps = {}; \ No newline at end of file diff --git a/roadside-forms-frontend/frontend_web_app/src/components/common/Header/header.scss b/roadside-forms-frontend/frontend_web_app/src/components/common/Header/header.scss index 89184f2b5..031786354 100644 --- a/roadside-forms-frontend/frontend_web_app/src/components/common/Header/header.scss +++ b/roadside-forms-frontend/frontend_web_app/src/components/common/Header/header.scss @@ -44,4 +44,30 @@ $primary-color: #003366; header { display: none; } +} + +// Styles for the user dropdown +.dropdown-toggle::after { + display: none; +} + +.dropdown-menu { + background-color: #f8f9fa; + border: 1px solid #dee2e6; +} + +.dropdown-item { + color: #212529; + + &:hover, &:focus { + background-color: #e9ecef; + } +} + +.user-info { + .dropdown-toggle { + &:hover, &:focus { + opacity: 0.8; + } + } } \ No newline at end of file diff --git a/roadside-forms-frontend/frontend_web_app/src/components/userAdminDashboard/formStatistics.js b/roadside-forms-frontend/frontend_web_app/src/components/userAdminDashboard/formStatistics.js new file mode 100644 index 000000000..716add56c --- /dev/null +++ b/roadside-forms-frontend/frontend_web_app/src/components/userAdminDashboard/formStatistics.js @@ -0,0 +1,88 @@ +import React, { useState, useEffect } from 'react'; +import { Container, Row, Col, Card, Alert } from 'react-bootstrap'; +import { FormStatisticsApi } from '../../api/formStatisticsApi'; +import './formStatistics.scss'; + +export const FormStatistics = () => { + const [formData, setFormData] = useState([]); + const [isLoading, setIsLoading] = useState(true); + const [error, setError] = useState(null); + + useEffect(() => { + fetchFormStatistics(); + }, []); + + const fetchFormStatistics = async () => { + try { + const response = await FormStatisticsApi.getStatistics(); + if (response.status === 200) { + setFormData(response.data); + } else { + throw new Error('Failed to fetch form statistics'); + } + } catch (err) { + setError(err.message); + } finally { + setIsLoading(false); + } + }; + + const getStatusClass = (availableForms) => { + if (availableForms < 50000) return 'danger'; + if (availableForms <= 60000) return 'warning'; + return 'success'; + }; + + if (isLoading) { + return
Loading...
; + } + + if (error) { + return {error}; + } + + return ( + +

Form Inventory Dashboard

+ + {formData.map((form) => ( + + + {form.form_type} Forms + +
+
+ {form.total_forms} + Total +
+
+ {form.leased_forms} + Leased +
+
+ {form.total_used_forms} + Used +
+
+ {form.available_forms} + Available +
+
+ {form.available_forms < 50000 && ( + + Critical: Available forms are below 50,000! + + )} + {form.available_forms >= 50000 && form.available_forms <= 60000 && ( + + Warning: Available forms are between 50,000 and 60,000! + + )} + + + + ))} + + + ); +}; \ No newline at end of file diff --git a/roadside-forms-frontend/frontend_web_app/src/components/userAdminDashboard/formStatistics.scss b/roadside-forms-frontend/frontend_web_app/src/components/userAdminDashboard/formStatistics.scss new file mode 100644 index 000000000..284e095ed --- /dev/null +++ b/roadside-forms-frontend/frontend_web_app/src/components/userAdminDashboard/formStatistics.scss @@ -0,0 +1,79 @@ +$color-available: #28a745; // Green +$color-total: #17a2b8; // Light blue +$color-leased: #ffc107; // Yellow +$color-used: #5a4fcf; // Purple +$color-warning: #ffc107; // Yellow +$color-danger: #dc3545; // Red +$color-text-dark: #212529; + +.circle-container { + display: flex; + justify-content: space-around; + flex-wrap: wrap; + margin-top: 20px; + + &.danger { + .circle.available { + background-color: $color-danger; + animation: pulse 2s infinite; + } + } + + &.warning { + .circle.available { + background-color: $color-warning; + animation: pulse 2s infinite; + } + } + + &.success { + .circle.available { + background-color: $color-available; + } + } +} + +.circle { + width: 120px; + height: 120px; + border-radius: 50%; + display: flex; + flex-direction: column; + justify-content: center; + align-items: center; + margin: 10px; + color: white; + transition: transform 0.3s ease; + + &:hover { + transform: scale(1.1); + } + + .number { + font-size: 24px; + font-weight: bold; + } + + .label { + font-size: 14px; + } + + &.total { background-color: $color-total; } + &.leased { + background-color: $color-leased; + color: $color-text-dark; + } + &.used { background-color: $color-used; } +} + +@keyframes pulse { + 0% { + transform: scale(1); + } + 50% { + transform: scale(1.05); + } + 100% { + transform: scale(1); + } +} \ No newline at end of file diff --git a/roadside-forms-frontend/frontend_web_app/src/routes/appRouter.js b/roadside-forms-frontend/frontend_web_app/src/routes/appRouter.js index 434ef3d05..f39593ee8 100644 --- a/roadside-forms-frontend/frontend_web_app/src/routes/appRouter.js +++ b/roadside-forms-frontend/frontend_web_app/src/routes/appRouter.js @@ -14,6 +14,7 @@ import { UserAdminDashboard } from "../components/userAdminDashboard/userAdminDa import { CreateEvent } from "../components/Event/createEvent"; import { ViewPastEvent } from "../components/ViewPastEvent/viewPastEvent"; import { Layout } from "../components/common/Layout/Layout"; +import { FormStatistics } from "../components/userAdminDashboard/formStatistics"; export const appRouter = createBrowserRouter( createRoutesFromElements( @@ -25,6 +26,7 @@ export const appRouter = createBrowserRouter( } /> } exact /> } exact /> + } exact /> } exact /> } /> From 8ab3c6bd80a3d1634aed965da349722d52b250a7 Mon Sep 17 00:00:00 2001 From: Ratheesh kumar R Date: Wed, 11 Sep 2024 16:50:52 -0700 Subject: [PATCH 02/22] Update all three services to use a common/models.py Removed all other models.py --- .../{prohibition_web_svc => common}/models.py | 0 python/common/ride_actions.py | 2 +- .../ride_actions/test_twelve_hr_event.py | 2 +- .../ride_actions/test_twenty_four_hr_event.py | 2 +- .../tests/ride_actions/test_vi_event.py | 2 +- python/form_handler/actions.py | 2 +- python/form_handler/helper.py | 2 +- python/form_handler/listener.py | 2 +- python/form_handler/models.py | 758 ------------------ python/prohibition_web_svc/app.py | 2 +- .../prohibition_web_svc/blueprints/static.py | 2 +- python/prohibition_web_svc/commands.py | 2 +- .../middleware/event_middleware.py | 2 +- .../middleware/form_middleware.py | 2 +- .../middleware/keycloak_middleware.py | 2 +- .../middleware/role_middleware.py | 2 +- .../middleware/user_middleware.py | 2 +- .../tests/test_admin_forms_api.py | 2 +- .../tests/test_admin_user_roles.py | 2 +- .../tests/test_admin_users.py | 2 +- .../tests/test_forms_api.py | 2 +- .../tests/test_icbc_resources.py | 2 +- .../tests/test_static_resources.py | 2 +- .../tests/test_user_roles.py | 2 +- .../prohibition_web_svc/tests/test_users.py | 2 +- python/task_scheduler/dbfuncs.py | 2 +- python/task_scheduler/main.py | 2 +- python/task_scheduler/models.py | 758 ------------------ 28 files changed, 25 insertions(+), 1541 deletions(-) rename python/{prohibition_web_svc => common}/models.py (100%) delete mode 100644 python/form_handler/models.py delete mode 100644 python/task_scheduler/models.py diff --git a/python/prohibition_web_svc/models.py b/python/common/models.py similarity index 100% rename from python/prohibition_web_svc/models.py rename to python/common/models.py diff --git a/python/common/ride_actions.py b/python/common/ride_actions.py index 14ac0584c..768b413d2 100644 --- a/python/common/ride_actions.py +++ b/python/common/ride_actions.py @@ -5,7 +5,7 @@ from python.common.config import Config import pytz -from python.form_handler.models import Agency, City, ImpoundLotOperator, JurisdictionCrossRef, Province, Vehicle, VehicleColour, VehicleStyle, VehicleType +from python.common.models import Agency, City, ImpoundLotOperator, JurisdictionCrossRef, Province, Vehicle, VehicleColour, VehicleStyle, VehicleType ride_url=Config.RIDE_API_URL ride_key=Config.RIDE_API_KEY diff --git a/python/common/tests/ride_actions/test_twelve_hr_event.py b/python/common/tests/ride_actions/test_twelve_hr_event.py index 1fa09cf48..1e4c4336e 100644 --- a/python/common/tests/ride_actions/test_twelve_hr_event.py +++ b/python/common/tests/ride_actions/test_twelve_hr_event.py @@ -5,7 +5,7 @@ from flask_api import FlaskAPI import python.common.ride_actions as ride_actions -from python.form_handler.models import db +from python.common.models import db from python.form_handler.config import Config from python.common.config import Config as CommonConfig diff --git a/python/common/tests/ride_actions/test_twenty_four_hr_event.py b/python/common/tests/ride_actions/test_twenty_four_hr_event.py index 510914cde..cee02a811 100644 --- a/python/common/tests/ride_actions/test_twenty_four_hr_event.py +++ b/python/common/tests/ride_actions/test_twenty_four_hr_event.py @@ -5,7 +5,7 @@ from flask_api import FlaskAPI import python.common.ride_actions as ride_actions -from python.form_handler.models import db +from python.common.models import db from python.form_handler.config import Config from python.common.config import Config as CommonConfig diff --git a/python/common/tests/ride_actions/test_vi_event.py b/python/common/tests/ride_actions/test_vi_event.py index 467509d5c..1737da49d 100644 --- a/python/common/tests/ride_actions/test_vi_event.py +++ b/python/common/tests/ride_actions/test_vi_event.py @@ -5,7 +5,7 @@ from flask_api import FlaskAPI import python.common.ride_actions as ride_actions -from python.form_handler.models import db +from python.common.models import db from python.form_handler.config import Config from python.common.config import Config as CommonConfig diff --git a/python/form_handler/actions.py b/python/form_handler/actions.py index cb89ca28f..40147d966 100644 --- a/python/form_handler/actions.py +++ b/python/form_handler/actions.py @@ -12,7 +12,7 @@ from datetime import datetime from minio import Minio from minio.error import S3Error -from python.form_handler.models import Event,FormStorageRefs,VIForm,TwentyFourHourForm,TwelveHourForm,IRPForm,User,AgencyCrossref,CityCrossRef,JurisdictionCrossRef,ImpoundReasonCodes,IloIdCrossRef,ImpoundLotOperator +from python.common.models import Event,FormStorageRefs,VIForm,TwentyFourHourForm,TwelveHourForm,IRPForm,User,AgencyCrossref,CityCrossRef,JurisdictionCrossRef,ImpoundReasonCodes,IloIdCrossRef,ImpoundLotOperator from python.form_handler.icbc_service import submit_to_icbc from python.form_handler.vips_service import create_vips_doc,create_vips_imp from python.form_handler.payloads import vips_payload,vips_document_payload diff --git a/python/form_handler/helper.py b/python/form_handler/helper.py index db3163726..a80b5cf6d 100644 --- a/python/form_handler/helper.py +++ b/python/form_handler/helper.py @@ -10,7 +10,7 @@ import logging import json from datetime import datetime -from python.form_handler.models import db, Event,FormStorageRefs +from python.common.models import db, Event,FormStorageRefs import pyaes, pbkdf2, binascii, os, secrets import base64 import fitz diff --git a/python/form_handler/listener.py b/python/form_handler/listener.py index 6ab11afe9..7ed60d2d2 100644 --- a/python/form_handler/listener.py +++ b/python/form_handler/listener.py @@ -12,7 +12,7 @@ from python.form_handler.helper import get_storage_ref_event_type,get_event_status from flask_api import FlaskAPI -from python.form_handler.models import db +from python.common.models import db logging.config.dictConfig(Config.LOGGING) diff --git a/python/form_handler/models.py b/python/form_handler/models.py deleted file mode 100644 index 78e8f7aa8..000000000 --- a/python/form_handler/models.py +++ /dev/null @@ -1,758 +0,0 @@ -from datetime import datetime, timedelta -from dataclasses import dataclass -from flask_sqlalchemy import SQLAlchemy -from flask_migrate import Migrate -import logging - -db = SQLAlchemy() -migrate = Migrate() - -@dataclass -class Form(db.Model): - id: str - form_type: str - lease_expiry: datetime - printed_timestamp: datetime - spoiled_timestamp: datetime - user_guid: str - - id = db.Column('id', db.String(20), primary_key=True) - form_type = db.Column(db.String(20), nullable=False) - lease_expiry = db.Column(db.Date, nullable=True) - printed_timestamp = db.Column(db.DateTime, nullable=True) - spoiled_timestamp = db.Column(db.DateTime, nullable=True) - user_guid = db.Column(db.String(80), db.ForeignKey( - 'user.user_guid'), nullable=True) - - # Note: The printed timestamp prior to v0.4.17 was saved in local Pacific time instead of GMT - - def __init__(self, form_id, form_type, printed=None, spoiled=None, lease_expiry=None, user_guid=None): - self.id = form_id - self.form_type = form_type - self.printed_timestamp = printed - self.spoiled_timestamp = spoiled - self.lease_expiry = lease_expiry - self.user_guid = user_guid - - def lease(self, user_guid): - today = datetime.now() - lease_expiry = today + timedelta(days=30) - self.lease_expiry = lease_expiry - self.user_guid = user_guid - logging.info("{} leased {} until {}".format( - self.user_guid, self.id, self.lease_expiry.strftime("%Y-%m-%d"))) - - @staticmethod - def _format_lease_expiry(lease_expiry): - if lease_expiry is None: - return '' - else: - return datetime.strftime(lease_expiry, "%Y-%m-%d") - - @staticmethod - def collection_to_dict(all_rows): - result_list = [] - for row in all_rows: - result_list.append(Form.serialize(row)) - return result_list - - -class User(db.Model): - user_guid = db.Column(db.String(120), primary_key=True) - business_guid = db.Column(db.String(120), nullable=True) - username = db.Column(db.String(80), nullable=False) - agency = db.Column(db.String(120), nullable=False) - badge_number = db.Column(db.String(12), nullable=False) - last_name = db.Column(db.String(40), nullable=False) - first_name = db.Column(db.String(40), nullable=True) - display_name = db.Column(db.String(80), nullable=True) - login = db.Column(db.String(80), nullable=False) - - def __init__(self, username, user_guid, agency, badge_number, last_name, login, business_guid='', display_name='', first_name=''): - self.username = username - self.user_guid = user_guid - self.agency = agency - self.badge_number = badge_number - self.last_name = last_name - self.first_name = first_name - self.business_guid = business_guid - self.display_name = display_name - self.login = login - - @staticmethod - def serialize(user): - return { - "username": user.username, - "user_guid": user.user_guid, - "agency": user.agency, - "badge_number": user.badge_number, - "first_name": user.first_name, - "last_name": user.last_name, - "display_name": user.display_name, - "login": user.login - } - - -class UserRole(db.Model): - role_name = db.Column(db.String(20), primary_key=True) - user_guid = db.Column(db.String(80), db.ForeignKey( - 'user.user_guid'), primary_key=True) - submitted_dt = db.Column(db.DateTime, nullable=True) - approved_dt = db.Column(db.DateTime, nullable=True) - - def __init__(self, role_name, user_guid, submitted_dt=None, approved_dt=None): - self.role_name = role_name - self.user_guid = user_guid - self.submitted_dt = submitted_dt - self.approved_dt = approved_dt - - @staticmethod - def serialize(role): - return { - "role_name": role.role_name, - "user_guid": role.user_guid, - "submitted_dt": role.submitted_dt, - "approved_dt": role.approved_dt - } - - @staticmethod - def serialize_all_users(rows): - return { - "agency": rows.agency, - "approved_dt": rows.approved_dt, - "badge_number": rows.badge_number, - "first_name": rows.first_name, - "last_name": rows.last_name, - "role_name": rows.role_name, - "submitted_dt": rows.submitted_dt, - "user_guid": rows.user_guid, - "username": rows.username, - "login": rows.login, - } - - @staticmethod - def collection_to_dict(all_rows, serialization_method: str): - result_list = [] - for row in all_rows: - method = getattr(UserRole, serialization_method) - result_list.append(method(row)) - return result_list - - @staticmethod - def collection_to_list_roles(all_rows): - result_list = [] - for row in all_rows: - result_list.append(row.role_name) - return result_list - - @staticmethod - def get_roles(user_guid): - rows = db.session.query(UserRole) \ - .filter(UserRole.user_guid == user_guid) \ - .filter(UserRole.approved_dt != None) \ - .all() - return UserRole.collection_to_list_roles(rows) - - -@dataclass -class Agency(db.Model): - __tablename__ = 'agency' - - id: int - vjur: str - agency_name: str - - id = db.Column(db.Integer, primary_key=True) - vjur = db.Column(db.String) - agency_name = db.Column(db.String) - - -@dataclass -class City(db.Model): - __tablename__ = 'city' - - id: int - objectCd: str - objectDsc: str - - id = db.Column(db.Integer, primary_key=True) - objectCd = db.Column(db.String) - objectDsc = db.Column(db.String) - - -@dataclass -class Country(db.Model): - __tablename__ = 'country' - - id: int - objectCd: str - objectDsc: str - - id = db.Column(db.Integer, primary_key=True) - objectCd = db.Column(db.String) - objectDsc = db.Column(db.String) - - -@dataclass -class ImpoundLotOperator(db.Model): - __tablename__ = 'impound_lot_operator' - - id: int - name: str - lot_address: str - city: str - phone: str - - id = db.Column(db.Integer, primary_key=True) - name = db.Column(db.String) - lot_address = db.Column(db.String) - city = db.Column(db.String) - phone = db.Column(db.String) - - -@dataclass -class Jurisdiction(db.Model): - __tablename__ = 'jurisdiction' - - id: int - objectCd: str - objectDsc: str - - id = db.Column(db.Integer, primary_key=True) - objectCd = db.Column(db.String) - objectDsc = db.Column(db.String) - - -@dataclass -class Permission(db.Model): - __tablename__ = 'permission' - - id: int - role: str - permission: str - - id = db.Column(db.Integer, primary_key=True) - role = db.Column(db.String) - permission = db.Column(db.String) - - -@dataclass -class Province(db.Model): - __tablename__ = 'province' - - id: int - objectCd: str - objectDsc: str - - id = db.Column(db.Integer, primary_key=True) - objectCd = db.Column(db.String) - objectDsc = db.Column(db.String) - - -@dataclass -class VehicleStyle(db.Model): - __tablename__ = 'vehicle_style' - - code: str - name: str - - code = db.Column(db.String, primary_key=True) - name = db.Column(db.String) - -@dataclass -class VehicleType(db.Model): - __tablename__ = 'vehicle_type' - - type_cd: int - description: str - - type_cd = db.Column(db.Integer, primary_key=True) - description = db.Column(db.String) - - -@dataclass -class Vehicle(db.Model): - __tablename__ = 'vehicle' - - id: int - mk: str - search: str - md: str - - id = db.Column(db.Integer, primary_key=True) - mk = db.Column(db.String) - search = db.Column(db.String) - md = db.Column(db.String) - - -@dataclass -class VehicleColour(db.Model): - __tablename__ = 'vehicle_colour' - - code: str - display_name: str - colour_class: str - - code = db.Column(db.String, primary_key=True) - display_name = db.Column(db.String) - colour_class = db.Column(db.String) - - -@dataclass -class Event(db.Model): - __tablename__ = 'event' - - event_id: int - icbc_sent_status: str - vi_sent_status: str - icbc_retry_count: int - vi_retry_count: int - driver_licence_no: str - driver_jurisdiction: str - driver_last_name: str - driver_given_name: str - driver_dob: datetime - driver_address: str - driver_city: str - driver_prov: str - driver_postal: str - driver_phone: str - date_of_driving: datetime - time_of_driving: str - vehicle_jurisdiction: str - vehicle_plate_no: str - vehicle_registration_no: str - vehicle_year: str - vehicle_mk_md: str - vehicle_style: str - vehicle_type: int - vehicle_colour: str - vehicle_vin_no: str - nsc_prov_state: str - location_of_keys: str - nsc_no: str - type_of_prohibition: str - owned_by_corp: bool - vehicle_released_to: str - date_released: datetime - time_released: str - intersection_or_address_of_offence: str - offence_city: str - corporation_name: str - regist_owner_last_name: str - regist_owner_first_name: str - regist_owner_address: str - regist_owner_dob: datetime - regist_owner_city: str - regist_owner_prov: str - regist_owner_postal: str - regist_owner_phone: str - regist_owner_email: str - agency_file_no: str - submitted: bool - confirmation_of_service: bool - confirmation_of_service_date: datetime - impound_lot_operator: int - task_processing_status: str - created_dt: datetime - updated_dt: datetime - created_by: str - updated_by: str - - event_id = db.Column(db.Integer, primary_key=True) - icbc_sent_status = db.Column(db.String) - vi_sent_status = db.Column(db.String) - icbc_retry_count = db.Column(db.Integer) - vi_retry_count = db.Column(db.Integer) - type_of_prohibition = db.Column(db.String) - driver_licence_no = db.Column(db.String) - driver_jurisdiction = db.Column(db.String) - driver_last_name = db.Column(db.String) - driver_given_name = db.Column(db.String) - driver_dob = db.Column(db.DateTime) - driver_address = db.Column(db.String) - driver_city = db.Column(db.String) - driver_prov = db.Column(db.String) - driver_postal = db.Column(db.String) - driver_phone = db.Column(db.String) - agency_file_no = db.Column(db.String) - date_of_driving = db.Column(db.DateTime) - time_of_driving = db.Column(db.String) - vehicle_jurisdiction = db.Column(db.String) - vehicle_plate_no = db.Column(db.String) - vehicle_registration_no = db.Column(db.String) - vehicle_year = db.Column(db.String) - vehicle_mk_md = db.Column(db.String) - vehicle_style = db.Column(db.String) - vehicle_type = db.Column(db.Integer, db.ForeignKey('vehicle_type.type_cd')) - vehicle_colour = db.Column(db.String) - vehicle_vin_no = db.Column(db.String) - nsc_prov_state = db.Column(db.String) - location_of_keys = db.Column(db.String) - nsc_no = db.Column(db.String) - submitted = db.Column(db.Boolean) - owned_by_corp = db.Column(db.Boolean) - vehicle_released_to = db.Column(db.String) - date_released = db.Column(db.DateTime) - time_released = db.Column(db.String) - intersection_or_address_of_offence = db.Column(db.String) - offence_city = db.Column(db.String) - corporation_name = db.Column(db.String) - regist_owner_last_name = db.Column(db.String) - regist_owner_first_name = db.Column(db.String) - regist_owner_address = db.Column(db.String) - regist_owner_dob = db.Column(db.DateTime) - regist_owner_city = db.Column(db.String) - regist_owner_prov = db.Column(db.String) - regist_owner_postal = db.Column(db.String) - regist_owner_phone = db.Column(db.String) - regist_owner_email = db.Column(db.String) - impound_lot_operator = db.Column( - db.Integer, db.ForeignKey('impound_lot_operator.id')) - confirmation_of_service = db.Column(db.Boolean) - confirmation_of_service_date = db.Column(db.DateTime) - task_processing_status = db.Column(db.String) - created_by = db.Column(db.String, db.ForeignKey('user.user_guid')) - updated_by = db.Column(db.String) - created_dt = db.Column(db.DateTime) - updated_dt = db.Column(db.DateTime) - - twenty_four_hour_form = db.relationship( - 'TwentyFourHourForm', - backref='event', - lazy='joined', - uselist=False) - - twelve_hour_form = db.relationship( - 'TwelveHourForm', - backref='event', - lazy='joined', - uselist=False) - - vi_form = db.relationship( - 'VIForm', - backref='event', - lazy='joined', - uselist=False) - - irp_form = db.relationship( - 'IRPForm', - backref='event', - lazy='joined', - uselist=False) - - -@dataclass -class TwentyFourHourForm(db.Model): - __tablename__ = 'twenty_four_hour_form' - - form_id: int - event_id: int - vehicle_impounded: str - reason_for_not_impounding: str - reasonable_ground_other_reason: str - prescribed_test_used: str - reasonable_date_of_test: datetime - reasonable_time_of_test: str - reason_for_not_using_prescribed_test: str - resonable_test_used_alcohol: str - reasonable_asd_expiry_date: datetime - reasonable_result_alcohol: str - reasonable_bac_result_mg: str - resonable_approved_instrument_used: str - reasonable_test_used_drugs: str - reasonable_can_drive_drug: bool - reasonable_can_drive_alcohol: bool - requested_can_drive_alcohol: bool - requested_can_drive_drug: bool - requested_approved_instrument_used: str - requested_BAC_result: str - requested_alcohol_test_result: str - requested_ASD_expiry_date: datetime - time_of_requested_test: str - requested_test_used_alcohol: str - requested_test_used_drug: str - requested_prescribed_test: str - witnessed_by_officer: bool - admission_by_driver: bool - independent_witness: bool - reasonable_ground_other: bool - twenty_four_hour_number: str - created_dt: datetime - updated_dt: datetime - - form_id = db.Column(db.Integer, primary_key=True) - event_id = db.Column(db.Integer, db.ForeignKey('event.event_id')) - vehicle_impounded = db.Column(db.String) - reason_for_not_impounding = db.Column(db.String) - reasonable_ground_other_reason = db.Column(db.String) - prescribed_test_used = db.Column(db.String) - reasonable_date_of_test = db.Column(db.DateTime) - reasonable_time_of_test = db.Column(db.String) - reason_for_not_using_prescribed_test = db.Column(db.String) - resonable_test_used_alcohol = db.Column(db.String) - reasonable_asd_expiry_date = db.Column(db.DateTime) - reasonable_result_alcohol = db.Column(db.String) - reasonable_bac_result_mg = db.Column(db.String) - resonable_approved_instrument_used = db.Column(db.String) - reasonable_test_used_drugs = db.Column(db.String) - reasonable_can_drive_drug = db.Column(db.Boolean) - reasonable_can_drive_alcohol = db.Column(db.Boolean) - requested_can_drive_alcohol = db.Column(db.Boolean) - requested_can_drive_drug = db.Column(db.Boolean) - requested_approved_instrument_used = db.Column(db.String) - requested_BAC_result = db.Column(db.String) - requested_alcohol_test_result = db.Column(db.String) - requested_ASD_expiry_date = db.Column(db.DateTime) - time_of_requested_test = db.Column(db.String) - requested_test_used_alcohol = db.Column(db.String) - requested_test_used_drug = db.Column(db.String) - requested_prescribed_test = db.Column(db.String) - witnessed_by_officer = db.Column(db.Boolean) - admission_by_driver = db.Column(db.Boolean) - independent_witness = db.Column(db.Boolean) - reasonable_ground_other = db.Column(db.Boolean) - twenty_four_hour_number = db.Column(db.String) - created_dt = db.Column(db.DateTime) - updated_dt = db.Column(db.DateTime) - - -@dataclass -class TwelveHourForm(db.Model): - __tablename__ = 'twelve_hour_form' - - form_id: int - event_id: int - created_dt: datetime - updated_dt: datetime - driver_phone: str - twelve_hour_number: str - - form_id = db.Column(db.Integer, primary_key=True) - event_id = db.Column(db.Integer, db.ForeignKey('event.event_id')) - created_dt = db.Column(db.DateTime) - updated_dt = db.Column(db.DateTime) - driver_phone = db.Column(db.String) - twelve_hour_number = db.Column(db.String) - - -@dataclass -class IRPForm(db.Model): - __tablename__ = 'irp_form' - - form_id: int - event_id: int - created_dt: datetime - updated_dt: datetime - - form_id = db.Column(db.Integer, primary_key=True) - event_id = db.Column(db.Integer, db.ForeignKey('event.event_id')) - created_dt = db.Column(db.DateTime) - updated_dt = db.Column(db.DateTime) - - -@dataclass -class VIForm(db.Model): - __tablename__ = 'vi_form' - - form_id: int - event_id: int - created_dt: datetime - updated_dt: datetime - gender: str - driver_is_regist_owner: bool - driver_licence_expiry: datetime - driver_licence_class: str - unlicenced_prohibition_number: str - belief_driver_bc_resident: str - out_of_province_dl: str - out_of_province_dl_number: str - out_of_province_dl_expiry: str - out_of_province_dl_jurisdiction: str - date_of_impound: datetime - irp_impound: str - irp_impound_duration: str - IRP_number: str - VI_number: str - excessive_speed: str - prohibited: bool - suspended: bool - street_racing: bool - stunt_driving: bool - motorcycle_seating: bool - motorcycle_restrictions: bool - unlicensed: bool - linkage_location_of_keys: bool - linkage_location_of_keys_explanation: str - linkage_driver_principal: bool - linkage_owner_in_vehicle: bool - linkage_owner_aware_possesion: bool - linkage_vehicle_transfer_notice: bool - linkage_other: bool - speed_limit: str - vehicle_speed: str - speed_estimation_technique: str - speed_confirmation_technique: str - incident_details: str - incident_details_extra_page: bool - - form_id = db.Column(db.Integer, primary_key=True) - event_id = db.Column(db.Integer, db.ForeignKey('event.event_id')) - gender = db.Column(db.String) - driver_is_regist_owner = db.Column(db.String) - driver_licence_expiry = db.Column(db.DateTime) - driver_licence_class = db.Column(db.String) - unlicenced_prohibition_number = db.Column(db.String) - belief_driver_bc_resident = db.Column(db.String) - out_of_province_dl = db.Column(db.String) - out_of_province_dl_number = db.Column(db.String) - out_of_province_dl_expiry = db.Column(db.String) - out_of_province_dl_jurisdiction = db.Column(db.String) - date_of_impound = db.Column(db.DateTime) - irp_impound = db.Column(db.String) - irp_impound_duration = db.Column(db.String) - IRP_number = db.Column(db.String) - VI_number = db.Column(db.String) - excessive_speed = db.Column(db.Boolean) - prohibited = db.Column(db.Boolean) - suspended = db.Column(db.Boolean) - street_racing = db.Column(db.Boolean) - stunt_driving = db.Column(db.Boolean) - motorcycle_seating = db.Column(db.Boolean) - motorcycle_restrictions = db.Column(db.Boolean) - unlicensed = db.Column(db.Boolean) - linkage_location_of_keys = db.Column(db.Boolean) - linkage_location_of_keys_explanation = db.Column(db.String) - linkage_driver_principal = db.Column(db.Boolean) - linkage_owner_in_vehicle = db.Column(db.Boolean) - linkage_owner_aware_possesion = db.Column(db.Boolean) - linkage_vehicle_transfer_notice = db.Column(db.Boolean) - linkage_other = db.Column(db.Boolean) - speed_limit = db.Column(db.String) - vehicle_speed = db.Column(db.String) - speed_estimation_technique = db.Column(db.String) - speed_confirmation_technique = db.Column(db.String) - incident_details = db.Column(db.String) - incident_details_extra_page = db.Column(db.Boolean) - created_dt = db.Column(db.DateTime) - updated_dt = db.Column(db.DateTime) - - -@dataclass -class FormStorageRefs(db.Model): - __tablename__ = 'form_storage_refs' - - form_id_24h: int - form_id_irp: int - form_id_vi: int - form_id_12h: int - event_id: int - form_type: str - # vi, irp, 24h, 12h - storage_key: str - encryptiv: str - created_dt: datetime - updated_dt: datetime - - storage_key = db.Column(db.String, primary_key=True) - form_id_24h = db.Column(db.Integer, db.ForeignKey( - 'twenty_four_hour_form.form_id')) - form_id_irp = db.Column(db.Integer, db.ForeignKey('irp_form.form_id')) - form_id_vi = db.Column(db.Integer, db.ForeignKey('vi_form.form_id')) - form_id_12h = db.Column( - db.Integer, db.ForeignKey('twelve_hour_form.form_id')) - event_id = db.Column(db.Integer, db.ForeignKey('event.event_id')) - form_type = db.Column(db.String) - encryptiv = db.Column(db.String) - created_dt = db.Column(db.DateTime) - updated_dt = db.Column(db.DateTime) - - - -@dataclass -class AgencyCrossref(db.Model): - __tablename__ = 'agency_cross_refs' - - agency_name: str - agency_id: str - agency_city: str - prime_vjur: str - icbc_detachment_name: str - icbc_city_name: str - vips_policedetachments_agency_id: str - vips_policedetachments_agency_nm: str - - agency_name = db.Column(db.String, primary_key=True) - agency_id = db.Column(db.String) - agency_city = db.Column(db.String) - prime_vjur = db.Column(db.String) - icbc_detachment_name = db.Column(db.String) - icbc_city_name = db.Column(db.String) - vips_policedetachments_agency_id = db.Column(db.String) - vips_policedetachments_agency_nm = db.Column(db.String) - -@dataclass -class JurisdictionCrossRef(db.Model): - __tablename__ = 'jurisdiction_cross_ref' - - jurisdiction_name: str - jurisdiction_code: str - prime_jurisdiction_code: str - icbc_jurisdiction_code: str - icbc_jurisdiction: str - vips_jurisdictions_objectCd: str - vips_jurisdictions_objectDsc: str - - jurisdiction_name = db.Column(db.String, primary_key=True) - jurisdiction_code = db.Column(db.String) - prime_jurisdiction_code = db.Column(db.String) - icbc_jurisdiction_code = db.Column(db.String) - icbc_jurisdiction = db.Column(db.String) - vips_jurisdictions_objectCd = db.Column(db.String) - vips_jurisdictions_objectDsc = db.Column(db.String) - -@dataclass -class CityCrossRef(db.Model): - __tablename__ = 'city_cross_ref' - - city_code: str - city_name: str - icbc_city_code: str - icbc_city_name: str - icbc_city_name_legacy: str - vips_city_name: str - - city_code = db.Column(db.String, primary_key=True) - city_name = db.Column(db.String) - icbc_city_code = db.Column(db.String) - icbc_city_name = db.Column(db.String) - icbc_city_name_legacy = db.Column(db.String) - vips_city_name = db.Column(db.String) - -@dataclass -class ImpoundReasonCodes(db.Model): - __tablename__ = 'impound_reason_codes' - - df_unique_code: str - impound_reason_name: str - vips_value_cd: str - vips_value_dsc: str - vips_value_abbreviated_dsc: str - - df_unique_code = db.Column(db.String, primary_key=True) - impound_reason_name = db.Column(db.String) - vips_value_cd = db.Column(db.String) - vips_value_dsc = db.Column(db.String) - vips_value_abbreviated_dsc = db.Column(db.String) - -@dataclass -class IloIdCrossRef(db.Model): - __tablename__ = 'ilo_cross_ref' - - ilo_name: str - vips_impound_lot_operator_id: int - - ilo_name = db.Column(db.String, primary_key=True) - vips_impound_lot_operator_id = db.Column(db.Integer) - - diff --git a/python/prohibition_web_svc/app.py b/python/prohibition_web_svc/app.py index 57e29aaa0..e4b6c7da4 100644 --- a/python/prohibition_web_svc/app.py +++ b/python/prohibition_web_svc/app.py @@ -2,7 +2,7 @@ import logging import pytz from datetime import datetime -from python.prohibition_web_svc.models import db, migrate, Form, UserRole, User +from python.common.models import db, migrate, Form, UserRole, User from python.prohibition_web_svc.config import Config from python.prohibition_web_svc.commands import register_commands from python.prohibition_web_svc.blueprints import static, forms, admin_forms, icbc, user_roles, admin_user_roles, admin_users, users, events diff --git a/python/prohibition_web_svc/blueprints/static.py b/python/prohibition_web_svc/blueprints/static.py index 8423c2aa8..b8aaea280 100644 --- a/python/prohibition_web_svc/blueprints/static.py +++ b/python/prohibition_web_svc/blueprints/static.py @@ -6,7 +6,7 @@ from flask import request, make_response, Blueprint from python.common.splunk import log_to_splunk from flask_cors import CORS -from python.prohibition_web_svc.models import db, Agency, City, Country, ImpoundLotOperator, Jurisdiction, Permission, Province, Vehicle, VehicleStyle, VehicleType, VehicleColour, NSCPuj, JurisdictionCountry +from python.common.models import db, Agency, City, Country, ImpoundLotOperator, Jurisdiction, Permission, Province, Vehicle, VehicleStyle, VehicleType, VehicleColour, NSCPuj, JurisdictionCountry import logging.config from flask import jsonify diff --git a/python/prohibition_web_svc/commands.py b/python/prohibition_web_svc/commands.py index e842ae306..030698563 100644 --- a/python/prohibition_web_svc/commands.py +++ b/python/prohibition_web_svc/commands.py @@ -1,6 +1,6 @@ import logging from python.prohibition_web_svc.config import Config -from python.prohibition_web_svc.models import db, migrate, Form, UserRole, User +from python.common.models import db, migrate, Form, UserRole, User def register_commands(app): @app.cli.command() diff --git a/python/prohibition_web_svc/middleware/event_middleware.py b/python/prohibition_web_svc/middleware/event_middleware.py index c1d3c8e1c..064592e32 100644 --- a/python/prohibition_web_svc/middleware/event_middleware.py +++ b/python/prohibition_web_svc/middleware/event_middleware.py @@ -11,7 +11,7 @@ from cerberus import Validator from base64 import b64decode from flask import jsonify, make_response -from python.prohibition_web_svc.models import db, Event, TwelveHourForm, TwentyFourHourForm, VIForm, IRPForm, FormStorageRefs +from python.common.models import db, Event, TwelveHourForm, TwentyFourHourForm, VIForm, IRPForm, FormStorageRefs from python.prohibition_web_svc.config import Config from python.prohibition_web_svc.business.cryptography_logic import encryptPdf_method1 import img2pdf diff --git a/python/prohibition_web_svc/middleware/form_middleware.py b/python/prohibition_web_svc/middleware/form_middleware.py index 736329333..251c25196 100644 --- a/python/prohibition_web_svc/middleware/form_middleware.py +++ b/python/prohibition_web_svc/middleware/form_middleware.py @@ -6,7 +6,7 @@ from cerberus import Validator from dataclasses import asdict from flask import jsonify, make_response -from python.prohibition_web_svc.models import db, Form +from python.common.models import db, Form from python.prohibition_web_svc.config import Config diff --git a/python/prohibition_web_svc/middleware/keycloak_middleware.py b/python/prohibition_web_svc/middleware/keycloak_middleware.py index 038f9e5e2..9ba2e8c2b 100644 --- a/python/prohibition_web_svc/middleware/keycloak_middleware.py +++ b/python/prohibition_web_svc/middleware/keycloak_middleware.py @@ -2,7 +2,7 @@ import jwt import json from python.common.helper import load_permissions_into_dict -from python.prohibition_web_svc.models import db, UserRole, Permission +from python.common.models import db, UserRole, Permission from python.prohibition_web_svc.config import Config diff --git a/python/prohibition_web_svc/middleware/role_middleware.py b/python/prohibition_web_svc/middleware/role_middleware.py index 38e19424f..0f6551949 100644 --- a/python/prohibition_web_svc/middleware/role_middleware.py +++ b/python/prohibition_web_svc/middleware/role_middleware.py @@ -3,7 +3,7 @@ from datetime import datetime import logging from python.prohibition_web_svc.config import Config -from python.prohibition_web_svc.models import db, UserRole, User +from python.common.models import db, UserRole, User def query_current_users_roles(**kwargs) -> tuple: diff --git a/python/prohibition_web_svc/middleware/user_middleware.py b/python/prohibition_web_svc/middleware/user_middleware.py index 8b633d704..fa9b817d4 100644 --- a/python/prohibition_web_svc/middleware/user_middleware.py +++ b/python/prohibition_web_svc/middleware/user_middleware.py @@ -5,7 +5,7 @@ import json from datetime import datetime import pytz -from python.prohibition_web_svc.models import db, User, UserRole +from python.common.models import db, User, UserRole import python.common.rsi_email as rsi_email from python.prohibition_web_svc.config import Config diff --git a/python/prohibition_web_svc/tests/test_admin_forms_api.py b/python/prohibition_web_svc/tests/test_admin_forms_api.py index ac85de3dd..c30e7a022 100644 --- a/python/prohibition_web_svc/tests/test_admin_forms_api.py +++ b/python/prohibition_web_svc/tests/test_admin_forms_api.py @@ -4,7 +4,7 @@ import base64 import logging from datetime import datetime, timedelta -from python.prohibition_web_svc.models import Form +from python.common.models import Form from python.prohibition_web_svc.app import db, create_app from python.prohibition_web_svc.config import Config diff --git a/python/prohibition_web_svc/tests/test_admin_user_roles.py b/python/prohibition_web_svc/tests/test_admin_user_roles.py index 5f38a1b17..b1dbc6d32 100644 --- a/python/prohibition_web_svc/tests/test_admin_user_roles.py +++ b/python/prohibition_web_svc/tests/test_admin_user_roles.py @@ -3,7 +3,7 @@ import responses from datetime import datetime import python.prohibition_web_svc.middleware.keycloak_middleware as middleware -from python.prohibition_web_svc.models import db, UserRole +from python.common.models import db, UserRole from python.prohibition_web_svc.app import create_app import logging import json diff --git a/python/prohibition_web_svc/tests/test_admin_users.py b/python/prohibition_web_svc/tests/test_admin_users.py index 52e922ac9..52efa4b2c 100644 --- a/python/prohibition_web_svc/tests/test_admin_users.py +++ b/python/prohibition_web_svc/tests/test_admin_users.py @@ -4,7 +4,7 @@ from python.prohibition_web_svc.config import Config from datetime import datetime import python.prohibition_web_svc.middleware.keycloak_middleware as middleware -from python.prohibition_web_svc.models import db, UserRole, User +from python.common.models import db, UserRole, User from python.prohibition_web_svc.app import create_app import logging import json diff --git a/python/prohibition_web_svc/tests/test_forms_api.py b/python/prohibition_web_svc/tests/test_forms_api.py index 08dbd5511..13c5f1306 100644 --- a/python/prohibition_web_svc/tests/test_forms_api.py +++ b/python/prohibition_web_svc/tests/test_forms_api.py @@ -5,7 +5,7 @@ import responses import python.prohibition_web_svc.middleware.keycloak_middleware as middleware from datetime import datetime, timedelta -from python.prohibition_web_svc.models import Form, UserRole +from python.common.models import Form, UserRole from python.prohibition_web_svc.app import db, create_app from python.prohibition_web_svc.config import Config diff --git a/python/prohibition_web_svc/tests/test_icbc_resources.py b/python/prohibition_web_svc/tests/test_icbc_resources.py index eb3eda831..0ed69539d 100644 --- a/python/prohibition_web_svc/tests/test_icbc_resources.py +++ b/python/prohibition_web_svc/tests/test_icbc_resources.py @@ -3,7 +3,7 @@ import urllib from datetime import datetime import python.prohibition_web_svc.middleware.keycloak_middleware as middleware -from python.prohibition_web_svc.models import db, UserRole +from python.common.models import db, UserRole from python.prohibition_web_svc.app import create_app from python.prohibition_web_svc.config import Config import logging diff --git a/python/prohibition_web_svc/tests/test_static_resources.py b/python/prohibition_web_svc/tests/test_static_resources.py index c64ed859c..e177a8d84 100644 --- a/python/prohibition_web_svc/tests/test_static_resources.py +++ b/python/prohibition_web_svc/tests/test_static_resources.py @@ -4,7 +4,7 @@ import json from datetime import datetime import python.prohibition_web_svc.middleware.keycloak_middleware as middleware -from python.prohibition_web_svc.models import db, UserRole +from python.common.models import db, UserRole from python.prohibition_web_svc.app import create_app from python.prohibition_web_svc.config import Config diff --git a/python/prohibition_web_svc/tests/test_user_roles.py b/python/prohibition_web_svc/tests/test_user_roles.py index d67eac35e..f7b86ce15 100644 --- a/python/prohibition_web_svc/tests/test_user_roles.py +++ b/python/prohibition_web_svc/tests/test_user_roles.py @@ -2,7 +2,7 @@ from datetime import datetime import responses import python.prohibition_web_svc.middleware.keycloak_middleware as middleware -from python.prohibition_web_svc.models import db, UserRole +from python.common.models import db, UserRole from python.prohibition_web_svc.app import create_app from python.prohibition_web_svc.config import Config import logging diff --git a/python/prohibition_web_svc/tests/test_users.py b/python/prohibition_web_svc/tests/test_users.py index ec0c4ef8e..6c13f4c23 100644 --- a/python/prohibition_web_svc/tests/test_users.py +++ b/python/prohibition_web_svc/tests/test_users.py @@ -2,7 +2,7 @@ import responses import json import python.prohibition_web_svc.middleware.keycloak_middleware as middleware -from python.prohibition_web_svc.models import db, User, UserRole +from python.common.models import db, User, UserRole from python.prohibition_web_svc.app import create_app from python.prohibition_web_svc.config import Config import logging diff --git a/python/task_scheduler/dbfuncs.py b/python/task_scheduler/dbfuncs.py index 77f5affc9..2d98f8360 100644 --- a/python/task_scheduler/dbfuncs.py +++ b/python/task_scheduler/dbfuncs.py @@ -4,7 +4,7 @@ import logging import logging.config from python.task_scheduler.config import Config -from python.task_scheduler.models import Event,FormStorageRefs +from python.common.models import Event,FormStorageRefs import logging import json from datetime import datetime diff --git a/python/task_scheduler/main.py b/python/task_scheduler/main.py index f6e6121da..c95b9345c 100644 --- a/python/task_scheduler/main.py +++ b/python/task_scheduler/main.py @@ -11,7 +11,7 @@ # app = Flask(__name__) from flask_api import FlaskAPI -from python.task_scheduler.models import db +from python.common.models import db logging.config.dictConfig(Config.LOGGING) app = FlaskAPI(__name__) diff --git a/python/task_scheduler/models.py b/python/task_scheduler/models.py deleted file mode 100644 index 78e8f7aa8..000000000 --- a/python/task_scheduler/models.py +++ /dev/null @@ -1,758 +0,0 @@ -from datetime import datetime, timedelta -from dataclasses import dataclass -from flask_sqlalchemy import SQLAlchemy -from flask_migrate import Migrate -import logging - -db = SQLAlchemy() -migrate = Migrate() - -@dataclass -class Form(db.Model): - id: str - form_type: str - lease_expiry: datetime - printed_timestamp: datetime - spoiled_timestamp: datetime - user_guid: str - - id = db.Column('id', db.String(20), primary_key=True) - form_type = db.Column(db.String(20), nullable=False) - lease_expiry = db.Column(db.Date, nullable=True) - printed_timestamp = db.Column(db.DateTime, nullable=True) - spoiled_timestamp = db.Column(db.DateTime, nullable=True) - user_guid = db.Column(db.String(80), db.ForeignKey( - 'user.user_guid'), nullable=True) - - # Note: The printed timestamp prior to v0.4.17 was saved in local Pacific time instead of GMT - - def __init__(self, form_id, form_type, printed=None, spoiled=None, lease_expiry=None, user_guid=None): - self.id = form_id - self.form_type = form_type - self.printed_timestamp = printed - self.spoiled_timestamp = spoiled - self.lease_expiry = lease_expiry - self.user_guid = user_guid - - def lease(self, user_guid): - today = datetime.now() - lease_expiry = today + timedelta(days=30) - self.lease_expiry = lease_expiry - self.user_guid = user_guid - logging.info("{} leased {} until {}".format( - self.user_guid, self.id, self.lease_expiry.strftime("%Y-%m-%d"))) - - @staticmethod - def _format_lease_expiry(lease_expiry): - if lease_expiry is None: - return '' - else: - return datetime.strftime(lease_expiry, "%Y-%m-%d") - - @staticmethod - def collection_to_dict(all_rows): - result_list = [] - for row in all_rows: - result_list.append(Form.serialize(row)) - return result_list - - -class User(db.Model): - user_guid = db.Column(db.String(120), primary_key=True) - business_guid = db.Column(db.String(120), nullable=True) - username = db.Column(db.String(80), nullable=False) - agency = db.Column(db.String(120), nullable=False) - badge_number = db.Column(db.String(12), nullable=False) - last_name = db.Column(db.String(40), nullable=False) - first_name = db.Column(db.String(40), nullable=True) - display_name = db.Column(db.String(80), nullable=True) - login = db.Column(db.String(80), nullable=False) - - def __init__(self, username, user_guid, agency, badge_number, last_name, login, business_guid='', display_name='', first_name=''): - self.username = username - self.user_guid = user_guid - self.agency = agency - self.badge_number = badge_number - self.last_name = last_name - self.first_name = first_name - self.business_guid = business_guid - self.display_name = display_name - self.login = login - - @staticmethod - def serialize(user): - return { - "username": user.username, - "user_guid": user.user_guid, - "agency": user.agency, - "badge_number": user.badge_number, - "first_name": user.first_name, - "last_name": user.last_name, - "display_name": user.display_name, - "login": user.login - } - - -class UserRole(db.Model): - role_name = db.Column(db.String(20), primary_key=True) - user_guid = db.Column(db.String(80), db.ForeignKey( - 'user.user_guid'), primary_key=True) - submitted_dt = db.Column(db.DateTime, nullable=True) - approved_dt = db.Column(db.DateTime, nullable=True) - - def __init__(self, role_name, user_guid, submitted_dt=None, approved_dt=None): - self.role_name = role_name - self.user_guid = user_guid - self.submitted_dt = submitted_dt - self.approved_dt = approved_dt - - @staticmethod - def serialize(role): - return { - "role_name": role.role_name, - "user_guid": role.user_guid, - "submitted_dt": role.submitted_dt, - "approved_dt": role.approved_dt - } - - @staticmethod - def serialize_all_users(rows): - return { - "agency": rows.agency, - "approved_dt": rows.approved_dt, - "badge_number": rows.badge_number, - "first_name": rows.first_name, - "last_name": rows.last_name, - "role_name": rows.role_name, - "submitted_dt": rows.submitted_dt, - "user_guid": rows.user_guid, - "username": rows.username, - "login": rows.login, - } - - @staticmethod - def collection_to_dict(all_rows, serialization_method: str): - result_list = [] - for row in all_rows: - method = getattr(UserRole, serialization_method) - result_list.append(method(row)) - return result_list - - @staticmethod - def collection_to_list_roles(all_rows): - result_list = [] - for row in all_rows: - result_list.append(row.role_name) - return result_list - - @staticmethod - def get_roles(user_guid): - rows = db.session.query(UserRole) \ - .filter(UserRole.user_guid == user_guid) \ - .filter(UserRole.approved_dt != None) \ - .all() - return UserRole.collection_to_list_roles(rows) - - -@dataclass -class Agency(db.Model): - __tablename__ = 'agency' - - id: int - vjur: str - agency_name: str - - id = db.Column(db.Integer, primary_key=True) - vjur = db.Column(db.String) - agency_name = db.Column(db.String) - - -@dataclass -class City(db.Model): - __tablename__ = 'city' - - id: int - objectCd: str - objectDsc: str - - id = db.Column(db.Integer, primary_key=True) - objectCd = db.Column(db.String) - objectDsc = db.Column(db.String) - - -@dataclass -class Country(db.Model): - __tablename__ = 'country' - - id: int - objectCd: str - objectDsc: str - - id = db.Column(db.Integer, primary_key=True) - objectCd = db.Column(db.String) - objectDsc = db.Column(db.String) - - -@dataclass -class ImpoundLotOperator(db.Model): - __tablename__ = 'impound_lot_operator' - - id: int - name: str - lot_address: str - city: str - phone: str - - id = db.Column(db.Integer, primary_key=True) - name = db.Column(db.String) - lot_address = db.Column(db.String) - city = db.Column(db.String) - phone = db.Column(db.String) - - -@dataclass -class Jurisdiction(db.Model): - __tablename__ = 'jurisdiction' - - id: int - objectCd: str - objectDsc: str - - id = db.Column(db.Integer, primary_key=True) - objectCd = db.Column(db.String) - objectDsc = db.Column(db.String) - - -@dataclass -class Permission(db.Model): - __tablename__ = 'permission' - - id: int - role: str - permission: str - - id = db.Column(db.Integer, primary_key=True) - role = db.Column(db.String) - permission = db.Column(db.String) - - -@dataclass -class Province(db.Model): - __tablename__ = 'province' - - id: int - objectCd: str - objectDsc: str - - id = db.Column(db.Integer, primary_key=True) - objectCd = db.Column(db.String) - objectDsc = db.Column(db.String) - - -@dataclass -class VehicleStyle(db.Model): - __tablename__ = 'vehicle_style' - - code: str - name: str - - code = db.Column(db.String, primary_key=True) - name = db.Column(db.String) - -@dataclass -class VehicleType(db.Model): - __tablename__ = 'vehicle_type' - - type_cd: int - description: str - - type_cd = db.Column(db.Integer, primary_key=True) - description = db.Column(db.String) - - -@dataclass -class Vehicle(db.Model): - __tablename__ = 'vehicle' - - id: int - mk: str - search: str - md: str - - id = db.Column(db.Integer, primary_key=True) - mk = db.Column(db.String) - search = db.Column(db.String) - md = db.Column(db.String) - - -@dataclass -class VehicleColour(db.Model): - __tablename__ = 'vehicle_colour' - - code: str - display_name: str - colour_class: str - - code = db.Column(db.String, primary_key=True) - display_name = db.Column(db.String) - colour_class = db.Column(db.String) - - -@dataclass -class Event(db.Model): - __tablename__ = 'event' - - event_id: int - icbc_sent_status: str - vi_sent_status: str - icbc_retry_count: int - vi_retry_count: int - driver_licence_no: str - driver_jurisdiction: str - driver_last_name: str - driver_given_name: str - driver_dob: datetime - driver_address: str - driver_city: str - driver_prov: str - driver_postal: str - driver_phone: str - date_of_driving: datetime - time_of_driving: str - vehicle_jurisdiction: str - vehicle_plate_no: str - vehicle_registration_no: str - vehicle_year: str - vehicle_mk_md: str - vehicle_style: str - vehicle_type: int - vehicle_colour: str - vehicle_vin_no: str - nsc_prov_state: str - location_of_keys: str - nsc_no: str - type_of_prohibition: str - owned_by_corp: bool - vehicle_released_to: str - date_released: datetime - time_released: str - intersection_or_address_of_offence: str - offence_city: str - corporation_name: str - regist_owner_last_name: str - regist_owner_first_name: str - regist_owner_address: str - regist_owner_dob: datetime - regist_owner_city: str - regist_owner_prov: str - regist_owner_postal: str - regist_owner_phone: str - regist_owner_email: str - agency_file_no: str - submitted: bool - confirmation_of_service: bool - confirmation_of_service_date: datetime - impound_lot_operator: int - task_processing_status: str - created_dt: datetime - updated_dt: datetime - created_by: str - updated_by: str - - event_id = db.Column(db.Integer, primary_key=True) - icbc_sent_status = db.Column(db.String) - vi_sent_status = db.Column(db.String) - icbc_retry_count = db.Column(db.Integer) - vi_retry_count = db.Column(db.Integer) - type_of_prohibition = db.Column(db.String) - driver_licence_no = db.Column(db.String) - driver_jurisdiction = db.Column(db.String) - driver_last_name = db.Column(db.String) - driver_given_name = db.Column(db.String) - driver_dob = db.Column(db.DateTime) - driver_address = db.Column(db.String) - driver_city = db.Column(db.String) - driver_prov = db.Column(db.String) - driver_postal = db.Column(db.String) - driver_phone = db.Column(db.String) - agency_file_no = db.Column(db.String) - date_of_driving = db.Column(db.DateTime) - time_of_driving = db.Column(db.String) - vehicle_jurisdiction = db.Column(db.String) - vehicle_plate_no = db.Column(db.String) - vehicle_registration_no = db.Column(db.String) - vehicle_year = db.Column(db.String) - vehicle_mk_md = db.Column(db.String) - vehicle_style = db.Column(db.String) - vehicle_type = db.Column(db.Integer, db.ForeignKey('vehicle_type.type_cd')) - vehicle_colour = db.Column(db.String) - vehicle_vin_no = db.Column(db.String) - nsc_prov_state = db.Column(db.String) - location_of_keys = db.Column(db.String) - nsc_no = db.Column(db.String) - submitted = db.Column(db.Boolean) - owned_by_corp = db.Column(db.Boolean) - vehicle_released_to = db.Column(db.String) - date_released = db.Column(db.DateTime) - time_released = db.Column(db.String) - intersection_or_address_of_offence = db.Column(db.String) - offence_city = db.Column(db.String) - corporation_name = db.Column(db.String) - regist_owner_last_name = db.Column(db.String) - regist_owner_first_name = db.Column(db.String) - regist_owner_address = db.Column(db.String) - regist_owner_dob = db.Column(db.DateTime) - regist_owner_city = db.Column(db.String) - regist_owner_prov = db.Column(db.String) - regist_owner_postal = db.Column(db.String) - regist_owner_phone = db.Column(db.String) - regist_owner_email = db.Column(db.String) - impound_lot_operator = db.Column( - db.Integer, db.ForeignKey('impound_lot_operator.id')) - confirmation_of_service = db.Column(db.Boolean) - confirmation_of_service_date = db.Column(db.DateTime) - task_processing_status = db.Column(db.String) - created_by = db.Column(db.String, db.ForeignKey('user.user_guid')) - updated_by = db.Column(db.String) - created_dt = db.Column(db.DateTime) - updated_dt = db.Column(db.DateTime) - - twenty_four_hour_form = db.relationship( - 'TwentyFourHourForm', - backref='event', - lazy='joined', - uselist=False) - - twelve_hour_form = db.relationship( - 'TwelveHourForm', - backref='event', - lazy='joined', - uselist=False) - - vi_form = db.relationship( - 'VIForm', - backref='event', - lazy='joined', - uselist=False) - - irp_form = db.relationship( - 'IRPForm', - backref='event', - lazy='joined', - uselist=False) - - -@dataclass -class TwentyFourHourForm(db.Model): - __tablename__ = 'twenty_four_hour_form' - - form_id: int - event_id: int - vehicle_impounded: str - reason_for_not_impounding: str - reasonable_ground_other_reason: str - prescribed_test_used: str - reasonable_date_of_test: datetime - reasonable_time_of_test: str - reason_for_not_using_prescribed_test: str - resonable_test_used_alcohol: str - reasonable_asd_expiry_date: datetime - reasonable_result_alcohol: str - reasonable_bac_result_mg: str - resonable_approved_instrument_used: str - reasonable_test_used_drugs: str - reasonable_can_drive_drug: bool - reasonable_can_drive_alcohol: bool - requested_can_drive_alcohol: bool - requested_can_drive_drug: bool - requested_approved_instrument_used: str - requested_BAC_result: str - requested_alcohol_test_result: str - requested_ASD_expiry_date: datetime - time_of_requested_test: str - requested_test_used_alcohol: str - requested_test_used_drug: str - requested_prescribed_test: str - witnessed_by_officer: bool - admission_by_driver: bool - independent_witness: bool - reasonable_ground_other: bool - twenty_four_hour_number: str - created_dt: datetime - updated_dt: datetime - - form_id = db.Column(db.Integer, primary_key=True) - event_id = db.Column(db.Integer, db.ForeignKey('event.event_id')) - vehicle_impounded = db.Column(db.String) - reason_for_not_impounding = db.Column(db.String) - reasonable_ground_other_reason = db.Column(db.String) - prescribed_test_used = db.Column(db.String) - reasonable_date_of_test = db.Column(db.DateTime) - reasonable_time_of_test = db.Column(db.String) - reason_for_not_using_prescribed_test = db.Column(db.String) - resonable_test_used_alcohol = db.Column(db.String) - reasonable_asd_expiry_date = db.Column(db.DateTime) - reasonable_result_alcohol = db.Column(db.String) - reasonable_bac_result_mg = db.Column(db.String) - resonable_approved_instrument_used = db.Column(db.String) - reasonable_test_used_drugs = db.Column(db.String) - reasonable_can_drive_drug = db.Column(db.Boolean) - reasonable_can_drive_alcohol = db.Column(db.Boolean) - requested_can_drive_alcohol = db.Column(db.Boolean) - requested_can_drive_drug = db.Column(db.Boolean) - requested_approved_instrument_used = db.Column(db.String) - requested_BAC_result = db.Column(db.String) - requested_alcohol_test_result = db.Column(db.String) - requested_ASD_expiry_date = db.Column(db.DateTime) - time_of_requested_test = db.Column(db.String) - requested_test_used_alcohol = db.Column(db.String) - requested_test_used_drug = db.Column(db.String) - requested_prescribed_test = db.Column(db.String) - witnessed_by_officer = db.Column(db.Boolean) - admission_by_driver = db.Column(db.Boolean) - independent_witness = db.Column(db.Boolean) - reasonable_ground_other = db.Column(db.Boolean) - twenty_four_hour_number = db.Column(db.String) - created_dt = db.Column(db.DateTime) - updated_dt = db.Column(db.DateTime) - - -@dataclass -class TwelveHourForm(db.Model): - __tablename__ = 'twelve_hour_form' - - form_id: int - event_id: int - created_dt: datetime - updated_dt: datetime - driver_phone: str - twelve_hour_number: str - - form_id = db.Column(db.Integer, primary_key=True) - event_id = db.Column(db.Integer, db.ForeignKey('event.event_id')) - created_dt = db.Column(db.DateTime) - updated_dt = db.Column(db.DateTime) - driver_phone = db.Column(db.String) - twelve_hour_number = db.Column(db.String) - - -@dataclass -class IRPForm(db.Model): - __tablename__ = 'irp_form' - - form_id: int - event_id: int - created_dt: datetime - updated_dt: datetime - - form_id = db.Column(db.Integer, primary_key=True) - event_id = db.Column(db.Integer, db.ForeignKey('event.event_id')) - created_dt = db.Column(db.DateTime) - updated_dt = db.Column(db.DateTime) - - -@dataclass -class VIForm(db.Model): - __tablename__ = 'vi_form' - - form_id: int - event_id: int - created_dt: datetime - updated_dt: datetime - gender: str - driver_is_regist_owner: bool - driver_licence_expiry: datetime - driver_licence_class: str - unlicenced_prohibition_number: str - belief_driver_bc_resident: str - out_of_province_dl: str - out_of_province_dl_number: str - out_of_province_dl_expiry: str - out_of_province_dl_jurisdiction: str - date_of_impound: datetime - irp_impound: str - irp_impound_duration: str - IRP_number: str - VI_number: str - excessive_speed: str - prohibited: bool - suspended: bool - street_racing: bool - stunt_driving: bool - motorcycle_seating: bool - motorcycle_restrictions: bool - unlicensed: bool - linkage_location_of_keys: bool - linkage_location_of_keys_explanation: str - linkage_driver_principal: bool - linkage_owner_in_vehicle: bool - linkage_owner_aware_possesion: bool - linkage_vehicle_transfer_notice: bool - linkage_other: bool - speed_limit: str - vehicle_speed: str - speed_estimation_technique: str - speed_confirmation_technique: str - incident_details: str - incident_details_extra_page: bool - - form_id = db.Column(db.Integer, primary_key=True) - event_id = db.Column(db.Integer, db.ForeignKey('event.event_id')) - gender = db.Column(db.String) - driver_is_regist_owner = db.Column(db.String) - driver_licence_expiry = db.Column(db.DateTime) - driver_licence_class = db.Column(db.String) - unlicenced_prohibition_number = db.Column(db.String) - belief_driver_bc_resident = db.Column(db.String) - out_of_province_dl = db.Column(db.String) - out_of_province_dl_number = db.Column(db.String) - out_of_province_dl_expiry = db.Column(db.String) - out_of_province_dl_jurisdiction = db.Column(db.String) - date_of_impound = db.Column(db.DateTime) - irp_impound = db.Column(db.String) - irp_impound_duration = db.Column(db.String) - IRP_number = db.Column(db.String) - VI_number = db.Column(db.String) - excessive_speed = db.Column(db.Boolean) - prohibited = db.Column(db.Boolean) - suspended = db.Column(db.Boolean) - street_racing = db.Column(db.Boolean) - stunt_driving = db.Column(db.Boolean) - motorcycle_seating = db.Column(db.Boolean) - motorcycle_restrictions = db.Column(db.Boolean) - unlicensed = db.Column(db.Boolean) - linkage_location_of_keys = db.Column(db.Boolean) - linkage_location_of_keys_explanation = db.Column(db.String) - linkage_driver_principal = db.Column(db.Boolean) - linkage_owner_in_vehicle = db.Column(db.Boolean) - linkage_owner_aware_possesion = db.Column(db.Boolean) - linkage_vehicle_transfer_notice = db.Column(db.Boolean) - linkage_other = db.Column(db.Boolean) - speed_limit = db.Column(db.String) - vehicle_speed = db.Column(db.String) - speed_estimation_technique = db.Column(db.String) - speed_confirmation_technique = db.Column(db.String) - incident_details = db.Column(db.String) - incident_details_extra_page = db.Column(db.Boolean) - created_dt = db.Column(db.DateTime) - updated_dt = db.Column(db.DateTime) - - -@dataclass -class FormStorageRefs(db.Model): - __tablename__ = 'form_storage_refs' - - form_id_24h: int - form_id_irp: int - form_id_vi: int - form_id_12h: int - event_id: int - form_type: str - # vi, irp, 24h, 12h - storage_key: str - encryptiv: str - created_dt: datetime - updated_dt: datetime - - storage_key = db.Column(db.String, primary_key=True) - form_id_24h = db.Column(db.Integer, db.ForeignKey( - 'twenty_four_hour_form.form_id')) - form_id_irp = db.Column(db.Integer, db.ForeignKey('irp_form.form_id')) - form_id_vi = db.Column(db.Integer, db.ForeignKey('vi_form.form_id')) - form_id_12h = db.Column( - db.Integer, db.ForeignKey('twelve_hour_form.form_id')) - event_id = db.Column(db.Integer, db.ForeignKey('event.event_id')) - form_type = db.Column(db.String) - encryptiv = db.Column(db.String) - created_dt = db.Column(db.DateTime) - updated_dt = db.Column(db.DateTime) - - - -@dataclass -class AgencyCrossref(db.Model): - __tablename__ = 'agency_cross_refs' - - agency_name: str - agency_id: str - agency_city: str - prime_vjur: str - icbc_detachment_name: str - icbc_city_name: str - vips_policedetachments_agency_id: str - vips_policedetachments_agency_nm: str - - agency_name = db.Column(db.String, primary_key=True) - agency_id = db.Column(db.String) - agency_city = db.Column(db.String) - prime_vjur = db.Column(db.String) - icbc_detachment_name = db.Column(db.String) - icbc_city_name = db.Column(db.String) - vips_policedetachments_agency_id = db.Column(db.String) - vips_policedetachments_agency_nm = db.Column(db.String) - -@dataclass -class JurisdictionCrossRef(db.Model): - __tablename__ = 'jurisdiction_cross_ref' - - jurisdiction_name: str - jurisdiction_code: str - prime_jurisdiction_code: str - icbc_jurisdiction_code: str - icbc_jurisdiction: str - vips_jurisdictions_objectCd: str - vips_jurisdictions_objectDsc: str - - jurisdiction_name = db.Column(db.String, primary_key=True) - jurisdiction_code = db.Column(db.String) - prime_jurisdiction_code = db.Column(db.String) - icbc_jurisdiction_code = db.Column(db.String) - icbc_jurisdiction = db.Column(db.String) - vips_jurisdictions_objectCd = db.Column(db.String) - vips_jurisdictions_objectDsc = db.Column(db.String) - -@dataclass -class CityCrossRef(db.Model): - __tablename__ = 'city_cross_ref' - - city_code: str - city_name: str - icbc_city_code: str - icbc_city_name: str - icbc_city_name_legacy: str - vips_city_name: str - - city_code = db.Column(db.String, primary_key=True) - city_name = db.Column(db.String) - icbc_city_code = db.Column(db.String) - icbc_city_name = db.Column(db.String) - icbc_city_name_legacy = db.Column(db.String) - vips_city_name = db.Column(db.String) - -@dataclass -class ImpoundReasonCodes(db.Model): - __tablename__ = 'impound_reason_codes' - - df_unique_code: str - impound_reason_name: str - vips_value_cd: str - vips_value_dsc: str - vips_value_abbreviated_dsc: str - - df_unique_code = db.Column(db.String, primary_key=True) - impound_reason_name = db.Column(db.String) - vips_value_cd = db.Column(db.String) - vips_value_dsc = db.Column(db.String) - vips_value_abbreviated_dsc = db.Column(db.String) - -@dataclass -class IloIdCrossRef(db.Model): - __tablename__ = 'ilo_cross_ref' - - ilo_name: str - vips_impound_lot_operator_id: int - - ilo_name = db.Column(db.String, primary_key=True) - vips_impound_lot_operator_id = db.Column(db.Integer) - - From e883c1f846bd3767d3e8513825e86d0e06325294 Mon Sep 17 00:00:00 2001 From: Ratheesh kumar R Date: Wed, 11 Sep 2024 17:05:42 -0700 Subject: [PATCH 03/22] Updating docker files --- python/form_handler/Dockerfile | 1 - python/form_handler/Dockerfile-local | 3 +-- python/task_scheduler/Dockerfile | 1 - python/task_scheduler/Dockerfile-local | 1 - 4 files changed, 1 insertion(+), 5 deletions(-) diff --git a/python/form_handler/Dockerfile b/python/form_handler/Dockerfile index ad7be8a28..096653208 100644 --- a/python/form_handler/Dockerfile +++ b/python/form_handler/Dockerfile @@ -17,7 +17,6 @@ RUN source /opt/app-root/etc/scl_enable && \ COPY __init__.py ${APP_ROOT}/src/python/ COPY common ${APP_ROOT}/src/python/common COPY form_handler ${APP_ROOT}/src/python/form_handler -COPY prohibition_web_svc/models.py ${APP_ROOT}/src/python/form_handler/models.py # RUN ["python", "-m", "pytest", "backend/form_handler"] diff --git a/python/form_handler/Dockerfile-local b/python/form_handler/Dockerfile-local index a7512d2a0..3b559be66 100644 --- a/python/form_handler/Dockerfile-local +++ b/python/form_handler/Dockerfile-local @@ -12,8 +12,7 @@ ENV PYTHONPATH /app/ COPY common /app/python/common COPY form_handler /app/python/form_handler -COPY prohibition_web_svc/models.py /app/python/form_handler/models.py # RUN ["python", "-m", "pytest"] -CMD [ "python", "./python/form_handler/listener.py" ] +CMD [ "python", "./python/form_handler/listener.py" ] \ No newline at end of file diff --git a/python/task_scheduler/Dockerfile b/python/task_scheduler/Dockerfile index a33d93d9f..0a31d0691 100644 --- a/python/task_scheduler/Dockerfile +++ b/python/task_scheduler/Dockerfile @@ -16,6 +16,5 @@ RUN source /opt/app-root/etc/scl_enable && \ COPY __init__.py ${APP_ROOT}/src/python/ COPY common ${APP_ROOT}/src/python/common COPY task_scheduler ${APP_ROOT}/src/python/task_scheduler -COPY prohibition_web_svc/models.py ${APP_ROOT}/src/python/task_scheduler/models.py CMD [ "gunicorn", "--error-logfile", "-", "--bind", "0.0.0.0:5000", "--pythonpath", "python/task_scheduler", "main:create_app()" ] \ No newline at end of file diff --git a/python/task_scheduler/Dockerfile-local b/python/task_scheduler/Dockerfile-local index a7cdc19a7..16c393a1e 100644 --- a/python/task_scheduler/Dockerfile-local +++ b/python/task_scheduler/Dockerfile-local @@ -13,6 +13,5 @@ ENV PYTHONPATH /app/ COPY common /app/python/common COPY task_scheduler /app/python/task_scheduler -COPY prohibition_web_svc/models.py /app/python/task_scheduler/models.py CMD [ "gunicorn", "--error-logfile", "-", "--bind", "0.0.0.0:5000", "--pythonpath", "/app/python/task_scheduler", "main:create_app()" ] \ No newline at end of file From 70ad3d18157f337f942609dc1177c383b5a0e694 Mon Sep 17 00:00:00 2001 From: Ratheesh kumar R Date: Fri, 13 Sep 2024 09:30:16 -0700 Subject: [PATCH 04/22] Record errors happening while creating event or event pdf --- python/common/enums.py | 86 ++++++++++++++ python/common/error_middleware.py | 112 ++++++++++++++++++ python/common/models.py | 26 ++++ .../prohibition_web_svc/blueprints/events.py | 5 +- .../middleware/event_middleware.py | 73 ++++++++++++ .../migrations/versions/6c4aa4e1039d_.py | 49 ++++++++ 6 files changed, 350 insertions(+), 1 deletion(-) create mode 100644 python/common/enums.py create mode 100644 python/common/error_middleware.py create mode 100644 python/prohibition_web_svc/migrations/versions/6c4aa4e1039d_.py diff --git a/python/common/enums.py b/python/common/enums.py new file mode 100644 index 000000000..020a92ff1 --- /dev/null +++ b/python/common/enums.py @@ -0,0 +1,86 @@ +from enum import Enum +from dataclasses import dataclass + +@dataclass +class EnumDetails: + code: str + description: str + +class BaseEnum(Enum): + def __str__(self): + return self.value.code + + @property + def code(self): + return self.value.code + + @property + def description(self): + return self.value.description + +class EventType(BaseEnum): + TWELVE_HOUR = EnumDetails("TwelveHour", "12 Hour Prohibition") + TWENTY_FOUR_HOUR = EnumDetails("TwentyFourHour", "24 Hour Prohibition") + IRP = EnumDetails("IRP", "Immediate Roadside Prohibition") + VI = EnumDetails("VI", "Vehicle Impoundment") + +class ErrorCategory(BaseEnum): + VALIDATION = EnumDetails("VALIDATION", "Validation Error") + SYSTEM = EnumDetails("SYSTEM", "System Error") + CONNECTION = EnumDetails("CONNECTION", "Connection Error") + DATA = EnumDetails("DATA", "Data Error") + OTHER = EnumDetails("OTHER", "Other Error") + +class ErrorSeverity(BaseEnum): + LOW = EnumDetails("LOW", "Low Severity") + MEDIUM = EnumDetails("MEDIUM", "Medium Severity") + HIGH = EnumDetails("HIGH", "High Severity") + CRITICAL = EnumDetails("CRITICAL", "Critical Severity") + +class ErrorStatus(BaseEnum): + NEW = EnumDetails("NEW", "New Error") + VIEWED = EnumDetails("VIEWED", "Viewed Error") + IN_PROGRESS = EnumDetails("IN_PROGRESS", "In Progress") + ASSIGNED = EnumDetails("ASSIGNED", "Assigned") + RESOLVED = EnumDetails("RESOLVED", "Resolved") + CANCELLED = EnumDetails("CANCELLED", "Cancelled") + CLOSED = EnumDetails("CLOSED", "Closed") + +@dataclass +class ErrorCodeDetails: + code: str + description: str # max 200 characters + category: ErrorCategory + severity: ErrorSeverity + resolution: str + is_business_error: bool + +class ErrorCode(BaseEnum): + # General error + G00 = ErrorCodeDetails("G00", "General error", ErrorCategory.OTHER, ErrorSeverity.LOW, + "Contact DF application support for further investigation", False) + + # Events related error + + E01 = ErrorCodeDetails("E01", "Event saving error", ErrorCategory.DATA, ErrorSeverity.HIGH, + "Contact DF application support for further investigation", False) + E02 = ErrorCodeDetails("E02", "Event PDF saving error", ErrorCategory.DATA, ErrorSeverity.HIGH, + "Contact DF application support for further investigation", False) + + # Add more error codes as needed... + + @property + def category(self): + return self.value.category + + @property + def severity(self): + return self.value.severity + + @property + def resolution(self): + return self.value.resolution + + @property + def is_business_error(self): + return self.value.is_business_error diff --git a/python/common/error_middleware.py b/python/common/error_middleware.py new file mode 100644 index 000000000..cf78d8ef2 --- /dev/null +++ b/python/common/error_middleware.py @@ -0,0 +1,112 @@ +# Error middleware functions +import json +import logging +import functools +import inspect +from flask import request, current_app +from sqlalchemy.exc import SQLAlchemyError +from datetime import datetime + +from python.common.enums import EventType, ErrorCode, ErrorSeverity, ErrorStatus, ErrorCategory +from python.common.models import db, DFErrors, Event + +def get_safe_payload(): + """ + Safely extract and serialize the request payload. + """ + try: + # Try to get JSON payload + payload = request.get_json(silent=True) + if payload is not None: + return json.dumps(payload) + + # If not JSON, try to get form data + payload = request.form.to_dict() + if payload: + return json.dumps(payload) + + # If no form data, get query parameters + payload = request.args.to_dict() + if payload: + return json.dumps(payload) + + # If all else fails, return a message indicating no payload + return json.dumps({"message": "No payload found in request"}) + except Exception as e: + return json.dumps({"error": "Failed to serialize payload", "details": str(e)}) + +def get_function_info(func): + """ + Extract detailed information about the function or method. + """ + module = inspect.getmodule(func) + if module: + module_name = module.__name__ + else: + module_name = "unknown_module" + + if inspect.ismethod(func): + class_name = func.__self__.__class__.__name__ + return f"{module_name}.{class_name}.{func.__name__}" + elif inspect.isfunction(func): + return f"{module_name}.{func.__name__}" + else: + return f"{module_name}.unknown_function" + +def record_error(error_code: ErrorCode, error_details, event_id: int, event_type: EventType = None, ticket_no=None, func=None, payload=None): + """ + Record an error in the database. + """ + try: + + function_path = get_function_info(func) if func else "unknown" + + if not payload: + payload = get_safe_payload() + + error_log = DFErrors( + error_cd=error_code, + error_cd_desc=error_code.description, + error_category_cd=error_code.category, + error_severity_level_cd=error_code.severity, + error_status_cd=ErrorStatus.NEW, + event_id=event_id, + event_type=event_type, + ticket_no=ticket_no, + req_payload=payload, + error_details=error_details, + error_path=function_path, + created_by='SYSTEM', + received_dt=datetime.now(), + ) + db.session.add(error_log) + db.session.commit() + logging.error(f"Error recorded: {error_code} - {error_code.description} - Event ID: {event_id} - Event Type: {event_type} - Function: {function_path} - {error_details}") + except SQLAlchemyError as e: + db.session.rollback() + logging.error(f"Failed to record error: {str(e)}") + +def error_handler(func): + """ + Decorator to handle errors in functions. + """ + @functools.wraps(func) + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except Exception as e: + error_code = ErrorCode.G00 # Default to general error + error_details = str(e) + + # Attempt to get event_id from kwargs or request + event_id = kwargs.get('event_id') + if not event_id and hasattr(request, 'view_args'): + event_id = request.view_args.get('event_id') + + if not event_id: + current_app.logger.warning("No event_id found for error logging") + event_id = None # or some default value + + record_error(error_code, error_details, event_id, func=func) + raise # Re-raise the exception after recording + return wrapper \ No newline at end of file diff --git a/python/common/models.py b/python/common/models.py index 7965edac8..66b396751 100644 --- a/python/common/models.py +++ b/python/common/models.py @@ -3,6 +3,7 @@ from flask_sqlalchemy import SQLAlchemy from flask_migrate import Migrate import logging +from python.common.enums import ErrorCategory, ErrorSeverity, ErrorStatus, ErrorCode, EventType db = SQLAlchemy() migrate = Migrate() @@ -790,3 +791,28 @@ class IloIdCrossRef(db.Model): vips_impound_lot_operator_id = db.Column(db.Integer) +@dataclass +class DFErrors(db.Model): + __tablename__ = 'df_errors' + + error_id: int = db.Column(db.Integer, primary_key=True) + error_cd: ErrorCode = db.Column(db.Enum(ErrorCode), nullable=False) + error_cd_desc: str = db.Column(db.String(200), nullable=False) + error_category_cd: ErrorCategory = db.Column(db.Enum(ErrorCategory), nullable=False) + error_severity_level_cd: ErrorSeverity = db.Column(db.Enum(ErrorSeverity), nullable=False) + error_details: str = db.Column(db.Text) + error_path: str = db.Column(db.String(200)) + event_id: int = db.Column(db.Integer, db.ForeignKey('event.event_id'), nullable=True) + event_type: EventType = db.Column(db.Enum(EventType), nullable=True) + ticket_no: str = db.Column(db.String(50)) + received_dt: datetime = db.Column(db.DateTime, default=datetime.now()) + error_status_cd: ErrorStatus = db.Column(db.Enum(ErrorStatus), default=ErrorStatus.NEW) + req_payload: str = db.Column(db.Text) + + created_by: str = db.Column(db.String(150)) + created_dt: datetime = db.Column(db.DateTime, default=datetime.now()) + updated_by: str = db.Column(db.String(150)) + updated_dt: datetime = db.Column(db.DateTime, onupdate=datetime.now()) + + def __repr__(self): + return f'' \ No newline at end of file diff --git a/python/prohibition_web_svc/blueprints/events.py b/python/prohibition_web_svc/blueprints/events.py index 6982cb1cc..1c1aa8a7f 100644 --- a/python/prohibition_web_svc/blueprints/events.py +++ b/python/prohibition_web_svc/blueprints/events.py @@ -58,9 +58,12 @@ def create(): {"try": http_responses.server_error_response, "fail": []}, ]}, {"try": event_middleware.save_event_data, "fail": [ + {"try": event_middleware.record_event_error, "fail": []}, {"try": http_responses.bad_request_response, "fail": []} ]}, - {"try": event_middleware.save_event_pdf, "fail": []}, + {"try": event_middleware.save_event_pdf, "fail": [ + {"try": event_middleware.record_event_error, "fail": []} + ]}, {"try": splunk.log_to_splunk, "fail": []}, {"try": http_responses.successful_create_response, "fail": []}, ], diff --git a/python/prohibition_web_svc/middleware/event_middleware.py b/python/prohibition_web_svc/middleware/event_middleware.py index 064592e32..43c59fd73 100644 --- a/python/prohibition_web_svc/middleware/event_middleware.py +++ b/python/prohibition_web_svc/middleware/event_middleware.py @@ -17,6 +17,8 @@ import img2pdf import uuid from split_image import split_image +from python.common.enums import ErrorCode, EventType +from python.common.error_middleware import record_error def validate_update(**kwargs) -> tuple: @@ -238,10 +240,20 @@ def save_event_data(**kwargs) -> tuple: if data.get('IRP'): return logging.debug('Saving Event') + db.session.add(event) db.session.commit() except Exception as e: logging.error(e, stack_info=True) + # Set error in kwargs to get consumed by the record_event_error function + kwargs['error'] = { + 'error_code': ErrorCode.E01, + 'error_details': str(e), + 'event_id': None, + 'event_type': get_event_type(data), + 'ticket_no': get_ticket_no(data), + 'func': save_event_data, + } return False, kwargs kwargs['response_dict'] = jsonify(event) kwargs['event'] = event @@ -352,6 +364,7 @@ def save_event_pdf(**kwargs) -> tuple: client.fput_object(Config.STORAGE_BUCKET_NAME, encoded_file_name, encoded_pdf_filepath) logging.debug('File uploaded') + form_storage = FormStorageRefs( form_id_12h=event.twelve_hour_form.form_id, @@ -366,6 +379,15 @@ def save_event_pdf(**kwargs) -> tuple: except Exception as e: logging.warning(str(e)) + # Set error in kwargs to get consumed by the record_event_error function + kwargs['error'] = { + 'error_code': ErrorCode.E02, + 'error_details': str(e), + 'event_id': event.event_id, + 'event_type': get_event_type(data), + 'ticket_no': get_ticket_no(data), + 'func': save_event_pdf, + } return False, kwargs return True, kwargs @@ -422,3 +444,54 @@ def validate_form_payload(**kwargs) -> tuple: return True, kwargs logging.warning("validation error: " + json.dumps('')) return False, kwargs + +def record_event_error(**kwargs): + """ + Record an error that occurred during event processing. + + Args: + **kwargs: Additional keyword arguments, including event_id and event_type. + """ + + + try: + error = kwargs.get('error') + + if error is None: + logging.warning("Error object is None") + return + record_error(**error) + + + except Exception as e: + # If recording the error itself fails, log it + logging.error(f"Failed to record error: {str(e)}") + return True, kwargs + + return True, kwargs + +def get_event_type(data): + event_type = None + if data.get('TwelveHour'): + event_type = EventType.TWELVE_HOUR + elif data.get('TwentyFourHour'): + event_type = EventType.TWENTY_FOUR_HOUR + elif data.get('IRP'): + event_type = EventType.IRP + elif data.get('VI'): + event_type = EventType.VI + else: + event_type = None + return event_type + +def get_ticket_no(data): + if data.get('TwelveHour'): + return data.get("twelve_hour_number") + elif data.get('TwentyFourHour'): + return data.get("twenty_four_hour_number") + elif data.get('IRP'): + return data.get('IRP_number') + elif data.get('VI'): + return data.get('VI_number') + else: + return None \ No newline at end of file diff --git a/python/prohibition_web_svc/migrations/versions/6c4aa4e1039d_.py b/python/prohibition_web_svc/migrations/versions/6c4aa4e1039d_.py new file mode 100644 index 000000000..28a0aa7ad --- /dev/null +++ b/python/prohibition_web_svc/migrations/versions/6c4aa4e1039d_.py @@ -0,0 +1,49 @@ +"""empty message + +Revision ID: 6c4aa4e1039d +Revises: 0fab578072b7 +Create Date: 2024-09-12 16:18:16.174245 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '6c4aa4e1039d' +down_revision = '0fab578072b7' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('df_errors', + sa.Column('error_id', sa.Integer(), nullable=False), + sa.Column('error_cd', sa.Enum('G00', 'E01', 'E02', name='errorcode'), nullable=False), + sa.Column('error_cd_desc', sa.String(length=200), nullable=False), + sa.Column('error_category_cd', sa.Enum('VALIDATION', 'SYSTEM', 'CONNECTION', 'DATA', 'OTHER', name='errorcategory'), nullable=False), + sa.Column('error_severity_level_cd', sa.Enum('LOW', 'MEDIUM', 'HIGH', 'CRITICAL', name='errorseverity'), nullable=False), + sa.Column('error_details', sa.Text(), nullable=True), + sa.Column('error_path', sa.String(length=200), nullable=True), + sa.Column('event_id', sa.Integer(), nullable=True), + sa.Column('event_type', sa.Enum('TWELVE_HOUR', 'TWENTY_FOUR_HOUR', 'IRP', 'VI', name='eventtype'), nullable=True), + sa.Column('ticket_no', sa.String(length=50), nullable=True), + sa.Column('received_dt', sa.DateTime(), nullable=True), + sa.Column('error_status_cd', sa.Enum('NEW', 'VIEWED', 'IN_PROGRESS', 'ASSIGNED', 'RESOLVED', 'CANCELLED', 'CLOSED', name='errorstatus'), nullable=True), + sa.Column('req_payload', sa.Text(), nullable=True), + sa.Column('created_by', sa.String(length=150), nullable=True), + sa.Column('created_dt', sa.DateTime(), nullable=True), + sa.Column('updated_by', sa.String(length=150), nullable=True), + sa.Column('updated_dt', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['event_id'], ['event.event_id'], ), + sa.PrimaryKeyConstraint('error_id') + ) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('df_errors') + # ### end Alembic commands ### From e4ba7981df2d0e9af11b80186696708185313134 Mon Sep 17 00:00:00 2001 From: Ratheesh kumar R Date: Fri, 13 Sep 2024 17:19:57 -0700 Subject: [PATCH 05/22] Adding error recording in task scheduler --- python/task_scheduler/dbfuncs.py | 3 +++ python/task_scheduler/eventqueuefuncs.py | 34 +++++++++++++++++++++--- python/task_scheduler/main.py | 2 +- 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/python/task_scheduler/dbfuncs.py b/python/task_scheduler/dbfuncs.py index 2d98f8360..2abd1b404 100644 --- a/python/task_scheduler/dbfuncs.py +++ b/python/task_scheduler/dbfuncs.py @@ -9,6 +9,7 @@ import json from datetime import datetime from sqlalchemy import or_ +from python.task_scheduler.eventqueuefuncs import record_queue_error def query_pending_events(app,db): """Query the database for pending events.""" @@ -48,6 +49,7 @@ def query_pending_events(app,db): continue except Exception as e: logging.error(e) + record_queue_error(app, None, query_pending_events, str(e) ) errmsg=f"Error querying pending events: {e}" return False,errmsg, None logging.debug(events) @@ -71,6 +73,7 @@ def update_event_status(app,db,event_id,status): db.session.commit() except Exception as e: logging.error(e) + record_queue_error(app, {'event_id': event_id}, update_event_status, str(e) ) errmsg=f"Error updating processed event status: {e}" return False,errmsg return True,None diff --git a/python/task_scheduler/eventqueuefuncs.py b/python/task_scheduler/eventqueuefuncs.py index 5b3ed73f4..73f0cf474 100644 --- a/python/task_scheduler/eventqueuefuncs.py +++ b/python/task_scheduler/eventqueuefuncs.py @@ -4,9 +4,11 @@ from python.task_scheduler.message import encode_message import logging import json +from python.common.error_middleware import record_error +from python.common.enums import ErrorCode +from python.common.models import Event,FormStorageRefs - -def add_to_event_queue(writer,message): +def add_to_event_queue(app, writer, message): logging.debug("inside add_to_event_queue()") errmsg='' try: @@ -14,8 +16,34 @@ def add_to_event_queue(writer,message): logging.debug('add_to_event_queue(): {}'.format(json.dumps(message))) if not writer.publish(queue_name, encode_message(message, Config.ENCRYPT_KEY)): logging.critical('unable to write to RabbitMQ {} queue'.format(queue_name)) + record_queue_error(app, message, add_to_event_queue, 'unable to write to RabbitMQ {} queue'.format(queue_name)) return False, errmsg except Exception as e: logging.error(e) + record_queue_error(app, message, add_to_event_queue, str(e)) return False, errmsg - return True, errmsg \ No newline at end of file + return True, errmsg + +def record_queue_error(app, event, func, error_details=None): + try: + + with app.app_context(): + + event_id = event.get('event_id') if event else None + event_type = event.get('event_type') if event else None + error_obj = { + 'error_code': ErrorCode.E03, + 'error_details': error_details, + 'event_id': event_id, + 'event_type': event_type, + 'payload': json.dumps(event) if event else None, + 'func': func, + } + + record_error(**error_obj) + except Exception as e: + # Log the error that occurred during the error recording process + logging.error(f"Error in record_queue_error: {str(e)}") + logging.error(f"Original error details: message={event}, func={func}, error_details={error_details}") + + \ No newline at end of file diff --git a/python/task_scheduler/main.py b/python/task_scheduler/main.py index c95b9345c..b1607edb8 100644 --- a/python/task_scheduler/main.py +++ b/python/task_scheduler/main.py @@ -40,7 +40,7 @@ def process_pending_events(): for event in all_events: logging.debug(event) try: - statusval1,errmsg=add_to_event_queue(writer,event) + statusval1, errmsg=add_to_event_queue(app, writer, event) if not statusval1: logging.error(errmsg) continue From 88bdce7bd6cb666b2c4787e3ca125fa1b902184d Mon Sep 17 00:00:00 2001 From: Ratheesh kumar R Date: Fri, 13 Sep 2024 17:24:33 -0700 Subject: [PATCH 06/22] Added error record to form services --- python/common/enums.py | 15 +++ python/common/error_middleware.py | 17 +-- python/common/models.py | 15 ++- .../blueprints/admin_forms.py | 1 + .../prohibition_web_svc/blueprints/forms.py | 2 + .../middleware/form_middleware.py | 123 +++++++++++++----- .../{6c4aa4e1039d_.py => 211a0fba4348_.py} | 18 +-- 7 files changed, 135 insertions(+), 56 deletions(-) rename python/prohibition_web_svc/migrations/versions/{6c4aa4e1039d_.py => 211a0fba4348_.py} (65%) diff --git a/python/common/enums.py b/python/common/enums.py index 020a92ff1..fec1382b3 100644 --- a/python/common/enums.py +++ b/python/common/enums.py @@ -23,6 +23,7 @@ class EventType(BaseEnum): TWENTY_FOUR_HOUR = EnumDetails("TwentyFourHour", "24 Hour Prohibition") IRP = EnumDetails("IRP", "Immediate Roadside Prohibition") VI = EnumDetails("VI", "Vehicle Impoundment") + OTHER = EnumDetails("OTHER", "Other") class ErrorCategory(BaseEnum): VALIDATION = EnumDetails("VALIDATION", "Validation Error") @@ -67,6 +68,20 @@ class ErrorCode(BaseEnum): E02 = ErrorCodeDetails("E02", "Event PDF saving error", ErrorCategory.DATA, ErrorSeverity.HIGH, "Contact DF application support for further investigation", False) + E03 = ErrorCodeDetails("E03", "Error putting event to queue", ErrorCategory.SYSTEM, ErrorSeverity.HIGH, + "Contact DF application support for further investigation", False) + + # Forms related error + + F01 = ErrorCodeDetails("F01", "Form id lease error", ErrorCategory.DATA, ErrorSeverity.HIGH, + "Contact DF application support for further investigation", False) + F02 = ErrorCodeDetails("F02", "Renew form id lease error", ErrorCategory.DATA, ErrorSeverity.HIGH, + "Contact DF application support for further investigation", False) + F03 = ErrorCodeDetails("F02", "Admin form create error", ErrorCategory.DATA, ErrorSeverity.HIGH, + "Contact DF application support for further investigation", False) + + + # Add more error codes as needed... @property diff --git a/python/common/error_middleware.py b/python/common/error_middleware.py index cf78d8ef2..db9c43f15 100644 --- a/python/common/error_middleware.py +++ b/python/common/error_middleware.py @@ -53,7 +53,7 @@ def get_function_info(func): else: return f"{module_name}.unknown_function" -def record_error(error_code: ErrorCode, error_details, event_id: int, event_type: EventType = None, ticket_no=None, func=None, payload=None): +def record_error(error_code: ErrorCode, error_details, event_id: int = None, event_type: EventType = None, ticket_no=None, func=None, payload=None): """ Record an error in the database. """ @@ -65,13 +65,14 @@ def record_error(error_code: ErrorCode, error_details, event_id: int, event_type payload = get_safe_payload() error_log = DFErrors( - error_cd=error_code, - error_cd_desc=error_code.description, - error_category_cd=error_code.category, - error_severity_level_cd=error_code.severity, - error_status_cd=ErrorStatus.NEW, + error_cd=str(error_code.code), + error_cd_desc=str(error_code.description), + error_category_cd=str(error_code.category), + error_resolution=str(error_code.resolution), + error_severity_level_cd=str(error_code.severity), + error_status_cd=str(ErrorStatus.NEW), event_id=event_id, - event_type=event_type, + event_type=str(event_type) if event_type else None, # Convert to string if not None ticket_no=ticket_no, req_payload=payload, error_details=error_details, @@ -84,7 +85,7 @@ def record_error(error_code: ErrorCode, error_details, event_id: int, event_type logging.error(f"Error recorded: {error_code} - {error_code.description} - Event ID: {event_id} - Event Type: {event_type} - Function: {function_path} - {error_details}") except SQLAlchemyError as e: db.session.rollback() - logging.error(f"Failed to record error: {str(e)}") + logging.error(f"Failed to record error: {error_code} - {error_code.description} - Event ID: {event_id} - Event Type: {event_type} - Function: {function_path} - {error_details}") def error_handler(func): """ diff --git a/python/common/models.py b/python/common/models.py index 66b396751..8ea64f896 100644 --- a/python/common/models.py +++ b/python/common/models.py @@ -3,7 +3,7 @@ from flask_sqlalchemy import SQLAlchemy from flask_migrate import Migrate import logging -from python.common.enums import ErrorCategory, ErrorSeverity, ErrorStatus, ErrorCode, EventType +from python.common.enums import ErrorStatus db = SQLAlchemy() migrate = Migrate() @@ -796,17 +796,18 @@ class DFErrors(db.Model): __tablename__ = 'df_errors' error_id: int = db.Column(db.Integer, primary_key=True) - error_cd: ErrorCode = db.Column(db.Enum(ErrorCode), nullable=False) + error_cd: str = db.Column(db.String(5), nullable=False) error_cd_desc: str = db.Column(db.String(200), nullable=False) - error_category_cd: ErrorCategory = db.Column(db.Enum(ErrorCategory), nullable=False) - error_severity_level_cd: ErrorSeverity = db.Column(db.Enum(ErrorSeverity), nullable=False) + error_resolution: str = db.Column(db.Text) + error_category_cd: str = db.Column(db.String(10), nullable=False) + error_severity_level_cd: str = db.Column(db.String(10), nullable=False) error_details: str = db.Column(db.Text) error_path: str = db.Column(db.String(200)) event_id: int = db.Column(db.Integer, db.ForeignKey('event.event_id'), nullable=True) - event_type: EventType = db.Column(db.Enum(EventType), nullable=True) - ticket_no: str = db.Column(db.String(50)) + event_type: str = db.Column(db.String(10), nullable=True) + ticket_no: str = db.Column(db.String(50), nullable=True) received_dt: datetime = db.Column(db.DateTime, default=datetime.now()) - error_status_cd: ErrorStatus = db.Column(db.Enum(ErrorStatus), default=ErrorStatus.NEW) + error_status_cd: str = db.Column(db.String(200), default=ErrorStatus.NEW) req_payload: str = db.Column(db.Text) created_by: str = db.Column(db.String(150)) diff --git a/python/prohibition_web_svc/blueprints/admin_forms.py b/python/prohibition_web_svc/blueprints/admin_forms.py index 0b2d139f4..f344ce13a 100644 --- a/python/prohibition_web_svc/blueprints/admin_forms.py +++ b/python/prohibition_web_svc/blueprints/admin_forms.py @@ -86,6 +86,7 @@ def create(): {"try": splunk_middleware.admin_create_form, "fail": []}, {"try": splunk.log_to_splunk, "fail": []}, {"try": form_middleware.admin_create_form, "fail": [ + {"try": form_middleware.record_form_error, "fail": []}, {"try": http_responses.server_error_response, "fail": []}, ]} ], diff --git a/python/prohibition_web_svc/blueprints/forms.py b/python/prohibition_web_svc/blueprints/forms.py index 5f5255c63..b071f01bb 100644 --- a/python/prohibition_web_svc/blueprints/forms.py +++ b/python/prohibition_web_svc/blueprints/forms.py @@ -52,6 +52,7 @@ def create(): {"try": http_responses.bad_request_response, "fail": []} ]}, {"try": form_middleware.lease_a_form_id, "fail": [ + {"try": form_middleware.record_form_error, "fail": []}, {"try": splunk_middleware.insufficient_form_ids, "fail": []}, {"try": splunk.log_to_splunk, "fail": []}, {"try": http_responses.server_error_response, "fail": []}, @@ -80,6 +81,7 @@ def update(): {"try": form_middleware.request_contains_a_payload, "fail": [ # Request contains no payload - renew form lease {"try": form_middleware.renew_form_id_lease, "fail": [ + {"try": form_middleware.record_form_error, "fail": []}, # {"try": splunk_middleware.unable_to_renew_lease, "fail": []}, # {"try": splunk.log_to_splunk, "fail": []}, {"try": http_responses.bad_request_response, "fail": []}, diff --git a/python/prohibition_web_svc/middleware/form_middleware.py b/python/prohibition_web_svc/middleware/form_middleware.py index 251c25196..1bde35586 100644 --- a/python/prohibition_web_svc/middleware/form_middleware.py +++ b/python/prohibition_web_svc/middleware/form_middleware.py @@ -8,7 +8,8 @@ from flask import jsonify, make_response from python.common.models import db, Form from python.prohibition_web_svc.config import Config - +from python.common.error_middleware import record_error +from python.common.enums import ErrorCode def validate_update(**kwargs) -> tuple: return True, kwargs @@ -22,25 +23,40 @@ def log_payload_to_splunk(**kwargs) -> tuple: def lease_a_form_id(**kwargs) -> tuple: - logging.debug('inside lease_a_form_id()') - data = kwargs.get('payload') - user_guid = kwargs.get('user_guid') - id_list = [] - for form_type in data: - ids = db.session.query(Form) \ - .filter(Form.form_type == form_type) \ - .filter(Form.user_guid == None) \ - .limit(data.get(form_type)) - if ids is None: - logging.warning('Insufficient unique ids available for {}'.format(form_type)) - return False, kwargs - for id in ids: - logging.debug(f'id: {id}') - id.lease(user_guid) - id_list.append(asdict(id)) try: + logging.debug('inside lease_a_form_id()') + data = kwargs.get('payload') + user_guid = kwargs.get('user_guid') + id_list = [] + for form_type in data: + ids = db.session.query(Form) \ + .filter(Form.form_type == form_type) \ + .filter(Form.user_guid == None) \ + .limit(data.get(form_type)) + + if ids is None: + logging.warning('Insufficient unique ids available for {}'.format(form_type)) + record_error( + { + 'error_code': ErrorCode.F01, + 'error_details': f'Insufficient unique ids available for {form_type}', + 'event_type': form_type, + 'func': lease_a_form_id, + } + ) + return False, kwargs + for id in ids: + logging.debug(f'id: {id}') + id.lease(user_guid) + id_list.append(asdict(id)) db.session.commit() except Exception as e: + kwargs['error'] = { + 'error_code': ErrorCode.F01, + 'error_details': str(e), + 'event_type': kwargs.get('form_type'), + 'func': lease_a_form_id, + } return False, kwargs kwargs['response_dict'] = jsonify({'forms':id_list}) return True, kwargs @@ -48,23 +64,38 @@ def lease_a_form_id(**kwargs) -> tuple: def renew_form_id_lease(**kwargs) -> tuple: logging.debug('inside renew_form_id_lease()') - form_type = kwargs.get('form_type') - user_guid = kwargs.get('user_guid') - form_id = kwargs.get('form_id') - form = db.session.query(Form) \ - .filter(Form.form_type == form_type) \ - .filter(Form.user_guid == user_guid) \ - .filter(Form.printed_timestamp == None) \ - .filter(Form.spoiled_timestamp == None) \ - .filter(Form.id == form_id) \ - .first() - if form is None: - logging.warning('User, {}, cannot renew the lease on {} form'.format(user_guid, form_id)) - return False, kwargs - form.lease(user_guid) + try: + form_type = kwargs.get('form_type') + user_guid = kwargs.get('user_guid') + form_id = kwargs.get('form_id') + form = db.session.query(Form) \ + .filter(Form.form_type == form_type) \ + .filter(Form.user_guid == user_guid) \ + .filter(Form.printed_timestamp == None) \ + .filter(Form.spoiled_timestamp == None) \ + .filter(Form.id == form_id) \ + .first() + if form is None: + logging.warning('User, {}, cannot renew the lease on {} form'.format(user_guid, form_id)) + record_error( + { + 'error_code': ErrorCode.F02, + 'error_details': 'User, {}, cannot renew the lease on {} form'.format(user_guid, form_id), + 'event_type': form_type, + 'func': renew_form_id_lease, + } + ) + return False, kwargs + form.lease(user_guid) db.session.commit() except Exception as e: + kwargs['error'] = { + 'error_code': ErrorCode.F02, + 'error_details': str(e), + 'event_type': kwargs.get('form_type'), + 'func': renew_form_id_lease, + } return False, kwargs kwargs['response_dict'] = Form.serialize(form) return True, kwargs @@ -115,7 +146,7 @@ def mark_form_as_printed_or_spoiled(**kwargs) -> tuple: # .filter(Form.id == number) \ # .first() # if form is None: -# logging.warning(f'{user_guid}, cannot update {payload.get(form)} as spoiled - record not found') +# logging.warning(f'{user_guid}, cannot update {payload.get(form)} as spoiled - record not found') # return False, kwargs # form.spoiled_timestamp = payload.get('spoiled_timestamp') # try: @@ -232,6 +263,12 @@ def admin_create_form(**kwargs) -> tuple: kwargs['response'] = make_response({"success": True}, 201) except Exception as e: logging.warning(str(e)) + kwargs['error'] = { + 'error_code': ErrorCode.F02, + 'error_details': str(e), + 'event_type': kwargs.get('form_type'), + 'func': renew_form_id_lease, + } return False, kwargs return True, kwargs @@ -245,3 +282,25 @@ def convert_vancouver_to_utc(iso_datetime_string: str) -> datetime: utc_timezone = pytz.timezone("UTC") printed = iso8601.parse_date(iso_datetime_string) return printed.astimezone(utc_timezone).replace(tzinfo=None) + +def record_form_error(**kwargs): + """ + Record an error that occurred during form processing. + + Args: + **kwargs: Additional keyword arguments, including form_id and form_type. + """ + try: + error = kwargs.get('error') + + if error is None: + logging.warning("Error object is None") + return + + record_error(**error) + + except Exception as e: + # If recording the error itself fails, log it + logging.error(f"Failed to record form error: {str(e)}") + + return True, kwargs diff --git a/python/prohibition_web_svc/migrations/versions/6c4aa4e1039d_.py b/python/prohibition_web_svc/migrations/versions/211a0fba4348_.py similarity index 65% rename from python/prohibition_web_svc/migrations/versions/6c4aa4e1039d_.py rename to python/prohibition_web_svc/migrations/versions/211a0fba4348_.py index 28a0aa7ad..42a838784 100644 --- a/python/prohibition_web_svc/migrations/versions/6c4aa4e1039d_.py +++ b/python/prohibition_web_svc/migrations/versions/211a0fba4348_.py @@ -1,8 +1,8 @@ """empty message -Revision ID: 6c4aa4e1039d +Revision ID: 211a0fba4348 Revises: 0fab578072b7 -Create Date: 2024-09-12 16:18:16.174245 +Create Date: 2024-09-13 13:59:03.417953 """ from alembic import op @@ -10,7 +10,7 @@ # revision identifiers, used by Alembic. -revision = '6c4aa4e1039d' +revision = '211a0fba4348' down_revision = '0fab578072b7' branch_labels = None depends_on = None @@ -20,17 +20,18 @@ def upgrade(): # ### commands auto generated by Alembic - please adjust! ### op.create_table('df_errors', sa.Column('error_id', sa.Integer(), nullable=False), - sa.Column('error_cd', sa.Enum('G00', 'E01', 'E02', name='errorcode'), nullable=False), + sa.Column('error_cd', sa.String(length=5), nullable=False), sa.Column('error_cd_desc', sa.String(length=200), nullable=False), - sa.Column('error_category_cd', sa.Enum('VALIDATION', 'SYSTEM', 'CONNECTION', 'DATA', 'OTHER', name='errorcategory'), nullable=False), - sa.Column('error_severity_level_cd', sa.Enum('LOW', 'MEDIUM', 'HIGH', 'CRITICAL', name='errorseverity'), nullable=False), + sa.Column('error_resolution', sa.Text(), nullable=True), + sa.Column('error_category_cd', sa.String(length=10), nullable=False), + sa.Column('error_severity_level_cd', sa.String(length=10), nullable=False), sa.Column('error_details', sa.Text(), nullable=True), sa.Column('error_path', sa.String(length=200), nullable=True), sa.Column('event_id', sa.Integer(), nullable=True), - sa.Column('event_type', sa.Enum('TWELVE_HOUR', 'TWENTY_FOUR_HOUR', 'IRP', 'VI', name='eventtype'), nullable=True), + sa.Column('event_type', sa.String(length=10), nullable=True), sa.Column('ticket_no', sa.String(length=50), nullable=True), sa.Column('received_dt', sa.DateTime(), nullable=True), - sa.Column('error_status_cd', sa.Enum('NEW', 'VIEWED', 'IN_PROGRESS', 'ASSIGNED', 'RESOLVED', 'CANCELLED', 'CLOSED', name='errorstatus'), nullable=True), + sa.Column('error_status_cd', sa.String(length=200), nullable=True), sa.Column('req_payload', sa.Text(), nullable=True), sa.Column('created_by', sa.String(length=150), nullable=True), sa.Column('created_dt', sa.DateTime(), nullable=True), @@ -39,7 +40,6 @@ def upgrade(): sa.ForeignKeyConstraint(['event_id'], ['event.event_id'], ), sa.PrimaryKeyConstraint('error_id') ) - # ### end Alembic commands ### From 6b1b416dddcd8c47a719b20dcf412f5de6ac0dd3 Mon Sep 17 00:00:00 2001 From: Ratheesh kumar R Date: Wed, 18 Sep 2024 09:12:33 -0700 Subject: [PATCH 07/22] DF-1758: Digital form error function integrated --- python/common/enums.py | 10 ++ python/common/error_middleware.py | 2 +- python/form_handler/actions.py | 184 ++++++++++++++++++++++++++++-- python/form_handler/business.py | 13 ++- python/form_handler/config.py | 1 + 5 files changed, 199 insertions(+), 11 deletions(-) diff --git a/python/common/enums.py b/python/common/enums.py index fec1382b3..c76f2c0aa 100644 --- a/python/common/enums.py +++ b/python/common/enums.py @@ -70,6 +70,16 @@ class ErrorCode(BaseEnum): E03 = ErrorCodeDetails("E03", "Error putting event to queue", ErrorCategory.SYSTEM, ErrorSeverity.HIGH, "Contact DF application support for further investigation", False) + E04 = ErrorCodeDetails("E04", "Form handler: Unknown Event Type", ErrorCategory.DATA, ErrorSeverity.CRITICAL, + "Contact DF application support for further investigation", False) + E05 = ErrorCodeDetails("E05", "Form handler: Retry count exceed maximum retires", ErrorCategory.DATA, ErrorSeverity.CRITICAL, + "Contact DF application support for further investigation", False) + E06 = ErrorCodeDetails("E06", "Form handler: Event On Hold", ErrorCategory.DATA, ErrorSeverity.CRITICAL, + "Contact DF application support for further investigation", False) + E07 = ErrorCodeDetails("E07", "Form handlerr: Event process error", ErrorCategory.DATA, ErrorSeverity.CRITICAL, + "Contact DF application support for further investigation", False) + E08 = ErrorCodeDetails("E08", "General Event process error", ErrorCategory.SYSTEM, ErrorSeverity.CRITICAL, + "Contact DF application support for further investigation", False) # Forms related error diff --git a/python/common/error_middleware.py b/python/common/error_middleware.py index db9c43f15..07413b9ee 100644 --- a/python/common/error_middleware.py +++ b/python/common/error_middleware.py @@ -33,7 +33,7 @@ def get_safe_payload(): # If all else fails, return a message indicating no payload return json.dumps({"message": "No payload found in request"}) except Exception as e: - return json.dumps({"error": "Failed to serialize payload", "details": str(e)}) + return json.dumps({"message": "No payload found in request"}) def get_function_info(func): """ diff --git a/python/form_handler/actions.py b/python/form_handler/actions.py index 40147d966..e150e1299 100644 --- a/python/form_handler/actions.py +++ b/python/form_handler/actions.py @@ -18,6 +18,8 @@ from python.form_handler.payloads import vips_payload,vips_document_payload from python.form_handler.message import encode_message from python.form_handler.helper import method2_decrypt,decryptPdf_method1,convertDateTime,convertDateTimeWithSecs +from python.common.error_middleware import record_error +from python.common.enums import ErrorCode import fitz import base64 @@ -30,7 +32,9 @@ def get_storage_ref_event_type(**args) -> tuple: Get the event type from the message """ logging.debug("inside actions_get_storage_ref_event_type()") + try: + application=args.get('app') db=args.get('db') message = args.get('message') @@ -176,7 +180,7 @@ def validate_event_retry_count(**args)->tuple: event_type=args.get('event_type') message = args.get('message') queue_name = message.get('queue_name', None) - retry_count=message.get('retry_count',0) + retry_count=message.get('retry_count', 0) retry_count=retry_count+1 args['retry_count']=retry_count args['message']['retry_count']=retry_count @@ -193,10 +197,31 @@ def validate_event_retry_count(**args)->tuple: put_to_queue_name=Config.STORAGE_FAIL_QUEUE_PERS args['stop_retry'] = True args['put_to_queue_name']=put_to_queue_name + + # Set error in args to get consumed by the record_event_error function + args['error'] = { + 'error_code': ErrorCode.E05, + 'error_details': 'Retry count exceeds for an event.', + 'event_id': message.get('event_id'), + 'event_type': message.get('event_type'), + 'func': validate_event_retry_count, + 'payload': message, + } + return False,args except Exception as e: logging.error(e) + # Set error in args to get consumed by the record_event_error function + message = args.get('message') + args['error'] = { + 'error_code': ErrorCode.E03, + 'error_details': str(e), + 'event_id': message.get('event_id'), + 'event_type': message.get('event_type'), + 'func': validate_event_retry_count, + 'payload': message, + } return False,args return True,args @@ -205,7 +230,6 @@ def validate_event_data(**args)->tuple: logging.debug("inside validate_event_data()") logging.debug(args) # TODO: validate vips payload - return True,args @@ -998,8 +1022,9 @@ def update_event_status_hold(**args)->tuple: try: application=args.get('app') db=args.get('db') - event_id=args.get('event_data').get('event_id') - event_type=args.get('event_type') + message=args.get('message') + event_id=message.get('event_id') if message else args.get('event_id') + event_type=message.get('event_type') if message else args.get('event_type') with application.app_context(): if event_type=='vi': event = db.session.query(Event) \ @@ -1015,8 +1040,34 @@ def update_event_status_hold(**args)->tuple: .one() event.icbc_sent_status = 'retrying' db.session.commit() + # Directly call record_event_error + error_args = { + 'error': { + 'error_code': ErrorCode.E06, + 'error_details': f'Holding a {event_type} event', + 'event_id': event_id, + 'event_type': event_type, + 'func': update_event_status_hold, + }, + 'message': args.get('message', {}), + 'app': application + } + record_event_error(**error_args) except Exception as e: logging.error(e) + # Directly call record_event_error + error_args = { + 'error': { + 'error_code': ErrorCode.E06, + 'error_details': f'Exception in update_event_status_hold: {str(e)}', + 'event_id': event_id, + 'event_type': event_type, + 'func': update_event_status_hold, + }, + 'message': args.get('message', {}), + 'app': application + } + record_event_error(**error_args) return False,args return True,args @@ -1043,8 +1094,35 @@ def update_event_status_error(**args)->tuple: .one() event.icbc_sent_status = 'error' db.session.commit() + + # Directly call record_event_error + error_args = { + 'error': { + 'error_code': ErrorCode.E07, + 'error_details': f'Failed to process {event_type} event', + 'event_id': event_id, + 'event_type': event_type, + 'func': update_event_status_error, + }, + 'message': args.get('message', {}), + 'app': application + } + record_event_error(**error_args) except Exception as e: logging.error(e) + # If an exception occurs, record it as an error + error_args = { + 'error': { + 'error_code': ErrorCode.E07, + 'error_details': f'Exception in update_event_status_error: {str(e)}', + 'event_id': event_id, + 'event_type': event_type, + 'func': update_event_status_error, + }, + 'message': args.get('message', {}), + 'app': application + } + record_event_error(**error_args) return False,args return True,args @@ -1055,8 +1133,9 @@ def update_event_status_error_retry(**args)->tuple: try: application=args.get('app') db=args.get('db') - event_id=args.get('event_data').get('event_id') - event_type=args.get('event_type') + message = args.get('message') + event_id=message.get('event_id') + event_type=message.get('event_type') stop_retry_flg=args.get('stop_retry_flg',False) with application.app_context(): if event_type=='vi': @@ -1094,10 +1173,29 @@ def add_to_persistent_failed_queue(**args)->tuple: writer = args.get('writer') logging.debug('add_to_hold_queue(): {}'.format(json.dumps(message))) if not writer.publish(config.STORAGE_FAIL_QUEUE_PERS, encode_message(message, config.ENCRYPT_KEY)): + # Set error in args to get consumed by the record_event_error function + args['error'] = { + 'error_code': ErrorCode.E03, + 'error_details': 'unable to write to RabbitMQ {} queue'.format(config.STORAGE_FAIL_QUEUE_PERS), + 'event_id': message.get('event_id'), + 'event_type': message.get('event_type'), + 'func': add_to_persistent_failed_queue, + 'payload': message, + } logging.critical('unable to write to RabbitMQ {} queue'.format(config.STORAGE_FAIL_QUEUE_PERS)) return False, args except Exception as e: logging.error(e) + # Set error in args to get consumed by the record_event_error function + message = args.get('message') + args['error'] = { + 'error_code': ErrorCode.E03, + 'error_details': str(e), + 'event_id': message.get('event_id'), + 'event_type': message.get('event_type'), + 'func': add_to_persistent_failed_queue, + 'payload': message, + } return False, args return True, args @@ -1139,7 +1237,7 @@ def add_to_hold_queue(**args)->tuple: def add_to_retry_queue(**args)->tuple: logging.debug("inside add_to_retry_queue()") logging.debug(args) - try: + try: config = args.get('config') message = args.get('message') put_to_queue_name=args.get('put_to_queue_name',None) @@ -1152,6 +1250,19 @@ def add_to_retry_queue(**args)->tuple: args['message']['queue_name']=put_to_queue_name if not writer.publish(put_to_queue_name, encode_message(message, config.ENCRYPT_KEY)): logging.critical('unable to write to RabbitMQ {} queue'.format(put_to_queue_name)) + # If an exception occurs, record it as an error + error_args = { + 'error': { + 'error_code': ErrorCode.E03, + 'error_details': 'unable to write to RabbitMQ {} queue'.format(put_to_queue_name), + 'event_id': message.get('event_id'), + 'event_type': message.get('event_type'), + 'func': add_to_retry_queue, + }, + 'message': args.get('message', {}), + 'app': args.get('app') + } + record_event_error(**error_args) return False, args except Exception as e: logging.error(e) @@ -1171,3 +1282,62 @@ def add_unknown_event_error_to_message(**args)->tuple: logging.error(e) return False, args return True,args + +def add_unknown_event_to_error(**args)->tuple: + logging.debug("inside add_unknown_event_error()") + logging.debug(args) + try: + message = args.get('message') + args['error'] = { + 'error_code': ErrorCode.E04, + 'error_details': 'Critical Error: Unknown Event Type', + 'event_id': message.get('event_id'), + 'event_type': message.get('event_type'), + 'func': add_unknown_event_to_error, + 'payload': message, + } + except Exception as e: + logging.error(e) + return False, args + return True,args + +def record_event_error(**args): + """ + Record an error that occurred during event processing. + + Args: + **args: Additional keyword arguments, including event_id and event_type. + """ + + + try: + error = args.get('error') + message = args.get('message') + + application = args.get('app') + with application.app_context(): + if error is None: + error_obj = { + 'error_code': ErrorCode.E08, + 'error_details': message.get('error_message', 'Unknown error'), + 'event_id': message.get('event_id'), + 'event_type': message.get('event_type'), + 'payload': json.dumps(message) if message else None, + 'func': record_event_error, + } + else: + error_obj = { + 'error_code': error.get('error_code'), + 'error_details': error.get('error_details'), + 'event_id': error.get('event_id'), + 'event_type': error.get('event_type'), + 'payload': json.dumps(message) if message else None, + 'func': error.get('func'), + } + + record_error(**error_obj) + return True, args + except Exception as e: + # If recording the error itself fails, log it + logging.error(f"Failed to record error: {str(e)}") + return True, args \ No newline at end of file diff --git a/python/form_handler/business.py b/python/form_handler/business.py index 50b4255ce..53ce15afc 100644 --- a/python/form_handler/business.py +++ b/python/form_handler/business.py @@ -17,8 +17,12 @@ def process_incoming_form() -> dict: return { "unknown_event": [ {"try": actions.add_unknown_event_error_to_message, "fail": []}, - {"try": actions.add_to_persistent_failed_queue, "fail": []}, - {"try": rsi_email.rsiops_unknown_event_type, "fail": []} + {"try": actions.add_to_persistent_failed_queue, "fail": [ + {"try": actions.record_event_error, "fail": []} + ]}, + {"try": rsi_email.rsiops_unknown_event_type, "fail": []}, + {"try": actions.add_unknown_event_to_error, "fail": []}, + {"try": actions.record_event_error, "fail": []} ], "vi": [ # DONE: query form data and event data using storage key from input @@ -40,7 +44,8 @@ def process_incoming_form() -> dict: {"try": actions.validate_event_retry_count, "fail": [ {"try": actions.add_to_retry_queue, "fail": []}, {"try": actions.update_event_status_error_retry, "fail": []}, - {"try": rsi_email.rsiops_event_to_retry_queue, "fail": []} + {"try": actions.record_event_error, "fail": []}, + {"try": rsi_email.rsiops_event_to_retry_queue, "fail": []}, ]}, {"try": actions.get_storage_ref_event_type, "fail": [ {"try": actions.add_to_retry_queue, "fail": []}, @@ -97,6 +102,7 @@ def process_incoming_form() -> dict: {"try": actions.validate_event_retry_count, "fail": [ {"try": actions.add_to_retry_queue, "fail": []}, {"try": actions.update_event_status_error_retry, "fail": []}, + {"try": actions.record_event_error, "fail": []}, {"try": rsi_email.rsiops_event_to_retry_queue, "fail": []} ]}, {"try": actions.get_storage_ref_event_type, "fail": [ @@ -141,6 +147,7 @@ def process_incoming_form() -> dict: {"try": actions.validate_event_retry_count, "fail": [ {"try": actions.add_to_retry_queue, "fail": []}, {"try": actions.update_event_status_error_retry, "fail": []}, + {"try": actions.record_event_error, "fail": []}, {"try": rsi_email.rsiops_event_to_retry_queue, "fail": []} ]}, {"try": actions.get_storage_ref_event_type, "fail": [ diff --git a/python/form_handler/config.py b/python/form_handler/config.py index 65bee5638..bbfcf9c63 100644 --- a/python/form_handler/config.py +++ b/python/form_handler/config.py @@ -63,6 +63,7 @@ class Config(): RSIOPS_EMAIL_ADDRESS = os.getenv('RSIOPS_EMAIL_ADDRESS') REPLY_EMAIL_ADDRESS = os.getenv('REPLY_EMAIL_ADDRESS', 'do-not-reply-rsi@gov.bc.ca') + BCC_EMAIL_ADDRESSES = os.getenv('BCC_EMAIL_ADDRESSES') VIPS_BCC_EMAIL_ADDRESSES = os.getenv('VIPS_BCC_EMAIL_ADDRESSES', '') TMP_STORAGE_LOCAL=os.getenv('TMP_STORAGE_LOCAL') From 5853654d734b8fa17db1e144263b6886986552c2 Mon Sep 17 00:00:00 2001 From: Ratheesh kumar R Date: Tue, 24 Sep 2024 09:46:04 -0700 Subject: [PATCH 08/22] Updated unit test --- python/common/error_middleware.py | 2 +- python/common/tests/record_error/__init__.py | 0 .../tests/record_error/test_record_error.py | 115 ++++++++++++++++++ 3 files changed, 116 insertions(+), 1 deletion(-) create mode 100644 python/common/tests/record_error/__init__.py create mode 100644 python/common/tests/record_error/test_record_error.py diff --git a/python/common/error_middleware.py b/python/common/error_middleware.py index 07413b9ee..cc58e3638 100644 --- a/python/common/error_middleware.py +++ b/python/common/error_middleware.py @@ -72,7 +72,7 @@ def record_error(error_code: ErrorCode, error_details, event_id: int = None, eve error_severity_level_cd=str(error_code.severity), error_status_cd=str(ErrorStatus.NEW), event_id=event_id, - event_type=str(event_type) if event_type else None, # Convert to string if not None + event_type=str(event_type) if event_type else None, ticket_no=ticket_no, req_payload=payload, error_details=error_details, diff --git a/python/common/tests/record_error/__init__.py b/python/common/tests/record_error/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/common/tests/record_error/test_record_error.py b/python/common/tests/record_error/test_record_error.py new file mode 100644 index 000000000..e9fe15964 --- /dev/null +++ b/python/common/tests/record_error/test_record_error.py @@ -0,0 +1,115 @@ +import datetime +import json +import pytest +from flask_api import FlaskAPI + +from python.common.error_middleware import record_error, get_safe_payload, get_function_info +from python.common.models import db, DFErrors +from python.form_handler.config import Config +from python.common.enums import ErrorCode, ErrorStatus, ErrorCategory, ErrorSeverity, EventType + +application = FlaskAPI(__name__) +application.config['SQLALCHEMY_DATABASE_URI'] = Config.DATABASE_URI +db.init_app(application) + +@pytest.fixture +def error_data(): + return { + 'error_code': ErrorCode.G00, + 'error_details': 'Test error details', + 'event_id': 123, + 'event_type': EventType.TWELVE_HOUR, + } + +@pytest.fixture +def app(): + return application + +@pytest.fixture +def client(app): + return app.test_client() + +def test_record_error_success(app, error_data): + with app.app_context(): + record_error( + error_data['error_code'], + error_data['error_details'], + error_data['event_id'], + error_data['event_type'] + ) + + error = DFErrors.query.first() + assert error is not None + assert error.error_cd == error_data['error_code'].code + assert error.error_cd_desc == error_data['error_code'].description + assert error.error_category_cd == str(error_data['error_code'].category) + assert error.error_severity_level_cd == str(error_data['error_code'].severity) + assert error.error_status_cd == str(ErrorStatus.NEW) + assert error.event_id == error_data['event_id'] + assert error.event_type == str(error_data['event_type']) + assert error.error_details == error_data['error_details'] + + # Clear the database after the test + db.session.query(DFErrors).delete() + db.session.commit() + +@pytest.mark.parametrize("error_code,event_id,event_type", [ + (ErrorCode.E01, 456, EventType.IRP), + (ErrorCode.F01, None, None), + (ErrorCode.G00, 789, EventType.VI), +]) +def test_record_error_different_scenarios(app, error_code, event_id, event_type): + with app.app_context(): + record_error(error_code, "Test error", event_id, event_type) + + error = DFErrors.query.first() + assert error is not None + assert error.error_cd == error_code.code + assert error.event_id == event_id + assert error.event_type == (str(event_type) if event_type else None) + + # Clear the database after each test + db.session.query(DFErrors).delete() + db.session.commit() + +def test_get_safe_payload_json(client): + with client.application.test_request_context(json={"key": "value"}): + payload = get_safe_payload() + assert payload == '{"key": "value"}' + +def test_get_safe_payload_form(client): + with client.application.test_request_context(data={"key": "value"}): + payload = get_safe_payload() + assert payload == '{"key": "value"}' + +def test_get_safe_payload_query(client): + with client.application.test_request_context("/?key=value"): + payload = get_safe_payload() + assert payload == '{"key": "value"}' + +def test_get_safe_payload_empty(app): + with app.test_request_context(): + payload = get_safe_payload() + assert payload == '{"message": "No payload found in request"}' + +def test_get_function_info(): + def test_function(): + pass + + function_info = get_function_info(test_function) + assert "test_function" in function_info + +def test_record_error_with_function_info(app): + def test_function(): + pass + + with app.app_context(): + record_error(ErrorCode.G00, "Test error", func=test_function) + + error = DFErrors.query.first() + assert error is not None + assert "test_function" in error.error_path + + # Clear the database after the test + db.session.query(DFErrors).delete() + db.session.commit() \ No newline at end of file From 953c61b9da6b03e0a10ff26189c55593188de2c3 Mon Sep 17 00:00:00 2001 From: Ratheesh kumar R Date: Wed, 25 Sep 2024 09:58:54 -0700 Subject: [PATCH 09/22] Extending record error to actions.py and ride_actions --- python/common/enums.py | 12 ++++++ python/common/ride_actions.py | 43 +++++++++++++++++++ python/form_handler/actions.py | 26 +++++++++++ python/form_handler/business.py | 14 ++++-- .../middleware/form_middleware.py | 4 +- 5 files changed, 94 insertions(+), 5 deletions(-) diff --git a/python/common/enums.py b/python/common/enums.py index c76f2c0aa..9fd3e8318 100644 --- a/python/common/enums.py +++ b/python/common/enums.py @@ -90,6 +90,18 @@ class ErrorCode(BaseEnum): F03 = ErrorCodeDetails("F02", "Admin form create error", ErrorCategory.DATA, ErrorSeverity.HIGH, "Contact DF application support for further investigation", False) + # Ride actions related error + + R01 = ErrorCodeDetails("R01", "Error in sending event to RIDE", ErrorCategory.CONNECTION, ErrorSeverity.HIGH, + "Contact DF application support for further investigation", False) + + # ICBC actions related error + + I01 = ErrorCodeDetails("I01", "Error in sending event to ICBC", ErrorCategory.CONNECTION, ErrorSeverity.HIGH, + "Contact DF application support for further investigation", False) + I02 = ErrorCodeDetails("I02", "Error in preparing payload to ICBC", ErrorCategory.DATA, ErrorSeverity.HIGH, + "Contact DF application support for further investigation", False) + # Add more error codes as needed... diff --git a/python/common/ride_actions.py b/python/common/ride_actions.py index 768b413d2..692fa1d02 100644 --- a/python/common/ride_actions.py +++ b/python/common/ride_actions.py @@ -3,6 +3,7 @@ import requests from python.common.helper import date_time_to_local_tz_string, format_date_only, yes_no_string_to_bool from python.common.config import Config +from python.common.enums import ErrorCode import pytz from python.common.models import Agency, City, ImpoundLotOperator, JurisdictionCrossRef, Province, Vehicle, VehicleColour, VehicleStyle, VehicleType @@ -39,6 +40,13 @@ def twelve_hours_event(**args): logging.debug(f'headers: {headers}') response = requests.post(endpoint, json=eventPayload, verify=False, headers=headers) if response.status_code != 200: + args['error'] = { + 'error_code': ErrorCode.R01, + 'error_details': 'error in sending 12hr_submitted event to RIDE', + 'event_type': '12hr', + 'func': twelve_hours_event, + 'ticket_no': args['form_data']['twelve_hour_number'] + } logging.error('error in sending 12hr_submitted event to RIDE') logging.error(f'error code: {response.status_code} error message: {response.json()}') return False, args @@ -47,6 +55,13 @@ def twelve_hours_event(**args): except Exception as e: logging.error('error in sending 12hr_submitted event to RIDE') logging.error(e) + args['error'] = { + 'error_code': ErrorCode.R01, + 'error_details': str(e), + 'event_type': '12hr', + 'func': twelve_hours_event, + 'ticket_no': args['form_data']['twelve_hour_number'] + } return False, args return True, args @@ -88,6 +103,13 @@ def twenty_four_hours_event(**args): logging.debug(f'headers: {headers}') response = requests.post(endpoint, json=eventPayload, verify=False,headers=headers) if response.status_code != 200: + args['error'] = { + 'error_code': ErrorCode.R01, + 'error_details': 'Error in sending 24hr_submitted event to RIDE', + 'event_type': '24hr', + 'func': twenty_four_hours_event, + 'ticket_no': args['form_data']['twenty_four_hour_number'] + } logging.error('error in sending 24hr_submitted event to RIDE') logging.error(f'error code: {response.status_code} error message: {response.json()}') return False, args @@ -96,6 +118,13 @@ def twenty_four_hours_event(**args): except Exception as e: logging.error('error in sending 24hr_submitted event to RIDE') logging.error(e) + args['error'] = { + 'error_code': ErrorCode.R01, + 'error_details': str(e), + 'event_type': '24hr', + 'func': twenty_four_hours_event, + 'ticket_no': args['form_data']['twenty_four_hour_number'] + } return False, args return True, args @@ -147,6 +176,13 @@ def vi_event(**args): logging.debug(f'headers: {headers}') response = requests.post(endpoint, json=eventPayload, verify=False,headers=headers) if response.status_code != 200: + args['error'] = { + 'error_code': ErrorCode.R01, + 'error_details': 'Error in sending vi_submitted event to RIDE', + 'event_type': 'VI', + 'func': vi_event, + 'ticket_no': args['form_data']['VI_number'] + } logging.error('error in sending vi_submitted event to RIDE') logging.error(f'error code: {response.status_code} error message: {response.json()}') return False, args @@ -155,6 +191,13 @@ def vi_event(**args): except Exception as e: logging.error('error in sending vi_submitted event to RIDE') logging.error(e) + args['error'] = { + 'error_code': ErrorCode.R01, + 'error_details': str(e), + 'event_type': 'VI', + 'func': vi_event, + 'ticket_no': args['form_data']['VI_number'] + } return False, args return True, args diff --git a/python/form_handler/actions.py b/python/form_handler/actions.py index e150e1299..31fbec84f 100644 --- a/python/form_handler/actions.py +++ b/python/form_handler/actions.py @@ -314,6 +314,7 @@ def get_storage_file(**args)->tuple: def prep_icbc_payload(**args)->tuple: logging.debug("inside prep_icbc_payload()") logging.debug(args) + message=args.get('message') try: pdf_data=args.get('file_data') @@ -503,6 +504,14 @@ def prep_icbc_payload(**args)->tuple: args['icbc_payload']=tmp_payload except Exception as e: logging.error(e) + args['error'] = { + 'error_code': ErrorCode.I02, + 'error_details': str(e), + 'event_id': message.get('event_id') if message else None, + 'event_type': message.get('event_type') if message else None, + 'func': prep_icbc_payload, + 'payload': event_data, + } return False,args return True,args @@ -510,6 +519,7 @@ def prep_icbc_payload(**args)->tuple: def send_to_icbc(**args)->tuple: logging.debug("inside send_to_icbc()") logging.debug(args) + message=args.get('message') try: logging.debug(args['icbc_payload']) icbc_payload=args.get('icbc_payload') @@ -518,9 +528,25 @@ def send_to_icbc(**args)->tuple: args['icbc_response_txt']=icbc_response_txt args['icbc_resp_code']=icbc_resp_code if send_status is False: + args['error'] = { + 'error_code': ErrorCode.I01, + 'error_details': 'Error in sending events to ICBC', + 'event_id': message.get('event_id') if message else None, + 'event_type': message.get('event_type') if message else None, + 'func': send_to_icbc, + 'payload': icbc_payload, + } return False,args except Exception as e: logging.error(e) + args['error'] = { + 'error_code': ErrorCode.I01, + 'error_details': str(e), + 'event_id': message.get('event_id') if message else None, + 'event_type': message.get('event_type') if message else None, + 'func': send_to_icbc, + 'payload': icbc_payload, + } return False,args return True,args diff --git a/python/form_handler/business.py b/python/form_handler/business.py index 53ce15afc..53e40be81 100644 --- a/python/form_handler/business.py +++ b/python/form_handler/business.py @@ -95,7 +95,9 @@ def process_incoming_form() -> dict: # {"try": actions.add_to_retry_queue, "fail": []}, # {"try": actions.update_event_status_hold, "fail": []}, # ]}, - {"try": ride_actions.vi_event, "fail": []}, + {"try": ride_actions.vi_event, "fail": [ + {"try": actions.record_event_error, "fail": []}, + ]}, {"try": actions.update_event_status, "fail": []}, ], "24h": [ @@ -130,13 +132,17 @@ def process_incoming_form() -> dict: {"try": actions.prep_icbc_payload, "fail": [ {"try": rsi_email.rsiops_event_to_error_queue, "fail": []}, {"try": actions.add_to_persistent_failed_queue, "fail": []}, + {"try": actions.record_event_error, "fail": []}, {"try": actions.update_event_status_error, "fail": []}, ]}, {"try": actions.send_to_icbc, "fail": [ {"try": actions.add_to_retry_queue, "fail": []}, + {"try": actions.record_event_error, "fail": []}, {"try": actions.update_event_status_hold, "fail": []}, ]}, - {"try": ride_actions.twenty_four_hours_event, "fail": []}, + {"try": ride_actions.twenty_four_hours_event, "fail": [ + {"try": actions.record_event_error, "fail": []}, + ]}, {"try": actions.update_event_status, "fail": []}, # {"try": actions.send_email, "fail": [ # # {"try": actions.add_to_failed_queue, "fail": []} @@ -181,7 +187,9 @@ def process_incoming_form() -> dict: {"try": actions.add_to_retry_queue, "fail": []}, {"try": actions.update_event_status_hold, "fail": []}, ]}, - {"try": ride_actions.twelve_hours_event, "fail": []}, + {"try": ride_actions.twelve_hours_event, "fail": [ + {"try": actions.record_event_error, "fail": []}, + ]}, {"try": actions.update_event_status, "fail": []}, # {"try": actions.send_email, "fail": [ # # {"try": actions.add_to_failed_queue, "fail": []} diff --git a/python/prohibition_web_svc/middleware/form_middleware.py b/python/prohibition_web_svc/middleware/form_middleware.py index 1bde35586..dbb95fca3 100644 --- a/python/prohibition_web_svc/middleware/form_middleware.py +++ b/python/prohibition_web_svc/middleware/form_middleware.py @@ -37,7 +37,7 @@ def lease_a_form_id(**kwargs) -> tuple: if ids is None: logging.warning('Insufficient unique ids available for {}'.format(form_type)) record_error( - { + **{ 'error_code': ErrorCode.F01, 'error_details': f'Insufficient unique ids available for {form_type}', 'event_type': form_type, @@ -79,7 +79,7 @@ def renew_form_id_lease(**kwargs) -> tuple: if form is None: logging.warning('User, {}, cannot renew the lease on {} form'.format(user_guid, form_id)) record_error( - { + **{ 'error_code': ErrorCode.F02, 'error_details': 'User, {}, cannot renew the lease on {} form'.format(user_guid, form_id), 'event_type': form_type, From b07b1f396d700741f5182f9cf6ec5b6d70b1aeb2 Mon Sep 17 00:00:00 2001 From: Ratheesh kumar R Date: Wed, 25 Sep 2024 17:23:00 -0700 Subject: [PATCH 10/22] Change the depth of error_details to store the full stack trace and more details --- python/common/error_middleware.py | 9 ++++++- python/common/models.py | 2 +- python/common/ride_actions.py | 12 +++++----- python/form_handler/actions.py | 24 ++++++++++--------- .../middleware/event_middleware.py | 4 ++-- .../middleware/form_middleware.py | 6 ++--- .../migrations/versions/211a0fba4348_.py | 2 +- 7 files changed, 34 insertions(+), 25 deletions(-) diff --git a/python/common/error_middleware.py b/python/common/error_middleware.py index cc58e3638..2754f2456 100644 --- a/python/common/error_middleware.py +++ b/python/common/error_middleware.py @@ -1,5 +1,6 @@ # Error middleware functions import json +import traceback import logging import functools import inspect @@ -63,6 +64,12 @@ def record_error(error_code: ErrorCode, error_details, event_id: int = None, eve if not payload: payload = get_safe_payload() + + # Handle stack trace extraction if error_details is an exception + if isinstance(error_details, Exception): + stack_trace = ''.join(traceback.format_exception(type(error_details), error_details, error_details.__traceback__)) + else: + stack_trace = str(error_details) error_log = DFErrors( error_cd=str(error_code.code), @@ -75,7 +82,7 @@ def record_error(error_code: ErrorCode, error_details, event_id: int = None, eve event_type=str(event_type) if event_type else None, ticket_no=ticket_no, req_payload=payload, - error_details=error_details, + error_details=stack_trace, error_path=function_path, created_by='SYSTEM', received_dt=datetime.now(), diff --git a/python/common/models.py b/python/common/models.py index 8ea64f896..22a326ca6 100644 --- a/python/common/models.py +++ b/python/common/models.py @@ -804,7 +804,7 @@ class DFErrors(db.Model): error_details: str = db.Column(db.Text) error_path: str = db.Column(db.String(200)) event_id: int = db.Column(db.Integer, db.ForeignKey('event.event_id'), nullable=True) - event_type: str = db.Column(db.String(10), nullable=True) + event_type: str = db.Column(db.String(30), nullable=True) ticket_no: str = db.Column(db.String(50), nullable=True) received_dt: datetime = db.Column(db.DateTime, default=datetime.now()) error_status_cd: str = db.Column(db.String(200), default=ErrorStatus.NEW) diff --git a/python/common/ride_actions.py b/python/common/ride_actions.py index 692fa1d02..b4fb092a3 100644 --- a/python/common/ride_actions.py +++ b/python/common/ride_actions.py @@ -42,7 +42,7 @@ def twelve_hours_event(**args): if response.status_code != 200: args['error'] = { 'error_code': ErrorCode.R01, - 'error_details': 'error in sending 12hr_submitted event to RIDE', + 'error_details': f'Error in sending 12hr_submitted event to RIDE, Response code: {response.status_code} response text: {response.json()}', 'event_type': '12hr', 'func': twelve_hours_event, 'ticket_no': args['form_data']['twelve_hour_number'] @@ -57,7 +57,7 @@ def twelve_hours_event(**args): logging.error(e) args['error'] = { 'error_code': ErrorCode.R01, - 'error_details': str(e), + 'error_details': e, 'event_type': '12hr', 'func': twelve_hours_event, 'ticket_no': args['form_data']['twelve_hour_number'] @@ -105,7 +105,7 @@ def twenty_four_hours_event(**args): if response.status_code != 200: args['error'] = { 'error_code': ErrorCode.R01, - 'error_details': 'Error in sending 24hr_submitted event to RIDE', + 'error_details': f'Error in sending 24hr_submitted event to RIDE, Response code: {response.status_code} response text: {response.json()}', 'event_type': '24hr', 'func': twenty_four_hours_event, 'ticket_no': args['form_data']['twenty_four_hour_number'] @@ -120,7 +120,7 @@ def twenty_four_hours_event(**args): logging.error(e) args['error'] = { 'error_code': ErrorCode.R01, - 'error_details': str(e), + 'error_details': e, 'event_type': '24hr', 'func': twenty_four_hours_event, 'ticket_no': args['form_data']['twenty_four_hour_number'] @@ -178,7 +178,7 @@ def vi_event(**args): if response.status_code != 200: args['error'] = { 'error_code': ErrorCode.R01, - 'error_details': 'Error in sending vi_submitted event to RIDE', + 'error_details': f'Error in sending vi_submitted event to RIDE, Response code: {response.status_code} response text: {response.json()}', 'event_type': 'VI', 'func': vi_event, 'ticket_no': args['form_data']['VI_number'] @@ -193,7 +193,7 @@ def vi_event(**args): logging.error(e) args['error'] = { 'error_code': ErrorCode.R01, - 'error_details': str(e), + 'error_details': e, 'event_type': 'VI', 'func': vi_event, 'ticket_no': args['form_data']['VI_number'] diff --git a/python/form_handler/actions.py b/python/form_handler/actions.py index 31fbec84f..a8cf52d79 100644 --- a/python/form_handler/actions.py +++ b/python/form_handler/actions.py @@ -199,11 +199,13 @@ def validate_event_retry_count(**args)->tuple: args['put_to_queue_name']=put_to_queue_name # Set error in args to get consumed by the record_event_error function + event_id = message.get('event_id', None) + event_type = message.get('event_type', None) args['error'] = { 'error_code': ErrorCode.E05, - 'error_details': 'Retry count exceeds for an event.', - 'event_id': message.get('event_id'), - 'event_type': message.get('event_type'), + 'error_details': f'Retry count exceeds for the event id: {event_id}, event_type: {event_type}', + 'event_id': event_id, + 'event_type': event_type, 'func': validate_event_retry_count, 'payload': message, } @@ -216,7 +218,7 @@ def validate_event_retry_count(**args)->tuple: message = args.get('message') args['error'] = { 'error_code': ErrorCode.E03, - 'error_details': str(e), + 'error_details': e, 'event_id': message.get('event_id'), 'event_type': message.get('event_type'), 'func': validate_event_retry_count, @@ -506,7 +508,7 @@ def prep_icbc_payload(**args)->tuple: logging.error(e) args['error'] = { 'error_code': ErrorCode.I02, - 'error_details': str(e), + 'error_details': e, 'event_id': message.get('event_id') if message else None, 'event_type': message.get('event_type') if message else None, 'func': prep_icbc_payload, @@ -530,7 +532,7 @@ def send_to_icbc(**args)->tuple: if send_status is False: args['error'] = { 'error_code': ErrorCode.I01, - 'error_details': 'Error in sending events to ICBC', + 'error_details': f'icbc_resp_code: {icbc_resp_code} icbc_response_txt: {icbc_response_txt}', 'event_id': message.get('event_id') if message else None, 'event_type': message.get('event_type') if message else None, 'func': send_to_icbc, @@ -541,7 +543,7 @@ def send_to_icbc(**args)->tuple: logging.error(e) args['error'] = { 'error_code': ErrorCode.I01, - 'error_details': str(e), + 'error_details': e, 'event_id': message.get('event_id') if message else None, 'event_type': message.get('event_type') if message else None, 'func': send_to_icbc, @@ -1070,7 +1072,7 @@ def update_event_status_hold(**args)->tuple: error_args = { 'error': { 'error_code': ErrorCode.E06, - 'error_details': f'Holding a {event_type} event', + 'error_details': f'Holding a event_id: {event_id}, event_type:{event_type}', 'event_id': event_id, 'event_type': event_type, 'func': update_event_status_hold, @@ -1085,7 +1087,7 @@ def update_event_status_hold(**args)->tuple: error_args = { 'error': { 'error_code': ErrorCode.E06, - 'error_details': f'Exception in update_event_status_hold: {str(e)}', + 'error_details': e, 'event_id': event_id, 'event_type': event_type, 'func': update_event_status_hold, @@ -1140,7 +1142,7 @@ def update_event_status_error(**args)->tuple: error_args = { 'error': { 'error_code': ErrorCode.E07, - 'error_details': f'Exception in update_event_status_error: {str(e)}', + 'error_details': e, 'event_id': event_id, 'event_type': event_type, 'func': update_event_status_error, @@ -1216,7 +1218,7 @@ def add_to_persistent_failed_queue(**args)->tuple: message = args.get('message') args['error'] = { 'error_code': ErrorCode.E03, - 'error_details': str(e), + 'error_details': e, 'event_id': message.get('event_id'), 'event_type': message.get('event_type'), 'func': add_to_persistent_failed_queue, diff --git a/python/prohibition_web_svc/middleware/event_middleware.py b/python/prohibition_web_svc/middleware/event_middleware.py index 43c59fd73..e2829cf91 100644 --- a/python/prohibition_web_svc/middleware/event_middleware.py +++ b/python/prohibition_web_svc/middleware/event_middleware.py @@ -248,7 +248,7 @@ def save_event_data(**kwargs) -> tuple: # Set error in kwargs to get consumed by the record_event_error function kwargs['error'] = { 'error_code': ErrorCode.E01, - 'error_details': str(e), + 'error_details': e, 'event_id': None, 'event_type': get_event_type(data), 'ticket_no': get_ticket_no(data), @@ -382,7 +382,7 @@ def save_event_pdf(**kwargs) -> tuple: # Set error in kwargs to get consumed by the record_event_error function kwargs['error'] = { 'error_code': ErrorCode.E02, - 'error_details': str(e), + 'error_details': e, 'event_id': event.event_id, 'event_type': get_event_type(data), 'ticket_no': get_ticket_no(data), diff --git a/python/prohibition_web_svc/middleware/form_middleware.py b/python/prohibition_web_svc/middleware/form_middleware.py index dbb95fca3..0eb8c497e 100644 --- a/python/prohibition_web_svc/middleware/form_middleware.py +++ b/python/prohibition_web_svc/middleware/form_middleware.py @@ -53,7 +53,7 @@ def lease_a_form_id(**kwargs) -> tuple: except Exception as e: kwargs['error'] = { 'error_code': ErrorCode.F01, - 'error_details': str(e), + 'error_details': e, 'event_type': kwargs.get('form_type'), 'func': lease_a_form_id, } @@ -92,7 +92,7 @@ def renew_form_id_lease(**kwargs) -> tuple: except Exception as e: kwargs['error'] = { 'error_code': ErrorCode.F02, - 'error_details': str(e), + 'error_details': e, 'event_type': kwargs.get('form_type'), 'func': renew_form_id_lease, } @@ -265,7 +265,7 @@ def admin_create_form(**kwargs) -> tuple: logging.warning(str(e)) kwargs['error'] = { 'error_code': ErrorCode.F02, - 'error_details': str(e), + 'error_details': e, 'event_type': kwargs.get('form_type'), 'func': renew_form_id_lease, } diff --git a/python/prohibition_web_svc/migrations/versions/211a0fba4348_.py b/python/prohibition_web_svc/migrations/versions/211a0fba4348_.py index 42a838784..2dcb0154c 100644 --- a/python/prohibition_web_svc/migrations/versions/211a0fba4348_.py +++ b/python/prohibition_web_svc/migrations/versions/211a0fba4348_.py @@ -28,7 +28,7 @@ def upgrade(): sa.Column('error_details', sa.Text(), nullable=True), sa.Column('error_path', sa.String(length=200), nullable=True), sa.Column('event_id', sa.Integer(), nullable=True), - sa.Column('event_type', sa.String(length=10), nullable=True), + sa.Column('event_type', sa.String(length=30), nullable=True), sa.Column('ticket_no', sa.String(length=50), nullable=True), sa.Column('received_dt', sa.DateTime(), nullable=True), sa.Column('error_status_cd', sa.String(length=200), nullable=True), From 33348bd411810702bb027e460f3f0bc80f44fa1f Mon Sep 17 00:00:00 2001 From: Adimar Borges Date: Thu, 26 Sep 2024 10:27:31 -0700 Subject: [PATCH 11/22] update for error logging --- python/common/ride_actions.py | 18 +++++++++----- python/form_handler/icbc_service.py | 37 ++++++++--------------------- 2 files changed, 22 insertions(+), 33 deletions(-) diff --git a/python/common/ride_actions.py b/python/common/ride_actions.py index b4fb092a3..55533f8be 100644 --- a/python/common/ride_actions.py +++ b/python/common/ride_actions.py @@ -45,7 +45,8 @@ def twelve_hours_event(**args): 'error_details': f'Error in sending 12hr_submitted event to RIDE, Response code: {response.status_code} response text: {response.json()}', 'event_type': '12hr', 'func': twelve_hours_event, - 'ticket_no': args['form_data']['twelve_hour_number'] + 'ticket_no': args['form_data']['twelve_hour_number'], + 'event_id': args['message']['event_id'] } logging.error('error in sending 12hr_submitted event to RIDE') logging.error(f'error code: {response.status_code} error message: {response.json()}') @@ -60,7 +61,8 @@ def twelve_hours_event(**args): 'error_details': e, 'event_type': '12hr', 'func': twelve_hours_event, - 'ticket_no': args['form_data']['twelve_hour_number'] + 'ticket_no': args['form_data']['twelve_hour_number'], + 'event_id': args['message']['event_id'] } return False, args @@ -108,7 +110,8 @@ def twenty_four_hours_event(**args): 'error_details': f'Error in sending 24hr_submitted event to RIDE, Response code: {response.status_code} response text: {response.json()}', 'event_type': '24hr', 'func': twenty_four_hours_event, - 'ticket_no': args['form_data']['twenty_four_hour_number'] + 'ticket_no': args['form_data']['twenty_four_hour_number'], + 'event_id': args['message']['event_id'] } logging.error('error in sending 24hr_submitted event to RIDE') logging.error(f'error code: {response.status_code} error message: {response.json()}') @@ -123,7 +126,8 @@ def twenty_four_hours_event(**args): 'error_details': e, 'event_type': '24hr', 'func': twenty_four_hours_event, - 'ticket_no': args['form_data']['twenty_four_hour_number'] + 'ticket_no': args['form_data']['twenty_four_hour_number'], + 'event_id': args['message']['event_id'] } return False, args @@ -181,7 +185,8 @@ def vi_event(**args): 'error_details': f'Error in sending vi_submitted event to RIDE, Response code: {response.status_code} response text: {response.json()}', 'event_type': 'VI', 'func': vi_event, - 'ticket_no': args['form_data']['VI_number'] + 'ticket_no': args['form_data']['VI_number'], + 'event_id': args['message']['event_id'] } logging.error('error in sending vi_submitted event to RIDE') logging.error(f'error code: {response.status_code} error message: {response.json()}') @@ -196,7 +201,8 @@ def vi_event(**args): 'error_details': e, 'event_type': 'VI', 'func': vi_event, - 'ticket_no': args['form_data']['VI_number'] + 'ticket_no': args['form_data']['VI_number'], + 'event_id': args['message']['event_id'] } return False, args diff --git a/python/form_handler/icbc_service.py b/python/form_handler/icbc_service.py index 4f336e980..522005950 100644 --- a/python/form_handler/icbc_service.py +++ b/python/form_handler/icbc_service.py @@ -1,34 +1,17 @@ import requests from requests.auth import HTTPBasicAuth from python.form_handler.config import Config -import time def submit_to_icbc(payload,logging) -> tuple: - # print("___ICBC__") - # url = "{}".format(Config.ICBC_API_ROOT) - # url=f'{Config.ICBC_API_ROOT}/vips/icbc/dfft/contravention' url=f'{Config.ICBC_API_SUBMIT_ROOT}/dfft/v1/contravention' - try: - # payload = kwargs['message']['icbc_submission'] - # # TODO remove for oc - # print("___Waiting for VPN") - # for i in range(0,30): - # print(i) - # time.sleep(1) - # print("_Sending to ICBC_") - # print(payload) - logging.info("_Sending to ICBC_") - icbc_response = requests.post(url, json=payload, timeout=60, auth=HTTPBasicAuth(Config.ICBC_API_SUBMIT_USERNAME, Config.ICBC_API_SUBMIT_PASSWORD)) - ##kwargs['response'] = make_response(icbc_response.text, icbc_response.status_code) - logging.info(icbc_response.text) - logging.info(icbc_response.status_code) - # print(icbc_response.text) - # print(icbc_response.status_code) - if(icbc_response.status_code!=200): - return False, icbc_response.text, icbc_response.status_code - except Exception as e: - # print("ERROR__in ICBC call_") - # print(e) - logging.error(e) - return False, None, None + + logging.info("_Sending to ICBC_") + icbc_response = requests.post(url, json=payload, timeout=60, auth=HTTPBasicAuth(Config.ICBC_API_SUBMIT_USERNAME, Config.ICBC_API_SUBMIT_PASSWORD)) + + logging.info(icbc_response.text) + logging.info(icbc_response.status_code) + + if(icbc_response.status_code!=200): + return False, icbc_response.text, icbc_response.status_code + return True, icbc_response.text, icbc_response.status_code From bcf9edfad28af12d8b123b6b87959c5f785b3c92 Mon Sep 17 00:00:00 2001 From: Ratheesh kumar R Date: Thu, 26 Sep 2024 11:36:02 -0700 Subject: [PATCH 12/22] DF-3085 : Fixing User table not showing admin --- .../userAdminDashboard/userAdminDashboard.js | 40 ++++++++++++------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/roadside-forms-frontend/frontend_web_app/src/components/userAdminDashboard/userAdminDashboard.js b/roadside-forms-frontend/frontend_web_app/src/components/userAdminDashboard/userAdminDashboard.js index d88339c48..36d38cc50 100644 --- a/roadside-forms-frontend/frontend_web_app/src/components/userAdminDashboard/userAdminDashboard.js +++ b/roadside-forms-frontend/frontend_web_app/src/components/userAdminDashboard/userAdminDashboard.js @@ -45,16 +45,39 @@ export const UserAdminDashboard = () => { } }, [userRoleData, navigate]); + const addUniqueIds = (users) => { + return users.map((user, index) => ({ + ...user, + uniqueId: `${user.user_guid}-${index}` + })); + }; + const getAllUsers = () => { UserApi.getAll().then((resp) => { - const uniqueUsers = removeDuplicates(resp.data); + const uniqueUsers = addUniqueIds(resp.data); + + // Group users by user_guid + const userGroups = uniqueUsers.reduce((groups, user) => { + if (!groups[user.user_guid]) { + groups[user.user_guid] = []; + } + groups[user.user_guid].push(user); + return groups; + }, {}); + + // Filter out users who have at least one administrator role + const filteredUsers = Object.values(userGroups) + .filter(group => !group.some(user => user.role_name === "administrator")) + .flatMap(group => group); + setSelectUsers( - uniqueUsers + filteredUsers .filter((user) => user.role_name === "officer") .map((user) => { return { label: user.login, value: user }; }) ); + setData(uniqueUsers); setLoading(false); }); @@ -113,17 +136,6 @@ export const UserAdminDashboard = () => { }; - const removeDuplicates = (users) => { - const uniqueUsers = {}; - return users.filter(user => { - if (!uniqueUsers[user.user_guid]) { - uniqueUsers[user.user_guid] = true; - return true; - } - return false; - }); - }; - const filteredData = useMemo(() => { if (showNewUsersOnly) { return data.filter(user => !user.approved_dt); @@ -254,7 +266,7 @@ export const UserAdminDashboard = () => { return ( Date: Thu, 26 Sep 2024 11:59:56 -0700 Subject: [PATCH 13/22] Adding the ticket no to get stored in error table --- python/form_handler/actions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/form_handler/actions.py b/python/form_handler/actions.py index a8cf52d79..5995f45b1 100644 --- a/python/form_handler/actions.py +++ b/python/form_handler/actions.py @@ -1360,6 +1360,7 @@ def record_event_error(**args): 'event_id': error.get('event_id'), 'event_type': error.get('event_type'), 'payload': json.dumps(message) if message else None, + 'ticket_no': error.get('ticket_no'), 'func': error.get('func'), } From 402d0270265b1b4de3c5893e41d6fa310c33710f Mon Sep 17 00:00:00 2001 From: Ratheesh kumar R Date: Wed, 2 Oct 2024 12:48:38 -0700 Subject: [PATCH 14/22] Adding Form Title --- python/prohibition_web_svc/middleware/form_middleware.py | 9 +++++++++ .../src/components/userAdminDashboard/formStatistics.js | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/python/prohibition_web_svc/middleware/form_middleware.py b/python/prohibition_web_svc/middleware/form_middleware.py index cd1b97cea..ba8237c2f 100644 --- a/python/prohibition_web_svc/middleware/form_middleware.py +++ b/python/prohibition_web_svc/middleware/form_middleware.py @@ -315,9 +315,18 @@ def get_form_statistics(**kwargs) -> tuple: else_=0 )).label('available_forms') ).group_by(Form.form_type).order_by(Form.form_type).all() + + form_names = { + '12Hour': '12 Hour Suspension (MV2906)', + '24Hour': '24 Hour Prohibition (MV2634E)', + 'VI': 'Vehicle Impoundment (MV2721 / MV2722)', + 'IRP': 'IRP (MV2723 / MV2724)', + } + stats = [ { + 'form_name': form_names.get(r.form_type, f"Form Type: {r.form_type}"), 'form_type': r.form_type, 'total_forms': r.total_forms, 'leased_forms': r.leased_forms, diff --git a/roadside-forms-frontend/frontend_web_app/src/components/userAdminDashboard/formStatistics.js b/roadside-forms-frontend/frontend_web_app/src/components/userAdminDashboard/formStatistics.js index 716add56c..b7fc152c1 100644 --- a/roadside-forms-frontend/frontend_web_app/src/components/userAdminDashboard/formStatistics.js +++ b/roadside-forms-frontend/frontend_web_app/src/components/userAdminDashboard/formStatistics.js @@ -48,7 +48,7 @@ export const FormStatistics = () => { {formData.map((form) => ( - {form.form_type} Forms + {form.form_name}
From b2dff0d8d659ffcb4c7cc1e320a236274278bf85 Mon Sep 17 00:00:00 2001 From: Adimar Borges <70654284+adimar-aot@users.noreply.github.com> Date: Thu, 3 Oct 2024 15:47:36 -0700 Subject: [PATCH 15/22] Send coordinates for event (#384) --- python/common/enums.py | 5 ++++ python/common/form_middleware.py | 16 +++++++++++ python/common/ride_actions.py | 18 ++++-------- python/form_handler/actions.py | 36 ++++++++++++++++++------ python/form_handler/business.py | 3 ++ python/form_handler/config.py | 5 ++-- python/form_handler/geocoding_service.py | 25 ++++++++++++++++ 7 files changed, 85 insertions(+), 23 deletions(-) create mode 100644 python/common/form_middleware.py create mode 100644 python/form_handler/geocoding_service.py diff --git a/python/common/enums.py b/python/common/enums.py index 9fd3e8318..03b0d9fcb 100644 --- a/python/common/enums.py +++ b/python/common/enums.py @@ -103,6 +103,11 @@ class ErrorCode(BaseEnum): "Contact DF application support for further investigation", False) + # Geocoding location related error + + L01 = ErrorCodeDetails("L01", "Error in getting coordinates", ErrorCategory.CONNECTION, ErrorSeverity.MEDIUM, + "Contact DF application support for further investigation", False) + # Add more error codes as needed... diff --git a/python/common/form_middleware.py b/python/common/form_middleware.py new file mode 100644 index 000000000..b55ad58d2 --- /dev/null +++ b/python/common/form_middleware.py @@ -0,0 +1,16 @@ + +import logging +from python.common.models import City + + +def get_city_name(city_code, args) -> str: + application = args.get('app') + db = args.get('db') + with application.app_context(): + city_data = db.session.query(City) \ + .filter(City.objectCd == city_code) \ + .all() + if len(city_data) == 0: + logging.error("city not found") + else: + return city_data[0].objectDsc \ No newline at end of file diff --git a/python/common/ride_actions.py b/python/common/ride_actions.py index 55533f8be..b363abbd0 100644 --- a/python/common/ride_actions.py +++ b/python/common/ride_actions.py @@ -1,6 +1,7 @@ import logging import logging.config import requests +from python.common import form_middleware from python.common.helper import date_time_to_local_tz_string, format_date_only, yes_no_string_to_bool from python.common.config import Config from python.common.enums import ErrorCode @@ -233,7 +234,10 @@ def fill_common_payload_record(args, payloadRecord): payloadRecord["timeReleased"] = args['event_data']['time_released'] payloadRecord["vehicleTypeDesc"] = get_vehicle_type(args) payloadRecord["addressOfOffence"] = args['event_data']['intersection_or_address_of_offence'] - payloadRecord["offenceCity"] = get_city_name(args['event_data']['offence_city'], args) + payloadRecord["offenceCity"] = form_middleware.get_city_name(args['event_data']['offence_city'], args) + if 'latitude' in args['event_data'] and 'longitude' in args['event_data']: + payloadRecord["latitude"] = args['event_data']['latitude'] + payloadRecord["longitude"] = args['event_data']['longitude'] payloadRecord["officerDisplayName"] = args['user_data']['display_name'] payloadRecord["officerBadgeNumber"] = args['user_data']['badge_number'] @@ -328,18 +332,6 @@ def get_vehicle_type(args) -> str: return vehicle_type_data[0].description return None - -def get_city_name(city_code, args) -> str: - application = args.get('app') - db = args.get('db') - with application.app_context(): - city_data = db.session.query(City) \ - .filter(City.objectCd == city_code) \ - .all() - if len(city_data) == 0: - logging.error("city not found") - else: - return city_data[0].objectDsc def get_impound_lot_operator(args) -> str: if args['event_data']['impound_lot_operator']: diff --git a/python/form_handler/actions.py b/python/form_handler/actions.py index 5995f45b1..95a8dcd70 100644 --- a/python/form_handler/actions.py +++ b/python/form_handler/actions.py @@ -1,23 +1,18 @@ import json -import csv -import pytz import logging import logging.config +from python.common import form_middleware from python.form_handler.config import Config -from cerberus import Validator -import base64 -from cerberus import errors import logging import json -from datetime import datetime from minio import Minio -from minio.error import S3Error from python.common.models import Event,FormStorageRefs,VIForm,TwentyFourHourForm,TwelveHourForm,IRPForm,User,AgencyCrossref,CityCrossRef,JurisdictionCrossRef,ImpoundReasonCodes,IloIdCrossRef,ImpoundLotOperator +from python.form_handler.geocoding_service import get_coordinates from python.form_handler.icbc_service import submit_to_icbc from python.form_handler.vips_service import create_vips_doc,create_vips_imp from python.form_handler.payloads import vips_payload,vips_document_payload from python.form_handler.message import encode_message -from python.form_handler.helper import method2_decrypt,decryptPdf_method1,convertDateTime,convertDateTimeWithSecs +from python.form_handler.helper import decryptPdf_method1,convertDateTime,convertDateTimeWithSecs from python.common.error_middleware import record_error from python.common.enums import ErrorCode @@ -1369,4 +1364,29 @@ def record_event_error(**args): except Exception as e: # If recording the error itself fails, log it logging.error(f"Failed to record error: {str(e)}") + return True, args + + +def get_event_coordinates(**args)->tuple: + logging.debug("inside get_event_coordinates()") + try: + address = args['event_data']['intersection_or_address_of_offence'] + city = form_middleware.get_city_name(args['event_data']['offence_city'], args) + + geocoding_status, latitude, longitude = get_coordinates(address, city) + args['event_data']['latitude'] = latitude + args['event_data']['longitude'] = longitude + + except Exception as e: + logging.error(f'Error getting coordinates: {e}') + args['error'] = { + 'error_code': ErrorCode.L01, + 'error_details': e, + 'event_type': args['message']['event_type'], + 'func': get_event_coordinates, + 'event_id': args['message']['event_id'] + } + record_event_error(**args) + + # Always return True to continue processing even if not able to get coordinates return True, args \ No newline at end of file diff --git a/python/form_handler/business.py b/python/form_handler/business.py index 53e40be81..35c52835d 100644 --- a/python/form_handler/business.py +++ b/python/form_handler/business.py @@ -95,6 +95,7 @@ def process_incoming_form() -> dict: # {"try": actions.add_to_retry_queue, "fail": []}, # {"try": actions.update_event_status_hold, "fail": []}, # ]}, + {"try": actions.get_event_coordinates, "fail": []}, {"try": ride_actions.vi_event, "fail": [ {"try": actions.record_event_error, "fail": []}, ]}, @@ -140,6 +141,7 @@ def process_incoming_form() -> dict: {"try": actions.record_event_error, "fail": []}, {"try": actions.update_event_status_hold, "fail": []}, ]}, + {"try": actions.get_event_coordinates, "fail": []}, {"try": ride_actions.twenty_four_hours_event, "fail": [ {"try": actions.record_event_error, "fail": []}, ]}, @@ -187,6 +189,7 @@ def process_incoming_form() -> dict: {"try": actions.add_to_retry_queue, "fail": []}, {"try": actions.update_event_status_hold, "fail": []}, ]}, + {"try": actions.get_event_coordinates, "fail": []}, {"try": ride_actions.twelve_hours_event, "fail": [ {"try": actions.record_event_error, "fail": []}, ]}, diff --git a/python/form_handler/config.py b/python/form_handler/config.py index bbfcf9c63..1189321d1 100644 --- a/python/form_handler/config.py +++ b/python/form_handler/config.py @@ -72,8 +72,9 @@ class Config(): VIPS_DPS_EMAIL = os.getenv('VIPS_DPS_EMAIL', 'do-not-reply-rsi@gov.bc.ca') - - + # Geocoding service details + GEOCODING_API_URL = os.getenv('GEOCODING_API_URL', 'http://localhost:8000') + GEOCODING_API_KEY = os.getenv('GEOCODING_API_KEY', 'TEST') LOGGING = { 'version': 1, diff --git a/python/form_handler/geocoding_service.py b/python/form_handler/geocoding_service.py new file mode 100644 index 000000000..99eaa33b3 --- /dev/null +++ b/python/form_handler/geocoding_service.py @@ -0,0 +1,25 @@ +import logging +import requests +from python.form_handler.config import Config + +geocoding_url = Config.GEOCODING_API_URL +geocoding_key = Config.GEOCODING_API_KEY + + +def get_coordinates(address: str, city: str)-> tuple: + logging.debug(f'Getting coordinates for address: {address} and city: {city}') + + url = f'{geocoding_url}/coordinates?address={address}&city={city}' + logging.debug(f'Geocoding API URL: {url}') + response = requests.get(url, headers={'api-key': geocoding_key}) + if response.status_code == 200: + get_coordinates_response = response.json() + if get_coordinates_response['province'] != 'BC': + logging.warning(f'Coordinates found for address: {address} and city: {city} but not in BC') + return False, None, None + return True, get_coordinates_response['latitude'], get_coordinates_response['longitude'] + elif response.status_code == 404: + logging.warning(f'No coordinates found for address: {address} and city: {city}') + return False, None, None + else: + raise Exception(f'Error getting coordinates for address: {address} and city: {city} -> Status: {response.status_code} Response: {response.text}') From c9d35b77e5feff8846ef4b9a77278cffa5427a07 Mon Sep 17 00:00:00 2001 From: Adimar Borges Date: Fri, 4 Oct 2024 09:49:12 -0700 Subject: [PATCH 16/22] update App version --- roadside-forms-frontend/frontend_web_app/package-lock.json | 4 ++-- roadside-forms-frontend/frontend_web_app/package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/roadside-forms-frontend/frontend_web_app/package-lock.json b/roadside-forms-frontend/frontend_web_app/package-lock.json index 399a132e8..8bfe965ac 100644 --- a/roadside-forms-frontend/frontend_web_app/package-lock.json +++ b/roadside-forms-frontend/frontend_web_app/package-lock.json @@ -1,12 +1,12 @@ { "name": "frontend_web_app", - "version": "2.2.0", + "version": "2.3.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "frontend_web_app", - "version": "2.2.0", + "version": "2.3.0", "dependencies": { "@react-keycloak/web": "^3.4.0", "@reduxjs/toolkit": "^2.0.1", diff --git a/roadside-forms-frontend/frontend_web_app/package.json b/roadside-forms-frontend/frontend_web_app/package.json index 360ceeef6..a5c4487c4 100644 --- a/roadside-forms-frontend/frontend_web_app/package.json +++ b/roadside-forms-frontend/frontend_web_app/package.json @@ -1,6 +1,6 @@ { "name": "frontend_web_app", - "version": "2.2.0", + "version": "2.3.0", "homepage": "/roadside-forms", "private": true, "dependencies": { From c48c31742f4f381e6590c9835df947aab3de98af Mon Sep 17 00:00:00 2001 From: Ratheesh kumar R Date: Fri, 11 Oct 2024 15:08:17 -0700 Subject: [PATCH 17/22] Adding Export to CSV function --- .../frontend_web_app/package-lock.json | 32 ++++ .../frontend_web_app/package.json | 1 + .../userAdminDashboard/userAdminDashboard.js | 180 +++++++++++------- 3 files changed, 147 insertions(+), 66 deletions(-) diff --git a/roadside-forms-frontend/frontend_web_app/package-lock.json b/roadside-forms-frontend/frontend_web_app/package-lock.json index 8bfe965ac..890c2e5d7 100644 --- a/roadside-forms-frontend/frontend_web_app/package-lock.json +++ b/roadside-forms-frontend/frontend_web_app/package-lock.json @@ -31,6 +31,7 @@ "react-bootstrap-table-next": "^4.0.3", "react-bootstrap-table2-filter": "^1.3.3", "react-bootstrap-table2-paginator": "^2.1.2", + "react-bootstrap-table2-toolkit": "^2.1.3", "react-dom": "^18.3.1", "react-redux": "^9.1.2", "react-router-dom": "6.19.0", @@ -16557,6 +16558,11 @@ "node": ">= 10.13.0" } }, + "node_modules/file-saver": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/file-saver/-/file-saver-2.0.2.tgz", + "integrity": "sha512-Wz3c3XQ5xroCxd1G8b7yL0Ehkf0TC9oYC6buPFkNnU9EnaPlifeAFCyCh+iewXTyFRcg0a6j3J7FmJsIhlhBdw==" + }, "node_modules/file-system-cache": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/file-system-cache/-/file-system-cache-2.3.0.tgz", @@ -25697,6 +25703,19 @@ "react-dom": "^16.3.0" } }, + "node_modules/react-bootstrap-table2-toolkit": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/react-bootstrap-table2-toolkit/-/react-bootstrap-table2-toolkit-2.1.3.tgz", + "integrity": "sha512-nKBSezHTOkO9k8YMMuJfPEZtBVfIYrJbmP8n3u7+AXRcOrOGygXyauNVKWqdKLchQlG/cW5QR0sPkFknpp5rjQ==", + "dependencies": { + "file-saver": "2.0.2" + }, + "peerDependencies": { + "prop-types": "^15.0.0", + "react": "^16.3.0", + "react-dom": "^16.3.0" + } + }, "node_modules/react-colorful": { "version": "5.6.1", "resolved": "https://registry.npmjs.org/react-colorful/-/react-colorful-5.6.1.tgz", @@ -45143,6 +45162,11 @@ } } }, + "file-saver": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/file-saver/-/file-saver-2.0.2.tgz", + "integrity": "sha512-Wz3c3XQ5xroCxd1G8b7yL0Ehkf0TC9oYC6buPFkNnU9EnaPlifeAFCyCh+iewXTyFRcg0a6j3J7FmJsIhlhBdw==" + }, "file-system-cache": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/file-system-cache/-/file-system-cache-2.3.0.tgz", @@ -52650,6 +52674,14 @@ "resolved": "https://registry.npmjs.org/react-bootstrap-table2-paginator/-/react-bootstrap-table2-paginator-2.1.2.tgz", "integrity": "sha512-LC5znEphhgKJvaSY1q8d+Gj0Nc/1X+VS3tKJjkmWmfv9P61YC/BnwJ+aoqEmQzsLiVGowrzss+i/u+Tip5H+Iw==" }, + "react-bootstrap-table2-toolkit": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/react-bootstrap-table2-toolkit/-/react-bootstrap-table2-toolkit-2.1.3.tgz", + "integrity": "sha512-nKBSezHTOkO9k8YMMuJfPEZtBVfIYrJbmP8n3u7+AXRcOrOGygXyauNVKWqdKLchQlG/cW5QR0sPkFknpp5rjQ==", + "requires": { + "file-saver": "2.0.2" + } + }, "react-colorful": { "version": "5.6.1", "resolved": "https://registry.npmjs.org/react-colorful/-/react-colorful-5.6.1.tgz", diff --git a/roadside-forms-frontend/frontend_web_app/package.json b/roadside-forms-frontend/frontend_web_app/package.json index a5c4487c4..5f44c8a66 100644 --- a/roadside-forms-frontend/frontend_web_app/package.json +++ b/roadside-forms-frontend/frontend_web_app/package.json @@ -27,6 +27,7 @@ "react-bootstrap-table-next": "^4.0.3", "react-bootstrap-table2-filter": "^1.3.3", "react-bootstrap-table2-paginator": "^2.1.2", + "react-bootstrap-table2-toolkit": "^2.1.3", "react-dom": "^18.3.1", "react-redux": "^9.1.2", "react-router-dom": "6.19.0", diff --git a/roadside-forms-frontend/frontend_web_app/src/components/userAdminDashboard/userAdminDashboard.js b/roadside-forms-frontend/frontend_web_app/src/components/userAdminDashboard/userAdminDashboard.js index 36d38cc50..663589c37 100644 --- a/roadside-forms-frontend/frontend_web_app/src/components/userAdminDashboard/userAdminDashboard.js +++ b/roadside-forms-frontend/frontend_web_app/src/components/userAdminDashboard/userAdminDashboard.js @@ -4,6 +4,7 @@ import { useNavigate } from "react-router-dom"; import BootstrapTable from 'react-bootstrap-table-next'; import filterFactory, { textFilter } from 'react-bootstrap-table2-filter'; import paginationFactory from 'react-bootstrap-table2-paginator'; +import ToolkitProvider, { CSVExport } from 'react-bootstrap-table2-toolkit/dist/react-bootstrap-table2-toolkit'; import Button from "react-bootstrap/Button"; import Row from "react-bootstrap/Row"; import Col from "react-bootstrap/Col"; @@ -19,6 +20,8 @@ import { UserApi } from "../../api/userApi"; import { SearchableSelect } from "../common/Select/SearchableSelect"; import "./userAdminDashboard.scss"; +const { ExportCSVButton } = CSVExport; + export const UserAdminDashboard = () => { const initialValues = { user: {}, @@ -135,7 +138,6 @@ export const UserAdminDashboard = () => { setTimeout(() => setShowSuccessMessage(false), 10000); // Hide after 10 seconds }; - const filteredData = useMemo(() => { if (showNewUsersOnly) { return data.filter(user => !user.approved_dt); @@ -216,6 +218,12 @@ export const UserAdminDashboard = () => { text: 'Date Applied', sort: true, formatter: (cell) => moment(cell).tz("America/Vancouver").format("YYYY-MM-DD HH:mm"), + csvFormatter: (cell) => { + if (cell === null || cell === undefined) { + return ''; + } + return moment(cell).tz("America/Vancouver").format("YYYY-MM-DD HH:mm"); + }, }, { dataField: 'last_active', @@ -227,10 +235,17 @@ export const UserAdminDashboard = () => { } return moment(cell).tz("America/Vancouver").format("YYYY-MM-DD HH:mm"); }, + csvFormatter: (cell) => { + if (cell === null || cell === undefined) { + return ''; + } + return moment(cell).tz("America/Vancouver").format("YYYY-MM-DD HH:mm"); + }, }, { dataField: 'action', text: 'Action', + csvExport: false, // This line excludes the column from CSV export formatter: (cellContent, row) => ( row.approved_dt ? ( + ); + }; + const renderTable = () => { if (filteredData.length === 0) { return ( @@ -263,18 +294,45 @@ export const UserAdminDashboard = () => {
); } + const currentDateTime = moment().format('YYYY-MM-DD_HH-mm-ss'); return ( - + exportCSV={{ + fileName: `DF_User_List_${currentDateTime}.csv`, + noAutoBOM: false, + exportAll: false, + onlyExportFiltered: true, + }} + > + { + props => ( +
+
+ setShowNewUsersOnly(e.target.checked)} + /> + +
+ +
+ ) + } + ); }; @@ -283,53 +341,44 @@ export const UserAdminDashboard = () => { } else { return ( <> - {showSuccessMessage && ( + {showSuccessMessage && ( setShowSuccessMessage(false)} dismissible> {successMessage} )}
-
- setShowNewUsersOnly(e.target.checked)} - /> -
{renderTable()} { - setSelectedUser(values.user.value); - setShowAddAdminModal(true); - }}> - {({ isSubmitting, values }) => ( - - - - - - - - - - - - - )} - + setSelectedUser(values.user.value); + setShowAddAdminModal(true); + }}> + {({ isSubmitting, values }) => ( + + + + + + + + + + + + + )} +
setShowConfirmModal(false)}> @@ -352,25 +401,24 @@ export const UserAdminDashboard = () => { - {/* Confirmation modal for adding administrator */} setShowAddAdminModal(false)}> - - Confirm Action - - - Are you sure you want to grant administrator role to {selectedUser?.first_name} {selectedUser?.last_name}? - - - - - - + + Confirm Action + + + Are you sure you want to grant administrator role to {selectedUser?.first_name} {selectedUser?.last_name}? + + + + + + ); } From f4e8f2841f486daee29e64dc6de451b3a9ae6b87 Mon Sep 17 00:00:00 2001 From: Ratheesh kumar R Date: Fri, 11 Oct 2024 15:08:56 -0700 Subject: [PATCH 18/22] Fixing form Lease function --- .../middleware/form_middleware.py | 29 ++++++++++++------- .../src/components/Dashboard/Dashboard.js | 6 ++-- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/python/prohibition_web_svc/middleware/form_middleware.py b/python/prohibition_web_svc/middleware/form_middleware.py index ba8237c2f..8df9f6c72 100644 --- a/python/prohibition_web_svc/middleware/form_middleware.py +++ b/python/prohibition_web_svc/middleware/form_middleware.py @@ -25,18 +25,21 @@ def log_payload_to_splunk(**kwargs) -> tuple: def lease_a_form_id(**kwargs) -> tuple: + logging.debug('inside lease_a_form_id()') + data = kwargs.get('payload') + user_guid = kwargs.get('user_guid') + id_list = [] + id_not_available = False try: - logging.debug('inside lease_a_form_id()') - data = kwargs.get('payload') - user_guid = kwargs.get('user_guid') - id_list = [] for form_type in data: ids = db.session.query(Form) \ .filter(Form.form_type == form_type) \ .filter(Form.user_guid == None) \ - .limit(data.get(form_type)) + .limit(data.get(form_type)) \ + .all() - if ids is None: + if not ids: + id_not_available = True logging.warning('Insufficient unique ids available for {}'.format(form_type)) record_error( **{ @@ -46,22 +49,26 @@ def lease_a_form_id(**kwargs) -> tuple: 'func': lease_a_form_id, } ) - return False, kwargs + for id in ids: logging.debug(f'id: {id}') id.lease(user_guid) id_list.append(asdict(id)) + db.session.commit() except Exception as e: + id_not_available = True kwargs['error'] = { 'error_code': ErrorCode.F01, 'error_details': e, 'event_type': kwargs.get('form_type'), 'func': lease_a_form_id, } - return False, kwargs - kwargs['response_dict'] = jsonify({'forms':id_list}) - return True, kwargs + + kwargs['response_dict'] = jsonify({'forms': id_list}) + is_successful = bool(id_list) and not id_not_available + return is_successful, kwargs + def renew_form_id_lease(**kwargs) -> tuple: @@ -359,7 +366,7 @@ def record_form_error(**kwargs): if error is None: logging.warning("Error object is None") - return + return True, kwargs record_error(**error) diff --git a/roadside-forms-frontend/frontend_web_app/src/components/Dashboard/Dashboard.js b/roadside-forms-frontend/frontend_web_app/src/components/Dashboard/Dashboard.js index 61331265f..d4bd948ab 100644 --- a/roadside-forms-frontend/frontend_web_app/src/components/Dashboard/Dashboard.js +++ b/roadside-forms-frontend/frontend_web_app/src/components/Dashboard/Dashboard.js @@ -284,8 +284,10 @@ export const Dashboard = () => { const fetchNeededIDs = async () => { const neededFormID = await getAllFormIDs(); const newIDs = await FormIDApi.post(neededFormID); - const seededIDs = await seedLeasedValues(newIDs.forms); - await db.formID.bulkPut(seededIDs); + if (newIDs) { + const seededIDs = await seedLeasedValues(newIDs.forms); + await db.formID.bulkPut(seededIDs); + } setFormIDsLoaded(true); }; From 0e79373a92408b82e45e50f0d6b3b4a2f9695b5c Mon Sep 17 00:00:00 2001 From: Adimar Borges Date: Fri, 11 Oct 2024 16:32:15 -0700 Subject: [PATCH 19/22] update location obj sent to RIDE --- python/common/ride_actions.py | 28 ++++++++++++++---------- python/form_handler/actions.py | 4 +++- python/form_handler/geocoding_service.py | 2 +- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/python/common/ride_actions.py b/python/common/ride_actions.py index b363abbd0..3a9d98b9b 100644 --- a/python/common/ride_actions.py +++ b/python/common/ride_actions.py @@ -26,13 +26,13 @@ def twelve_hours_event(**args): eventPayload = {} payloadRecord = {} eventPayload['typeofevent'] = twelve_hours_submitted - eventPayload['twelveHoursPayload'] = [] payloadRecord["eventType"] = twelve_hours_submitted payloadRecord["twelveHourNumber"] = args['form_data']['twelve_hour_number'] payloadRecord["typeOfProhibition"] = args['event_data']['type_of_prohibition'] fill_common_payload_record(args, payloadRecord) - eventPayload['twelveHoursPayload'].append(payloadRecord) + eventPayload['twelveHoursPayload'] = payloadRecord + fill_location(args, eventPayload) endpoint = f"{ride_url}/dfV2events/12hrsubmitted" headers = {'ride-api-key': ride_key} @@ -79,7 +79,6 @@ def twenty_four_hours_event(**args): eventPayload = {} payloadRecord = {} eventPayload['typeofevent'] = twenty_four_hours_submitted - eventPayload['twentyFourHoursPayload'] = [] payloadRecord["eventType"] = twenty_four_hours_submitted payloadRecord["twentyFourHrNo"] = args['form_data']['twenty_four_hour_number'] @@ -96,8 +95,8 @@ def twenty_four_hours_event(**args): payloadRecord["requestedApprovedInstrumentUsed"] = args['form_data']['requested_approved_instrument_used'] payloadRecord["requestedTestUsedAlcohol"] = args['form_data']['requested_test_used_alcohol'] payloadRecord["requestedTestUsedDrug"] = args['form_data']['requested_test_used_drug'] - - eventPayload['twentyFourHoursPayload'].append(payloadRecord) + eventPayload['twentyFourHoursPayload'] = payloadRecord + fill_location(args, eventPayload) endpoint = f"{ride_url}/dfV2events/24hrsubmitted" headers = {'ride-api-key': ride_key} @@ -145,8 +144,7 @@ def vi_event(**args): eventPayload = {} payloadRecord = {} eventPayload['typeofevent'] = vi_submitted - eventPayload['viPayload'] = [] - + payloadRecord["eventType"] = vi_submitted payloadRecord["viNumber"] = args['form_data']['VI_number'] fill_common_payload_record(args, payloadRecord) @@ -172,7 +170,8 @@ def vi_event(**args): payloadRecord["vehicleSpeed"] = args['form_data']['vehicle_speed'] payloadRecord["speedEstimationTechnique"] = args['form_data']['speed_estimation_technique'] payloadRecord["speedConfirmationTechnique"] = args['form_data']['speed_confirmation_technique'] - eventPayload['viPayload'].append(payloadRecord) + eventPayload['viPayload'] = payloadRecord + fill_location(args, eventPayload) endpoint = f"{ride_url}/dfV2events/visubmitted" headers = {'ride-api-key': ride_key} @@ -209,8 +208,17 @@ def vi_event(**args): return True, args +def fill_location(args, eventPayload): + if 'latitude' in args['event_data'] and 'longitude' in args['event_data']: + eventPayload["locationRequestPayload"] = {} + eventPayload["locationRequestPayload"]["latitude"] = args['event_data']['latitude'] + eventPayload["locationRequestPayload"]["longitude"] = args['event_data']['longitude'] + eventPayload["locationRequestPayload"]["requestedAddress"] = args['event_data']['requested_address'] + eventPayload["locationRequestPayload"]["fullAddress"] = args['event_data']['full_address'] + def fill_common_payload_record(args, payloadRecord): + payloadRecord["eventID"] = args['message']['event_id'] payloadRecord["eventVersion"] = 1.0 payloadRecord["eventDtm"] = date_time_to_local_tz_string(args['event_data']['created_dt']) payloadRecord["driverLicenceNumber"] = args['event_data']['driver_licence_no'] @@ -235,10 +243,6 @@ def fill_common_payload_record(args, payloadRecord): payloadRecord["vehicleTypeDesc"] = get_vehicle_type(args) payloadRecord["addressOfOffence"] = args['event_data']['intersection_or_address_of_offence'] payloadRecord["offenceCity"] = form_middleware.get_city_name(args['event_data']['offence_city'], args) - if 'latitude' in args['event_data'] and 'longitude' in args['event_data']: - payloadRecord["latitude"] = args['event_data']['latitude'] - payloadRecord["longitude"] = args['event_data']['longitude'] - payloadRecord["officerDisplayName"] = args['user_data']['display_name'] payloadRecord["officerBadgeNumber"] = args['user_data']['badge_number'] payloadRecord["enforcementAgencyName"] = args['user_data']['agency'] diff --git a/python/form_handler/actions.py b/python/form_handler/actions.py index 95a8dcd70..9f5d84bee 100644 --- a/python/form_handler/actions.py +++ b/python/form_handler/actions.py @@ -1373,9 +1373,11 @@ def get_event_coordinates(**args)->tuple: address = args['event_data']['intersection_or_address_of_offence'] city = form_middleware.get_city_name(args['event_data']['offence_city'], args) - geocoding_status, latitude, longitude = get_coordinates(address, city) + geocoding_status, latitude, longitude, full_address = get_coordinates(address, city) args['event_data']['latitude'] = latitude args['event_data']['longitude'] = longitude + args['event_data']['full_address'] = full_address + args['event_data']['requested_address'] = f'{address}, {city}' except Exception as e: logging.error(f'Error getting coordinates: {e}') diff --git a/python/form_handler/geocoding_service.py b/python/form_handler/geocoding_service.py index 99eaa33b3..1ce751529 100644 --- a/python/form_handler/geocoding_service.py +++ b/python/form_handler/geocoding_service.py @@ -17,7 +17,7 @@ def get_coordinates(address: str, city: str)-> tuple: if get_coordinates_response['province'] != 'BC': logging.warning(f'Coordinates found for address: {address} and city: {city} but not in BC') return False, None, None - return True, get_coordinates_response['latitude'], get_coordinates_response['longitude'] + return True, get_coordinates_response['latitude'], get_coordinates_response['longitude'], get_coordinates_response['address'] elif response.status_code == 404: logging.warning(f'No coordinates found for address: {address} and city: {city}') return False, None, None From 118d3f786e1464d1d3eaacfe04f56839843e4662 Mon Sep 17 00:00:00 2001 From: Ratheesh kumar R Date: Tue, 15 Oct 2024 14:09:30 -0700 Subject: [PATCH 20/22] Fix - Export CSV --- .../userAdminDashboard/userAdminDashboard.js | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/roadside-forms-frontend/frontend_web_app/src/components/userAdminDashboard/userAdminDashboard.js b/roadside-forms-frontend/frontend_web_app/src/components/userAdminDashboard/userAdminDashboard.js index 663589c37..39d4591d7 100644 --- a/roadside-forms-frontend/frontend_web_app/src/components/userAdminDashboard/userAdminDashboard.js +++ b/roadside-forms-frontend/frontend_web_app/src/components/userAdminDashboard/userAdminDashboard.js @@ -20,7 +20,6 @@ import { UserApi } from "../../api/userApi"; import { SearchableSelect } from "../common/Select/SearchableSelect"; import "./userAdminDashboard.scss"; -const { ExportCSVButton } = CSVExport; export const UserAdminDashboard = () => { const initialValues = { @@ -145,6 +144,10 @@ export const UserAdminDashboard = () => { return data; }, [data, showNewUsersOnly]); + const disableFilter = () => { + setShowNewUsersOnly(false); + }; + const paginationOptions = { sizePerPageList: [ { text: '10', value: 10 }, @@ -287,7 +290,18 @@ export const UserAdminDashboard = () => { return (
{showNewUsersOnly ? ( -

No pending user requests found. Please disable the filter to view all users.

+ <> +

No pending user requests found. Please click 'Show All Users' to view all users.

+ + + ) : (

No users found matching your search criteria. Please try adjusting your search parameters.

)} From ddb5300909cd96857d9af96704fed3a2305cbb70 Mon Sep 17 00:00:00 2001 From: Adimar Borges Date: Wed, 16 Oct 2024 11:32:25 -0700 Subject: [PATCH 21/22] fix fetch form IDs process --- .../middleware/form_middleware.py | 48 ++++++++++--------- .../src/components/Dashboard/Dashboard.js | 10 ++-- 2 files changed, 31 insertions(+), 27 deletions(-) diff --git a/python/prohibition_web_svc/middleware/form_middleware.py b/python/prohibition_web_svc/middleware/form_middleware.py index 8df9f6c72..835a1da64 100644 --- a/python/prohibition_web_svc/middleware/form_middleware.py +++ b/python/prohibition_web_svc/middleware/form_middleware.py @@ -32,28 +32,30 @@ def lease_a_form_id(**kwargs) -> tuple: id_not_available = False try: for form_type in data: - ids = db.session.query(Form) \ - .filter(Form.form_type == form_type) \ - .filter(Form.user_guid == None) \ - .limit(data.get(form_type)) \ - .all() - - if not ids: - id_not_available = True - logging.warning('Insufficient unique ids available for {}'.format(form_type)) - record_error( - **{ - 'error_code': ErrorCode.F01, - 'error_details': f'Insufficient unique ids available for {form_type}', - 'event_type': form_type, - 'func': lease_a_form_id, - } - ) - - for id in ids: - logging.debug(f'id: {id}') - id.lease(user_guid) - id_list.append(asdict(id)) + form_type_count = data.get(form_type) + if form_type_count > 0: + ids = db.session.query(Form) \ + .filter(Form.form_type == form_type) \ + .filter(Form.user_guid == None) \ + .limit(data.get(form_type)) \ + .all() + + if not ids: + id_not_available = True + logging.warning('Insufficient unique ids available for {}'.format(form_type)) + record_error( + **{ + 'error_code': ErrorCode.F01, + 'error_details': f'Insufficient unique ids available for {form_type}', + 'event_type': form_type, + 'func': lease_a_form_id, + } + ) + + for id in ids: + logging.debug(f'id: {id}') + id.lease(user_guid) + id_list.append(asdict(id)) db.session.commit() except Exception as e: @@ -66,7 +68,7 @@ def lease_a_form_id(**kwargs) -> tuple: } kwargs['response_dict'] = jsonify({'forms': id_list}) - is_successful = bool(id_list) and not id_not_available + is_successful = not id_not_available return is_successful, kwargs diff --git a/roadside-forms-frontend/frontend_web_app/src/components/Dashboard/Dashboard.js b/roadside-forms-frontend/frontend_web_app/src/components/Dashboard/Dashboard.js index d4bd948ab..00b3544bb 100644 --- a/roadside-forms-frontend/frontend_web_app/src/components/Dashboard/Dashboard.js +++ b/roadside-forms-frontend/frontend_web_app/src/components/Dashboard/Dashboard.js @@ -283,10 +283,12 @@ export const Dashboard = () => { const fetchNeededIDs = async () => { const neededFormID = await getAllFormIDs(); - const newIDs = await FormIDApi.post(neededFormID); - if (newIDs) { - const seededIDs = await seedLeasedValues(newIDs.forms); - await db.formID.bulkPut(seededIDs); + if (neededFormID["12Hour"] > 0 || neededFormID["24Hour"] > 0 || neededFormID["VI"] > 0) { + const newIDs = await FormIDApi.post(neededFormID); + if (newIDs) { + const seededIDs = await seedLeasedValues(newIDs.forms); + await db.formID.bulkPut(seededIDs); + } } setFormIDsLoaded(true); }; From 60e489bd6478c869e568e209be8be946dcb8082a Mon Sep 17 00:00:00 2001 From: Adimar Borges Date: Wed, 16 Oct 2024 11:42:39 -0700 Subject: [PATCH 22/22] Update form_middleware.py --- python/prohibition_web_svc/middleware/form_middleware.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/prohibition_web_svc/middleware/form_middleware.py b/python/prohibition_web_svc/middleware/form_middleware.py index 835a1da64..83a8c7e2a 100644 --- a/python/prohibition_web_svc/middleware/form_middleware.py +++ b/python/prohibition_web_svc/middleware/form_middleware.py @@ -37,7 +37,7 @@ def lease_a_form_id(**kwargs) -> tuple: ids = db.session.query(Form) \ .filter(Form.form_type == form_type) \ .filter(Form.user_guid == None) \ - .limit(data.get(form_type)) \ + .limit(form_type_count) \ .all() if not ids: