Placeholder pull request for project-wide code review

.ebextensions/00_ami.config

@@ -1,20 +1,11 @@
 #
-# AWS configuration for Hartsfield
+# AWS configuration for Squiggy
 #
-
 packages:
   yum:
-    amazon-linux-extras: []
-    awslogs: []
     gcc-c++: []
     git: []
-    mod_ssl: []
 
-commands:
-  01_postgres_activate:
-    command: sudo amazon-linux-extras enable postgresql14
-  02_postgres_install:
-    command: sudo yum install -y postgresql
 
 option_settings:
   aws:elasticbeanstalk:cloudwatch:logs:
@@ -24,25 +15,28 @@ option_settings:
   aws:elasticbeanstalk:environment:proxy:
     ProxyServer: apache
   aws:elasticbeanstalk:environment:proxy:staticfiles:
-    /static: dist/static
+    /assets: dist/static/assets
+    /favicon.ico: dist/static/favicon.ico
 
 files:
-  /etc/awslogs/awscli.conf:
-    mode: '000600'
-    owner: root
-    group: root
-    content: |
-      [plugins]
-      cwlogs = cwlogs
-      [default]
-      region = `{"Ref":"AWS::Region"}`
-
-  /etc/awslogs/config/logs.conf:
+  /opt/aws/amazon-cloudwatch-agent/bin/config.json:
     mode: '000644'
     owner: root
     group: root
     content: |
-      [/var/app/current/hartsfield.log]
-      log_group_name=`{"Fn::Join":["/", ["/aws/elasticbeanstalk", { "Ref":"AWSEBEnvironmentName" }, "var/app/current/damien.log"]]}`
-      log_stream_name={instance_id}
-      file=/var/app/current/hartsfield.log*
+      {
+        "logs": {
+          "logs_collected": {
+            "files": {
+              "collect_list": [
+                {
+                  "file_path": "/var/app/current/heartsfield.log*",
+                  "log_group_name": "`{"Fn::Join":["/", ["/aws/elasticbeanstalk", "var/app/current/heartsfield.log"]]}`",
+                  "log_stream_name": "{instance_id}"
+                }
+              ]
+            }
+          }
+        }
+      }
+

.env

@@ -1,7 +1,7 @@
 ####
 # Vue.js environment variables and modes: https://cli.vuejs.org/guide/mode-and-env.html
-# Only variables that start with 'VUE_APP_' will be statically embedded into the client bundle.
+# Only variables that start with 'VITE_APP_' will be statically embedded into the client bundle.
 ####
 
-VUE_APP_API_BASE_URL=''
-VUE_APP_DEBUG=false
+VITE_APP_API_BASE_URL=''
+VITE_APP_DEBUG=false

.env.development

@@ -1,7 +1,7 @@
 ####
 # Vue.js environment variables and modes: https://cli.vuejs.org/guide/mode-and-env.html
-# Only variables that start with 'VUE_APP_' will be statically embedded into the client bundle.
+# Only variables that start with 'VITE_APP_' will be statically embedded into the client bundle.
 ####
 
-VUE_APP_API_BASE_URL='http://localhost:5000'
-VUE_APP_DEBUG=true
+VITE_APP_API_BASE_URL='http://localhost:5000'
+VITE_APP_DEBUG=true

.platform/hooks/postdeploy/00_update_apache_config.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+sudo mv /tmp/hartsfield.conf /etc/httpd/conf.d/hartsfield.conf
+sudo mv /tmp/ssl.conf /etc/httpd/conf.d/ssl.conf
+sudo /bin/systemctl restart httpd.service

.platform/hooks/postdeploy/01_start_awslogsd.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -c file:/opt/aws/amazon-cloudwatch-agent/bin/config.json -s

README.md

@@ -4,39 +4,58 @@ Hartsfield humbly supports UC Berkeley's DataHub.
 
 ![Hartsfield, re-imagined as a field of hearts.](src/assets/hEartsfield.png)
 
-## Installation
 
-* Install Python 3.8
-* Create your virtual environment (venv)
-* Install dependencies
+
+
+## To run this locally:
+
+### Install a python 3.11 venv in the project directory, activate it, and install requirements:
 
 ```
-pip3 install -r requirements.txt [--upgrade]
+python3.11 -m venv ./venv
+source ./venv/bin/activate
+pip install -r requirements.txt
 ```
 
+### Install npm (the Node.js package manager), adjust the version, install the project dependencies, and do the "audit fix" if it complains that you should do so:
 
+https://docs.npmjs.com/downloading-and-installing-node-js-and-npm
 
-### Create local configurations
+```
+npm install -g npm@9.8.1
+npm install
+npm audit fix
+```
+
+### Securely install local configurations with secrets
 
-If you plan to use any resources outside localhost, put your configurations in a separately encrypted area:
+Put your configurations in a separately encrypted area outside of the project folder, which you will later export to environment variables. Ensure that your uid is in the AUTHORIZED_USERS list within that file.
 
 ```
 mkdir /Volumes/XYZ/hartsfield_config
-export HARTSFIELD_LOCAL_CONFIGS=/Volumes/XYZ/hartsfield_config
 ```
 
-## Greg does a jam like this from a pair of terminals in VSCode to run this locally:
+### Run one terminal session for the python back end...
 
 ```
 source venv/bin/activate
-export HARTSFIELD_LOCAL_CONFIGS=/Users/gregm/rip_hartsfield/hartsfield_config
+source .env.development
+export HARTSFIELD_LOCAL_CONFIGS=/Volumes/XYZ/hartsfield_config
 export HARTSFIELD_ENV=development
 venv/bin/python application.py
 ```
-and
+### ...and another terminal session for the Node.js front end:
 ```
-source venv/bin/activate
-export HARTSFIELD_LOCAL_CONFIGS=/Users/gregm/rip_hartsfield/hartsfield_config
+source .env.development
+export HARTSFIELD_LOCAL_CONFIGS=/Volumes/XYZ/hartsfield_config
 export HARTSFIELD_ENV=development
 npm run serve-vue
 ```
+
+## Using the application
+
+Browse to http://localhost:8080/ -- but note that the first access will take up to several minutes as all of the Node.js stuff does its thing! Subsequent access are fast.
+
+## A diagram of the intended function of the application:
+
+![Diagram of Hartsfield front end, back end, GCP components, and their relationships.](src/assets/2023-10-03_Hartsfield_diagram.png)

buildspec.yml

@@ -0,0 +1,32 @@
+version: 0.2
+
+phases:
+  install:
+    runtime-versions:
+      nodejs: 20
+      python: 3.11
+    commands:
+      - node -v
+      - npm install
+  pre_build:
+    commands:
+      - echo "pre_build phase"
+  build:
+    commands:
+      - npm run build-vue
+  post_build:
+    commands:
+      - ./scripts/codebuild/create-build-summary.sh
+artifacts:
+  files:
+  - '.ebextensions/**/*'
+  - 'dist/**/*'
+  - 'requirements.txt'
+  - 'hartsfield/**/*'
+  - 'scripts/**/*'
+  - 'application.py'
+  - 'consoler.py'
+  - 'config/**/*'
+  - '.platform/**/*'
+  - 'Procfile'
+

config/default.py

@@ -25,6 +25,13 @@
 
 import logging
 
+AWS_SECRETS_REGION = "us-west-2"
+AWS_SECRETS_NAME_AUTHORIZED_USERS = "AUTHORIZED_USERS"
+AWS_SECRETS_NAME_GCP_JSON_CREDENTIALS = "GCP_JSON_CREDENTIALS"
+
+CAS_SERVER = 'https://auth-test.berkeley.edu/cas/'
+CAS_LOGOUT_URL = 'https://auth-test.berkeley.edu/cas/logout'
+
 DEV_AUTH_ENABLED = False
 DEV_AUTH_PASSWORD = 'another secret'
 

hartsfield/api/auth_helper.py

@@ -0,0 +1,59 @@
+"""
+Copyright ©2022. The Regents of the University of California (Regents). All Rights Reserved.
+
+Permission to use, copy, modify, and distribute this software and its documentation
+for educational, research, and not-for-profit purposes, without fee and without a
+signed licensing agreement, is hereby granted, provided that the above copyright
+notice, this paragraph and the following two paragraphs appear in all copies,
+modifications, and distributions.
+
+Contact The Office of Technology Licensing, UC Berkeley, 2150 Shattuck Avenue,
+Suite 510, Berkeley, CA 94720-1620, (510) 643-7201, otl@berkeley.edu,
+http://ipira.berkeley.edu/industry-info for commercial licensing opportunities.
+
+IN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL,
+INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF
+THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN ADVISED
+OF THE POSSIBILITY OF SUCH DAMAGE.
+
+REGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE
+SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED HEREUNDER IS PROVIDED
+"AS IS". REGENTS HAS NO OBLIGATION TO PROVIDE MAINTENANCE, SUPPORT, UPDATES,
+ENHANCEMENTS, OR MODIFICATIONS.
+"""
+
+from functools import wraps
+
+from flask import current_app as app, request
+from flask_login import current_user
+from hartsfield.api.errors import UnauthorizedRequestError
+from hartsfield.models.user import find_by_uid
+
+
+def auth_required(f):
+    @wraps(f)
+    def decorated(*args, **kwargs):
+        if not current_user.is_authenticated:
+            auth = request.authorization
+            if not auth or not valid_worker_credentials(auth.username, auth.password):
+                raise UnauthorizedRequestError('Invalid credentials.')
+        return f(*args, **kwargs)
+    return decorated
+
+
+def authorzied_user_required(f):
+    @wraps(f)
+    def decorated(*args, **kwargs):
+        uid = current_user.uid
+        user = find_by_uid(uid)
+        if user is None:
+            auth = request.authorization
+            if not auth:
+                raise UnauthorizedRequestError('Invalid credentials.')
+        return f(*args, **kwargs)
+    return decorated
+
+
+def valid_worker_credentials(username, password):
+    return username == app.config['API_USERNAME'] and password == app.config['API_PASSWORD']

hartsfield/api/datahub_archive_url_fetch.py

@@ -22,79 +22,72 @@
 "AS IS". REGENTS HAS NO OBLIGATION TO PROVIDE MAINTENANCE, SUPPORT, UPDATES,
 ENHANCEMENTS, OR MODIFICATIONS.
 """
-from collections import OrderedDict
-import json
-
-from flask import current_app as app
-from hartsfield import __version__ as version
-from hartsfield.configs import load_configs
-from hartsfield.api.config_controller import load_json
-from hartsfield.lib.http import tolerant_jsonify
-from hartsfield.lib.util import get_eb_environment
-
-import requests
 import datetime
+import json
+import re
 
-from google.oauth2 import service_account
+from flask import current_app as app, request
 from google.cloud import storage
+from google.oauth2 import service_account
+from hartsfield.api.auth_helper import authorzied_user_required
+from hartsfield.lib.http import tolerant_jsonify
+import hartsfield.api.read_aws_secret
 
 PUBLIC_CONFIGS = [
     'DEV_AUTH_ENABLED',
     'HARTSFIELD_ENV',
     'TIMEZONE',
 ]
 
-gcp_json_credentials = app.config['GCP_JSON_CREDENTIALS']
-gcp_json_credentials_dict = json.loads(gcp_json_credentials)
+AWS_SECRETS_NAME_GCP_JSON_CREDENTIALS = app.config['AWS_SECRETS_NAME_GCP_JSON_CREDENTIALS']
 
-# TODO: pass in gs url as input value to @app.route('/api/fetch_url_direct') from form user front-end form submission
-gs_source_url="gs://ucb-datahub-archived-homedirs/spring-2021/datahub.berkeley.edu/peterphu-2edo.tar.gz"
-# This will probably be request.args['gs_source_url'] in the def block...but that whole "request" business needs to be brought in etc.
 
-@app.route('/api/fetch_url_direct')
+gcp_json_credentials_from_aws = hartsfield.api.read_aws_secret.read_aws_secret(AWS_SECRETS_NAME_GCP_JSON_CREDENTIALS)
+gcp_json_credentials_dict = json.loads(gcp_json_credentials_from_aws)
+
+@app.route('/api/fetch_url_direct', methods=['POST'])
+@authorzied_user_required
 def fetch_url_direct():
 
-    # parse the input gs url to get bucket and blob names
-    bucket_and_blob_string = gs_source_url.replace("gs://", "")
-    bucket_and_blob_list = bucket_and_blob_string.split("/")
+    params = request.get_json()
+    gs_source_url = params.get('gs_source_url')
+    gs_source_url = gs_source_url.strip()
+    if not re.match(r'gs://.{3,}/.+', gs_source_url):
+        error_message = 'The submitted data \"' + gs_source_url + '\" is not a valid gs_source_url.'
+        v = {'response': error_message, 'status': 'error'}
+        return tolerant_jsonify(v)
+
+    bucket_and_blob_string = gs_source_url.replace('gs://', '')
+    bucket_and_blob_list = bucket_and_blob_string.split('/')
     bucket_name = bucket_and_blob_list.pop(0)
-    blob_name = "/".join(bucket_and_blob_list)
+    blob_name = '/'.join(bucket_and_blob_list)
 
     # instantiate gcp storage client plus with bucket and blob objects
     credentials = service_account.Credentials.from_service_account_info(gcp_json_credentials_dict)
     storage_client = storage.Client(project=gcp_json_credentials_dict['project_id'], credentials=credentials)
     bucket = storage_client.bucket(bucket_name)
     blob = bucket.blob(blob_name)
 
-    # do some checks to confirm that the bucket and blob exist
     try:
         stats = storage.Blob(bucket=bucket, name=blob_name).exists(storage_client)
     except Exception as e:
-        error_message = "There was a problem trying to get stats on the requested blob \"" + blob_name + "\" in the requested bucket \"" + bucket_name +"\":\n\n " + str(e)
+        error_message = 'There was an exception trying to do the GCP storage operation with the submitted data \"' + \
+                        gs_source_url + \
+                        '\'. When GCP tried, it told us: \"' + \
+                        str(e) + '\"'
         v = {'response': error_message, 'status': 'error'}
         return tolerant_jsonify(v)
     if stats:
         # if the bucket and blob exist, generate a signed url for the blob...
         # ...and package it as a Hartsfield back-end internal response
         gcp_response = blob.generate_signed_url(
-            version="v4",
+            version='v4',
             expiration=datetime.timedelta(days=7),
-            method="GET",
+            method='GET',
         )
         v = {'response': gcp_response, 'status': 'success'}
     else:
-        gcp_response = "File \"" + blob_name + "\"does not exist in bucket \"" + bucket_name + "\""
+        gcp_response = 'GCP tried, but could not locate a file \"' + blob_name + '\" in a bucket called \"' + bucket_name + '\".'
         v = {'response': gcp_response, 'status': 'error'}
 
     return tolerant_jsonify(v)
-
-"""
-To make/fix/clean:
-
-- Make the Web request form / wire up web front end portion of app
-
-- CalNet auth in front of web app
-
-- All of the other ignorant/non-ideal coding practices I've done...!
-"""
-