Merge pull request #1063 from blue-marble/develop

GridPath v0.16.0

.github/workflows/test_gridpath.yml

@@ -18,7 +18,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ "3.8", "3.9", "3.10" ]
+        python-version: [ "3.9", "3.10", "3.11" ]
 
     steps:
     - uses: actions/checkout@v3

MANIFEST.in

@@ -1 +1,3 @@
 include gridpath/project/operations/operational_types/opchar_param_requirements.csv
+include db/db_schema.sql
+include db/data/*.*

README.md

@@ -8,10 +8,10 @@
 # Welcome to GridPath
 
 <p align="center" width="100%">
-    <img src="doc/graphics/gridpath_logo.png" width="20%"/>
+    <img src="https://github.com/blue-marble/gridpath/blob/main/doc/graphics/gridpath_logo.png?raw=true" width="20%"/>
 </p>
 
-![Approaches](doc/graphics/approaches.png)
+![Approaches](https://github.com/blue-marble/gridpath/blob/main/doc/graphics/approaches.png?raw=true)
 
 
 GridPath is a versatile power-system planning platform capable of a range of
@@ -24,28 +24,27 @@ GridPath's documentation is hosted on [Read the Docs](https://gridpath.readthedo
 # Installation
 
 ## Python
-GridPath is tested on Python 3.8, 3.9, and 3.10. We recommend using Python 3.9. Get 
-Python
-[here](https://www.python.org/downloads/ "Python download").
+GridPath is tested on Python 3.9, 3.10, and 3.11. Get one of those Python versions [here](https://www.python.org/downloads/ "Python download").
 
 ## GridPath Python environment
 You should create a Python environment for your GridPath installation, e.g. via 
 `venv`, [a lightweight environment manager](https://docs.python.org/3/library/venv.html "venv") 
 that is part of the standard Python distribution. Make sure to [create](https://docs.python.org/3/library/venv.html#creating-virtual-environments "create") [activate](https://docs.python.org/3/library/venv.html#how-venvs-work "activate") the environment before installing GridPath.
 
-## Install GridPath
+## Install GridPath from PyPi
 
-Once you have _created and activated_ the GridPath Python environment, you can install GridPath and the Python packages it uses.
+Once you have _created and activated_ the GridPath Python environment, you 
+can install the latest version of GridPath from PyPi with:
 
-For most users, installing GridPath's base set of Python packages and those needed 
-to use the graphical user interface would be sufficient. You can do so by navigating to the GridPath root 
-directory (which is where this `README.md` file is located) and running:
 ```bash
-pip install .[ui]
+pip install GridPath
 ```
 
-You can install all needed Python 
-packages, including the developer extras by running:
+## Install GridPath from source
+
+You can alternatively download the GridPath source code and install from 
+source.
+
 ```bash
 pip install .[all]
 ```
@@ -56,13 +55,6 @@ pip install .[all]
 You will need a solver to use this platform. GridPath assumes you will be using Cbc (Coin-or branch and cut) by default, but you can specify a 
 different solver.
 
-## Testing your installation
-
-To test the GridPath codebase, [make sure the GridPath environment you installed to 
-is activated](https://docs.python.org/3/library/venv.html#how-venvs-work "activate") and use the unittest module as follows from the root directory:
-```bash
-python -m unittest discover tests
-```
 
 # Usage
 
@@ -90,11 +82,3 @@ solver, check the help menu, e.g.:
 ```bash
 gridpath_run --help
 ```
-
-
-## Help
-In general, you can check usage of GridPath's scripts by calling the `--help` 
-option, e.g.:
-```bash
-python get_scenario_inputs.py --help
-```

db/common_functions.py

@@ -45,6 +45,7 @@ def connect_to_database(db_path="../db/io.db", timeout=5, detect_types=0):
     return conn
 
 
+# TODO: move to spin_database_lock_generic
 def spin_on_database_lock(
     conn, cursor, sql, data, many=True, max_attempts=61, interval=10, quiet=True
 ):
@@ -76,10 +77,7 @@ def spin_on_database_lock(
         print(sql)
 
     for i in range(0, max_attempts):
-        if i == 0:
-            # print("initial attempt")
-            pass
-        else:
+        if i > 0:
             print("...retrying (attempt {} of {})...".format(i, max_attempts))
         try:
             if many:
@@ -109,3 +107,55 @@ def spin_on_database_lock(
         else:
             # print("...done.")
             break
+
+
+def spin_on_database_lock_generic(
+    command,
+    max_attempts=61,
+    interval=10,
+):
+    """
+    :param command:
+    :param max_attempts: how long to wait for the database lock to be
+        released; the default is 600 seconds, but that can be overridden
+    :param interval: how frequently to poll the database for whether
+        the lock has been released; the default is 10 seconds, but that can
+        be overridden
+
+    If the database is locked, wait for the lock to be released for a
+    certain amount of time and occasionally retry to execute the SQL
+    statement until the timeout.
+
+    To lock the database deliberately, run the following:
+        PRAGMA locking_mode = EXCLUSIVE;
+        BEGIN EXCLUSIVE;
+    The database will be locked until you run:
+        COMMIT;
+    """
+    for i in range(0, max_attempts):
+        if i > 0:
+            print("...retrying (attempt {} of {})...".format(i, max_attempts))
+        try:
+            command
+        except sqlite3.OperationalError as e:
+            if "locked" in str(e):
+                print(
+                    "Database is locked, sleeping for {} seconds, "
+                    "then retrying.".format(interval)
+                )
+                if i == max_attempts:
+                    print(
+                        "Database still locked after {} seconds. "
+                        "Exiting.".format(max_attempts * interval)
+                    )
+                    sys.exit(1)
+                else:
+                    time.sleep(interval)
+            else:
+                print("Error while running the following query:\n", sql)
+                traceback.print_exc()
+                sys.exit()
+        # Do this if exception not caught
+        else:
+            # print("...done.")
+            break

db/create_database.py

@@ -1,4 +1,4 @@
-# Copyright 2016-2020 Blue Marble Analytics LLC.
+# Copyright 2016-2023 Blue Marble Analytics LLC.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,20 +13,20 @@
 # limitations under the License.
 
 """
-This script create an empty GridPath database with the appropriate table structure.
+Create an empty GridPath database with the appropriate table structure.
 
 The user may specify the name and location of the GridPath database path using the
 *--database* flag.
 
->>> python create_database.py --database PATH/DO/DB
+>>> gridpath_create_database --database PATH/DO/DB
 
 The default schema for the GridPath SQLite database is in db_schema.sql.
 
 .. _database-structure-section-ref:
 
 """
 
-from builtins import str
+
 from argparse import ArgumentParser
 import csv
 import os.path
@@ -135,9 +135,6 @@ def load_data(conn, omit_data, custom_units):
         # Data for plotting
         load_viz_technologies(conn=conn)
 
-    else:
-        pass
-
 
 def load_mod_months(conn):
     sql = """