move python side of workbench to the invest repo

.env-example

@@ -1,20 +1,16 @@
 # Nothing here is required, but it's sometimes useful in development
 # to customize the linkage to the python side of this application
-# by defining custom paths to the invest and server executeables.
+# by defining custom paths to the invest executeable.
 
-# USAGE: Create a `.env` file in the project root that looks similar to this one. This file is loaded by main.js if it is launched in dev mode. 
+# USAGE: Create a `.env` file in the project root that looks similar to
+# this one. This file is loaded by main.js if it is launched in dev mode. 
 
 # Use-case 1. PyInstaller-built invest binaries in some custom location.
 # leave off extensions - the app will add ".exe" if it detects Windows.
 INVEST="build/invest/invest"
-SERVER="build/invest/server"
 
 # Use-case 2. Paths to executeables in a python environment
-# This is useful when actively developing `server.py` because it 
-# avoids the need to build binaries with PyInstaller. Note, this time
-# we also need a path to python executeable.
-#PYTHON="env/bin/python"
-#SERVER="src/server.py"
+# This is useful when actively developing the flask app.
 #INVEST="env/bin/invest"
 
 

.github/workflows/build-electron.yml

@@ -9,7 +9,7 @@ jobs:
       max-parallel: 4
       fail-fast: false
       matrix:
-        os: [macos-latest, windows-latest, ubuntu-latest]
+        os: [macos-latest, windows-latest]
         node-version: [12.x]
 
     steps:
@@ -20,93 +20,14 @@ jobs:
       uses: actions/setup-node@v1
       with:
         node-version: ${{ matrix.node-version }}
-
-    - name: Set env var for invest version
-      shell: bash
-      run: echo "INVEST_VERSION=$(jq -r .invest.version package.json)" >> $GITHUB_ENV
-
-    - name: Clone natcap/invest
-      uses: actions/checkout@v2
-      with:
-        repository: natcap/invest
-        ref: refs/tags/${{env.INVEST_VERSION}}
-        path: ./invest
-
-    - name: Install libspatialindex for ubuntu
-      if: matrix.os == 'ubuntu-latest'
-      run:
-        sudo apt-get install libspatialindex-dev
-
-    - name: Set up conda for non-Windows
-      uses: conda-incubator/setup-miniconda@v2
-      if: matrix.os != 'windows-latest'
-      with:
-        activate-environment: invest-env
-        auto-update-conda: true
-        python-version: 3.7
-        channels: conda-forge
-
-    - name: Install python dependencies for non-Windows
-      if: matrix.os != 'windows-latest'
-      shell: bash -l {0}
-      working-directory: ./invest
-      run: |
-          conda upgrade -y pip setuptools
-          conda install toml requests
-          conda install $(python -c "import toml;print(' '.join(toml.load('pyproject.toml')['build-system']['requires']))")
-          python ./scripts/convert-requirements-to-conda-yml.py \
-                requirements.txt \
-                requirements-dev.txt > requirements-all.yml
-          conda env update --file requirements-all.yml
-
-          # Libtiff from conda defaults channel gets around issue with missing libwebp
-          conda install --yes libtiff
-
-          # The dependencies not required by natcap.invest
-          conda install flask
-
-    - name: Set up Windows python
-      if: matrix.os == 'windows-latest'
-      uses: actions/setup-python@v1
-      with:
-        python-version: 3.7
-        architecture: x64
-
-    - name: Install python build dependencies for Windows
-      if: matrix.os == 'windows-latest'
-      working-directory: ./invest
-      shell: bash -l {0}
-      run: |
-          python -m pip install --upgrade pip nose setuptools toml twine ${{ matrix.numpy }}
-          pip install $(python -c "import toml;print(' '.join(toml.load('pyproject.toml')['build-system']['requires']))")
-
-    - name: Install python runtime dependencies for Windows
-      if: matrix.os == 'windows-latest'
-      working-directory: ./invest
-      shell: bash -l {0}
-      env:
-        PIP_EXTRA_INDEX_URL: "http://pypi.naturalcapitalproject.org/simple/"
-        PIP_TRUSTED_HOST: "pypi.naturalcapitalproject.org"
-        PIP_PREFER_BINARY: 1
-        SITE_PACKAGES: "C:/hostedtoolcache/windows/Python/3.7.9/x64/lib/site-packages"
-      run: |
-        python -m pip install -r requirements.txt -r requirements-dev.txt
-        pip install flask
-        cp ${{ env.SITE_PACKAGES }}/shapely/DLLs/geos_c.dll ${{ env.SITE_PACKAGES }}/shapely/DLLs/geos.dll
-
-    - name: Install natcap.invest python package
-      shell: bash -l {0}
-      working-directory: ./invest
-      run: make install
-
-    - name: Freeze Python environment
-      shell: bash -l {0}
-      run: |
-        python -m PyInstaller --workpath build/pyi-build --clean --distpath build ./invest-flask.spec
 
     - name: NPM Install
       run: npm install
 
+    - name: Fetch InVEST Binaries
+      shell: bash
+      run: npm run fetch-invest
+
     - name: Build & Release Electron application
       uses: samuelmeuli/action-electron-builder@v1
       if: ${{ always() }}
@@ -125,6 +46,7 @@ jobs:
 
     - name: Test electron app with puppeteer
       uses: GabrielBB/xvfb-action@v1
+      if: matrix.os != 'macos-latest'
       with:
         run: npm run test-electron-app
 

package.json

@@ -6,13 +6,14 @@
   "main": "src/main.js",
   "homepage": "./",
   "scripts": {
-    "start": "npx electron . --dev",
+    "start": "electron . --dev",
     "lint": "eslint --cache --color --ext .jsx,.js src",
     "test": "cross-env DEBUG_PRINT_LIMIT=10 jest --testMatch **/tests/*.test.js",
     "test-flask-app": "jest --runInBand --testMatch **/tests/binary_tests/flaskapp.test.js",
     "test-electron-app": "jest --runInBand --testMatch **/tests/binary_tests/puppet.test.js",
     "postinstall": "electron-builder install-app-deps",
-    "build": "npx ./build.js",
+    "fetch-invest": "node ./scripts/fetch_invest_binaries.js && unzip ./build/binaries.zip -d build/invest/",
+    "build": "node ./build.js",
     "dist": "electron-builder -c.extraMetadata.main=build/main.js",
     "pack": "electron-builder --dir -c.extraMetadata.main=build/main.js"
   },
@@ -56,7 +57,14 @@
     ]
   },
   "invest": {
-    "version": "3.8.9"
+    "hostname": "https://storage.googleapis.com",
+    "bucket": "natcap-dev-build-artifacts",
+    "fork": "invest/davemfish",
+    "version": "3.8.9.post1130+gdd914a7a",
+    "target": {
+      "macos": "mac_invest_binaries.zip",
+      "windows": "windows_invest_binaries.zip"
+    }
   },
   "keywords": [],
   "author": "dmf",

readme.md

@@ -9,29 +9,34 @@ workflows of an InVEST user.
 ## To develop and launch this Application
 
 * `npm install`  
-* clone natcap.invest and checkout a recent revision (e.g. `main`)  
-* setup a conda* environment with deps for `natcap.invest Flask PyInstaller`  
-* build invest binaries  
-	`python -m PyInstaller --workpath build/pyi-build --clean --distpath build ./invest-flask.spec`  
-* `npm start`  
+* bind to an `invest` executeable (see package.json "invest" for a compatible version)
 
-(* invest-flask.spec script assumes a conda environment)
+In production, the invest exe comes from prebuilt binaries that are an artifact of the `invest` build process.  
 
-## To build this application
+For development, choose either:  
+  **A.** Duplicate the production setup by fetching prebuilt binaries `npm run fetch-invest`  
+  **B.** Use any other locally installed, compatible, invest CLI (e.g. from a local python environment). To configure this, see `.env-example`
 
-`npm run build`  -- calls babel to transpile ES6 and jsx code to commonjs
+* `npm start`
 
-`npm run dist`  -- packages build source into an electron application using electron-builder
+## To package this app for distribution  
 
+`npm run build`  - calls babel to transpile ES6 and jsx code to commonjs; moves other resources (CSS, JSON) to the build directory
 
-### To run linter or tests
+`npm run dist`  - packages build source into an electron application using electron-builder
+
+
+### To run various scripts and local programs
+See the "scripts" section of `package.json` and run them like:  
 `npm run lint`  
 `npm run test`  
 
-To run these or other command-line utils of locally installed packages outside the context of the `package.json scripts`, use `npx` (e.g. `npx eslint ...`) as a shortcut to the executeable. 
+To run other scripts or CLIs of locally installed packages, 
+prefix commands with `npx` (e.g. `npx eslint ...`). Otherwise, only
+globally installed packages are on the PATH. 
 
-### To run a single test file:
+#### E.g. run a single test file:
 `npx jest --coverage=false --verbose app.test.js`  
 
-To run snippets of code outside the electron runtime, but with the same ECMAscript features and babel configurations, use `node -r @babel/register script.js`.  
+To run javascript outside the electron runtime, but with the same ECMAscript features and babel configurations, use `node -r @babel/register script.js`.  
 

scripts/fetch_invest_binaries.js

@@ -0,0 +1,59 @@
+const https = require('https');
+const fs = require('fs');
+const path = require('path');
+const url = require('url');
+const pkg = require('../package');
+
+let filePrefix;
+// The invest build process only builds for these OS
+// The prefixes on the zip file are defined by invest's Makefile $OSNAME
+switch (process.platform) {
+  case 'win32':
+    filePrefix = 'windows';
+    break;
+  case 'darwin':
+    filePrefix = 'mac';
+    break;
+  default:
+    throw new Error(
+      `No prebuilt invest binaries are available for ${process.platform}`
+    );
+}
+
+const HOSTNAME = pkg.invest.hostname;
+const BUCKET = pkg.invest.bucket;
+const FORK = pkg.invest.fork;
+const VERSION = pkg.invest.version;
+const SRCFILE = `${filePrefix}_invest_binaries.zip`;
+const DESTFILE = path.resolve('build/binaries.zip');
+
+const urladdress = url.resolve(
+  HOSTNAME, path.join(BUCKET, FORK, VERSION, SRCFILE)
+);
+
+/**
+ * Download a file from src to dest.
+ *
+ * @param  {string} src - url for a single publicly hosted file
+ * @param  {string} dest - local path for saving the file
+ */
+function download(src, dest) {
+  console.log(`downloading ${src}`);
+  fs.existsSync(path.dirname(dest)) || fs.mkdirSync(path.dirname(dest));
+  const fileStream = fs.createWriteStream(dest);
+  https.get(src, (response) => {
+    console.log(`http status: ${response.statusCode}`);
+    if (response.statusCode !== 200) {
+      fileStream.close();
+      return;
+    }
+    response.pipe(fileStream);
+    fileStream.on('finish', () => {
+      fileStream.close();
+    });
+  }).on('error', (e) => {
+    console.log(e);
+  });
+}
+
+download(urladdress, DESTFILE);