Skip to content
This repository has been archived by the owner on Oct 18, 2022. It is now read-only.

Commit

Permalink
Merge pull request #72 from davemfish/task/70-move-python-to-invest-repo
Browse files Browse the repository at this point in the history
move python side of workbench to the invest repo
  • Loading branch information
phargogh authored Jan 8, 2021
2 parents c3b3041 + 9e42849 commit edb83d8
Show file tree
Hide file tree
Showing 11 changed files with 133 additions and 467 deletions.
12 changes: 4 additions & 8 deletions .env-example
Original file line number Diff line number Diff line change
@@ -1,20 +1,16 @@
# Nothing here is required, but it's sometimes useful in development
# to customize the linkage to the python side of this application
# by defining custom paths to the invest and server executeables.
# by defining custom paths to the invest executeable.

# USAGE: Create a `.env` file in the project root that looks similar to this one. This file is loaded by main.js if it is launched in dev mode.
# USAGE: Create a `.env` file in the project root that looks similar to
# this one. This file is loaded by main.js if it is launched in dev mode.

# Use-case 1. PyInstaller-built invest binaries in some custom location.
# leave off extensions - the app will add ".exe" if it detects Windows.
INVEST="build/invest/invest"
SERVER="build/invest/server"

# Use-case 2. Paths to executeables in a python environment
# This is useful when actively developing `server.py` because it
# avoids the need to build binaries with PyInstaller. Note, this time
# we also need a path to python executeable.
#PYTHON="env/bin/python"
#SERVER="src/server.py"
# This is useful when actively developing the flask app.
#INVEST="env/bin/invest"


Expand Down
90 changes: 6 additions & 84 deletions .github/workflows/build-electron.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ jobs:
max-parallel: 4
fail-fast: false
matrix:
os: [macos-latest, windows-latest, ubuntu-latest]
os: [macos-latest, windows-latest]
node-version: [12.x]

steps:
Expand All @@ -20,93 +20,14 @@ jobs:
uses: actions/setup-node@v1
with:
node-version: ${{ matrix.node-version }}

- name: Set env var for invest version
shell: bash
run: echo "INVEST_VERSION=$(jq -r .invest.version package.json)" >> $GITHUB_ENV

- name: Clone natcap/invest
uses: actions/checkout@v2
with:
repository: natcap/invest
ref: refs/tags/${{env.INVEST_VERSION}}
path: ./invest

- name: Install libspatialindex for ubuntu
if: matrix.os == 'ubuntu-latest'
run:
sudo apt-get install libspatialindex-dev

- name: Set up conda for non-Windows
uses: conda-incubator/setup-miniconda@v2
if: matrix.os != 'windows-latest'
with:
activate-environment: invest-env
auto-update-conda: true
python-version: 3.7
channels: conda-forge

- name: Install python dependencies for non-Windows
if: matrix.os != 'windows-latest'
shell: bash -l {0}
working-directory: ./invest
run: |
conda upgrade -y pip setuptools
conda install toml requests
conda install $(python -c "import toml;print(' '.join(toml.load('pyproject.toml')['build-system']['requires']))")
python ./scripts/convert-requirements-to-conda-yml.py \
requirements.txt \
requirements-dev.txt > requirements-all.yml
conda env update --file requirements-all.yml
# Libtiff from conda defaults channel gets around issue with missing libwebp
conda install --yes libtiff
# The dependencies not required by natcap.invest
conda install flask
- name: Set up Windows python
if: matrix.os == 'windows-latest'
uses: actions/setup-python@v1
with:
python-version: 3.7
architecture: x64

- name: Install python build dependencies for Windows
if: matrix.os == 'windows-latest'
working-directory: ./invest
shell: bash -l {0}
run: |
python -m pip install --upgrade pip nose setuptools toml twine ${{ matrix.numpy }}
pip install $(python -c "import toml;print(' '.join(toml.load('pyproject.toml')['build-system']['requires']))")
- name: Install python runtime dependencies for Windows
if: matrix.os == 'windows-latest'
working-directory: ./invest
shell: bash -l {0}
env:
PIP_EXTRA_INDEX_URL: "http://pypi.naturalcapitalproject.org/simple/"
PIP_TRUSTED_HOST: "pypi.naturalcapitalproject.org"
PIP_PREFER_BINARY: 1
SITE_PACKAGES: "C:/hostedtoolcache/windows/Python/3.7.9/x64/lib/site-packages"
run: |
python -m pip install -r requirements.txt -r requirements-dev.txt
pip install flask
cp ${{ env.SITE_PACKAGES }}/shapely/DLLs/geos_c.dll ${{ env.SITE_PACKAGES }}/shapely/DLLs/geos.dll
- name: Install natcap.invest python package
shell: bash -l {0}
working-directory: ./invest
run: make install

- name: Freeze Python environment
shell: bash -l {0}
run: |
python -m PyInstaller --workpath build/pyi-build --clean --distpath build ./invest-flask.spec

- name: NPM Install
run: npm install

- name: Fetch InVEST Binaries
shell: bash
run: npm run fetch-invest

- name: Build & Release Electron application
uses: samuelmeuli/action-electron-builder@v1
if: ${{ always() }}
Expand All @@ -125,6 +46,7 @@ jobs:

- name: Test electron app with puppeteer
uses: GabrielBB/xvfb-action@v1
if: matrix.os != 'macos-latest'
with:
run: npm run test-electron-app

Expand Down
104 changes: 0 additions & 104 deletions invest-flask.spec

This file was deleted.

14 changes: 11 additions & 3 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,14 @@
"main": "src/main.js",
"homepage": "./",
"scripts": {
"start": "npx electron . --dev",
"start": "electron . --dev",
"lint": "eslint --cache --color --ext .jsx,.js src",
"test": "cross-env DEBUG_PRINT_LIMIT=10 jest --testMatch **/tests/*.test.js",
"test-flask-app": "jest --runInBand --testMatch **/tests/binary_tests/flaskapp.test.js",
"test-electron-app": "jest --runInBand --testMatch **/tests/binary_tests/puppet.test.js",
"postinstall": "electron-builder install-app-deps",
"build": "npx ./build.js",
"fetch-invest": "node ./scripts/fetch_invest_binaries.js && unzip ./build/binaries.zip -d build/invest/",
"build": "node ./build.js",
"dist": "electron-builder -c.extraMetadata.main=build/main.js",
"pack": "electron-builder --dir -c.extraMetadata.main=build/main.js"
},
Expand Down Expand Up @@ -56,7 +57,14 @@
]
},
"invest": {
"version": "3.8.9"
"hostname": "https://storage.googleapis.com",
"bucket": "natcap-dev-build-artifacts",
"fork": "invest/davemfish",
"version": "3.8.9.post1130+gdd914a7a",
"target": {
"macos": "mac_invest_binaries.zip",
"windows": "windows_invest_binaries.zip"
}
},
"keywords": [],
"author": "dmf",
Expand Down
31 changes: 18 additions & 13 deletions readme.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,29 +9,34 @@ workflows of an InVEST user.
## To develop and launch this Application

* `npm install`
* clone natcap.invest and checkout a recent revision (e.g. `main`)
* setup a conda* environment with deps for `natcap.invest Flask PyInstaller`
* build invest binaries
`python -m PyInstaller --workpath build/pyi-build --clean --distpath build ./invest-flask.spec`
* `npm start`
* bind to an `invest` executeable (see package.json "invest" for a compatible version)

(* invest-flask.spec script assumes a conda environment)
In production, the invest exe comes from prebuilt binaries that are an artifact of the `invest` build process.

## To build this application
For development, choose either:
**A.** Duplicate the production setup by fetching prebuilt binaries `npm run fetch-invest`
**B.** Use any other locally installed, compatible, invest CLI (e.g. from a local python environment). To configure this, see `.env-example`

`npm run build` -- calls babel to transpile ES6 and jsx code to commonjs
* `npm start`

`npm run dist` -- packages build source into an electron application using electron-builder
## To package this app for distribution

`npm run build` - calls babel to transpile ES6 and jsx code to commonjs; moves other resources (CSS, JSON) to the build directory

### To run linter or tests
`npm run dist` - packages build source into an electron application using electron-builder


### To run various scripts and local programs
See the "scripts" section of `package.json` and run them like:
`npm run lint`
`npm run test`

To run these or other command-line utils of locally installed packages outside the context of the `package.json scripts`, use `npx` (e.g. `npx eslint ...`) as a shortcut to the executeable.
To run other scripts or CLIs of locally installed packages,
prefix commands with `npx` (e.g. `npx eslint ...`). Otherwise, only
globally installed packages are on the PATH.

### To run a single test file:
#### E.g. run a single test file:
`npx jest --coverage=false --verbose app.test.js`

To run snippets of code outside the electron runtime, but with the same ECMAscript features and babel configurations, use `node -r @babel/register script.js`.
To run javascript outside the electron runtime, but with the same ECMAscript features and babel configurations, use `node -r @babel/register script.js`.

59 changes: 59 additions & 0 deletions scripts/fetch_invest_binaries.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
const https = require('https');
const fs = require('fs');
const path = require('path');
const url = require('url');
const pkg = require('../package');

let filePrefix;
// The invest build process only builds for these OS
// The prefixes on the zip file are defined by invest's Makefile $OSNAME
switch (process.platform) {
case 'win32':
filePrefix = 'windows';
break;
case 'darwin':
filePrefix = 'mac';
break;
default:
throw new Error(
`No prebuilt invest binaries are available for ${process.platform}`
);
}

const HOSTNAME = pkg.invest.hostname;
const BUCKET = pkg.invest.bucket;
const FORK = pkg.invest.fork;
const VERSION = pkg.invest.version;
const SRCFILE = `${filePrefix}_invest_binaries.zip`;
const DESTFILE = path.resolve('build/binaries.zip');

const urladdress = url.resolve(
HOSTNAME, path.join(BUCKET, FORK, VERSION, SRCFILE)
);

/**
* Download a file from src to dest.
*
* @param {string} src - url for a single publicly hosted file
* @param {string} dest - local path for saving the file
*/
function download(src, dest) {
console.log(`downloading ${src}`);
fs.existsSync(path.dirname(dest)) || fs.mkdirSync(path.dirname(dest));
const fileStream = fs.createWriteStream(dest);
https.get(src, (response) => {
console.log(`http status: ${response.statusCode}`);
if (response.statusCode !== 200) {
fileStream.close();
return;
}
response.pipe(fileStream);
fileStream.on('finish', () => {
fileStream.close();
});
}).on('error', (e) => {
console.log(e);
});
}

download(urladdress, DESTFILE);
Loading

0 comments on commit edb83d8

Please sign in to comment.