From 6f75e19dfddf2c4a0e8cf9ceb69cd8236c030426 Mon Sep 17 00:00:00 2001 From: davemfish Date: Sat, 21 Nov 2020 10:02:41 -0800 Subject: [PATCH 01/25] updated the Actions build process to download prebuilt invest binaries instead of building from source --- .github/workflows/build-electron.yml | 90 +++------------------------- package.json | 9 ++- scripts/fetch_invest_binaries.js | 63 +++++++++++++++++++ 3 files changed, 78 insertions(+), 84 deletions(-) create mode 100644 scripts/fetch_invest_binaries.js diff --git a/.github/workflows/build-electron.yml b/.github/workflows/build-electron.yml index 0b15a5df..c0db8d0e 100644 --- a/.github/workflows/build-electron.yml +++ b/.github/workflows/build-electron.yml @@ -20,93 +20,16 @@ jobs: uses: actions/setup-node@v1 with: node-version: ${{ matrix.node-version }} - - - name: Set env var for invest version - shell: bash - run: echo "INVEST_VERSION=$(jq -r .invest.version package.json)" >> $GITHUB_ENV - - - name: Clone natcap/invest - uses: actions/checkout@v2 - with: - repository: natcap/invest - ref: refs/tags/${{env.INVEST_VERSION}} - path: ./invest - - - name: Install libspatialindex for ubuntu - if: matrix.os == 'ubuntu-latest' - run: - sudo apt-get install libspatialindex-dev - - - name: Set up conda for non-Windows - uses: conda-incubator/setup-miniconda@v2 - if: matrix.os != 'windows-latest' - with: - activate-environment: invest-env - auto-update-conda: true - python-version: 3.7 - channels: conda-forge - - - name: Install python dependencies for non-Windows - if: matrix.os != 'windows-latest' - shell: bash -l {0} - working-directory: ./invest - run: | - conda upgrade -y pip setuptools - conda install toml requests - conda install $(python -c "import toml;print(' '.join(toml.load('pyproject.toml')['build-system']['requires']))") - python ./scripts/convert-requirements-to-conda-yml.py \ - requirements.txt \ - requirements-dev.txt > requirements-all.yml - conda env update --file requirements-all.yml - - # Libtiff from conda defaults channel gets around issue with missing libwebp - conda install --yes libtiff - - # The dependencies not required by natcap.invest - conda install flask - - - name: Set up Windows python - if: matrix.os == 'windows-latest' - uses: actions/setup-python@v1 - with: - python-version: 3.7 - architecture: x64 - - - name: Install python build dependencies for Windows - if: matrix.os == 'windows-latest' - working-directory: ./invest - shell: bash -l {0} - run: | - python -m pip install --upgrade pip nose setuptools toml twine ${{ matrix.numpy }} - pip install $(python -c "import toml;print(' '.join(toml.load('pyproject.toml')['build-system']['requires']))") - - - name: Install python runtime dependencies for Windows - if: matrix.os == 'windows-latest' - working-directory: ./invest - shell: bash -l {0} - env: - PIP_EXTRA_INDEX_URL: "http://pypi.naturalcapitalproject.org/simple/" - PIP_TRUSTED_HOST: "pypi.naturalcapitalproject.org" - PIP_PREFER_BINARY: 1 - SITE_PACKAGES: "C:/hostedtoolcache/windows/Python/3.7.9/x64/lib/site-packages" - run: | - python -m pip install -r requirements.txt -r requirements-dev.txt - pip install flask - cp ${{ env.SITE_PACKAGES }}/shapely/DLLs/geos_c.dll ${{ env.SITE_PACKAGES }}/shapely/DLLs/geos.dll - - - name: Install natcap.invest python package - shell: bash -l {0} - working-directory: ./invest - run: make install - - - name: Freeze Python environment - shell: bash -l {0} - run: | - python -m PyInstaller --workpath build/pyi-build --clean --distpath build ./invest-flask.spec - name: NPM Install run: npm install + - name: Fetch InVEST Binaries + shell: bash + run: | + npx node ./scripts/fetch_invest_binaries.js ${{matrix.os}} + unzip ./build/binaries.zip -d build/invest/ + - name: Build & Release Electron application uses: samuelmeuli/action-electron-builder@v1 if: ${{ always() }} @@ -125,6 +48,7 @@ jobs: - name: Test electron app with puppeteer uses: GabrielBB/xvfb-action@v1 + if: ${{ matrix.os }} != 'macos-latest' with: run: npm run test-electron-app diff --git a/package.json b/package.json index 569c3a98..3c40b769 100644 --- a/package.json +++ b/package.json @@ -56,7 +56,14 @@ ] }, "invest": { - "version": "3.8.9" + "hostname": "https://storage.googleapis.com", + "bucket": "natcap-dev-build-artifacts", + "fork": "invest/davemfish", + "version": "3.8.0.post1935+gb52992af", + "target": { + "macos": "mac_invest_binaries.zip", + "windows": "windows_invest_binaries.zip" + } }, "keywords": [], "author": "dmf", diff --git a/scripts/fetch_invest_binaries.js b/scripts/fetch_invest_binaries.js new file mode 100644 index 00000000..29701bad --- /dev/null +++ b/scripts/fetch_invest_binaries.js @@ -0,0 +1,63 @@ +/** + * 2020-07-30 - the workbench build process no longer depends on + * pre-built binaries, instead building them itself, so this script + * is not being used anymore. + * + * This is a utility script for fetching prebuilt invest binaries + * that include the server.py module and thus are compatible with the + * workbench. + */ + +const https = require('https') +const fs = require('fs') +const path = require('path') +const url = require('url') +const package = require('../package') + +// TODO: better to just detect the platform with node instead of passing +const args = process.argv.slice(2) +if (args.length !== 1) { + throw new Error('expected exactly 1 argument: the current OS'); +} +let fileSuffix; +switch (args[0]) { + case 'windows-latest': + filePrefix = 'windows' + break + case 'macos-latest': + filePrefix = 'macos' + break + case 'ubuntu-latest': + filePrefix = 'linux' + break + default: + throw new Error("expected argument to be in ['windows-latest, 'macos-latest', 'ubuntu-latest']") +} + +const HOSTNAME = package.invest.hostname +const BUCKET = package.invest.bucket +const FORK = package.invest.fork +const VERSION = package.invest.version +const SRCFILE = `${filePrefix}_invest_binaries.zip` +const DESTFILE = path.resolve('build/binaries.zip'); + +const urladdress = url.resolve(HOSTNAME, path.join(BUCKET, FORK, VERSION, SRCFILE)) + +const download = function(url, dest) { + fs.existsSync(path.dirname(dest)) || fs.mkdirSync(path.dirname(dest)); + const fileStream = fs.createWriteStream(dest) + const request = https.get(url, function(response) { + console.log(response.statusCode) + if (response.statusCode != 200) { + fileStream.close() + console.log(url) + return + } + response.pipe(fileStream) + fileStream.on('finish', function() { + fileStream.close() + }) + }) +} + +download(urladdress, DESTFILE) From 00400c38f4dd09dd8be5cefd3d95d0885b39befa Mon Sep 17 00:00:00 2001 From: davemfish Date: Sat, 21 Nov 2020 15:00:11 -0800 Subject: [PATCH 02/25] documentation for the new invest-binding process --- .env-example | 12 ++++-------- readme.md | 26 +++++++++++++++----------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.env-example b/.env-example index 2d59783f..2bda52f2 100644 --- a/.env-example +++ b/.env-example @@ -1,20 +1,16 @@ # Nothing here is required, but it's sometimes useful in development # to customize the linkage to the python side of this application -# by defining custom paths to the invest and server executeables. +# by defining custom paths to the invest executeable. -# USAGE: Create a `.env` file in the project root that looks similar to this one. This file is loaded by main.js if it is launched in dev mode. +# USAGE: Create a `.env` file in the project root that looks similar to +# this one. This file is loaded by main.js if it is launched in dev mode. # Use-case 1. PyInstaller-built invest binaries in some custom location. # leave off extensions - the app will add ".exe" if it detects Windows. INVEST="build/invest/invest" -SERVER="build/invest/server" # Use-case 2. Paths to executeables in a python environment -# This is useful when actively developing `server.py` because it -# avoids the need to build binaries with PyInstaller. Note, this time -# we also need a path to python executeable. -#PYTHON="env/bin/python" -#SERVER="src/server.py" +# This is useful when actively developing the flask app. #INVEST="env/bin/invest" diff --git a/readme.md b/readme.md index c815b94e..99f4add1 100644 --- a/readme.md +++ b/readme.md @@ -8,30 +8,34 @@ workflows of an InVEST user. ## To develop and launch this Application +* `npm start` but first, * `npm install` -* clone natcap.invest and checkout a recent revision (e.g. `main`) -* setup a conda* environment with deps for `natcap.invest Flask PyInstaller` -* build invest binaries - `python -m PyInstaller --workpath build/pyi-build --clean --distpath build ./invest-flask.spec` -* `npm start` +* bind to an `invest` executeable -(* invest-flask.spec script assumes a conda environment) +In production, the invest exe comes from prebuilt binaries that are an artifact of the `invest` build process. + +For development, there are options: +* A. Duplicate the production setup by fetching prebuilt binaries +`npm run fetch-invest` +* B. use an any other locally installed, compatible, invest CLI (e.g. from an active python environment). To configure this, see `.env-example` -## To build this application `npm run build` -- calls babel to transpile ES6 and jsx code to commonjs `npm run dist` -- packages build source into an electron application using electron-builder -### To run linter or tests +### To run various scripts and local programs +See the "scripts" section of `package.json` and run of them like: `npm run lint` `npm run test` -To run these or other command-line utils of locally installed packages outside the context of the `package.json scripts`, use `npx` (e.g. `npx eslint ...`) as a shortcut to the executeable. +To run other scripts or CLIs of locally installed packages, +prefix commands with `npx` (e.g. `npx eslint ...`). Otherwise, only +globally installed packages are on the PATH. -### To run a single test file: +#### E.g. run a single test file: `npx jest --coverage=false --verbose app.test.js` -To run snippets of code outside the electron runtime, but with the same ECMAscript features and babel configurations, use `node -r @babel/register script.js`. +To run javascript outside the electron runtime, but with the same ECMAscript features and babel configurations, use `node -r @babel/register script.js`. From 1026ff281e22959d570579ee599cb5c5fda0cdbd Mon Sep 17 00:00:00 2001 From: davemfish Date: Sat, 21 Nov 2020 15:16:27 -0800 Subject: [PATCH 03/25] make app compatible with the new invest cli that includes the serve entrypoint --- src/main.js | 6 ++--- src/main_helpers.js | 41 ++++++++++------------------- tests/binary_tests/flaskapp.test.js | 4 +-- 3 files changed, 19 insertions(+), 32 deletions(-) diff --git a/src/main.js b/src/main.js index 92740022..a1311893 100644 --- a/src/main.js +++ b/src/main.js @@ -30,13 +30,13 @@ const createWindow = async () => { // The main process needs to know the location of the invest server binary. // The renderer process needs the invest cli binary. We can find them // together here and pass data to the renderer upon request. - const binaries = await findInvestBinaries(isDevMode); - const mainProcessVars = { investExe: binaries.invest }; + const investExe = await findInvestBinaries(isDevMode); + const mainProcessVars = { investExe: investExe }; ipcMain.on('variable-request', (event, arg) => { event.reply('variable-reply', mainProcessVars); }); - createPythonFlaskProcess(binaries.server, isDevMode); + createPythonFlaskProcess(investExe); // Wait for a response from the server before loading the app await getFlaskIsReady(); diff --git a/src/main_helpers.js b/src/main_helpers.js index bf6605a5..02785903 100644 --- a/src/main_helpers.js +++ b/src/main_helpers.js @@ -6,15 +6,14 @@ const { getLogger } = require('./logger'); const logger = getLogger(__filename.split('/').slice(-1)[0]); /** - * Find paths to local invest binaries under dev or production environments. + * Find paths to local invest executeable under dev or production environments. * * @param {boolean} isDevMode - a boolean designating dev mode or not. - * @returns {Promise} Resolves object with filepaths to invest binaries + * @returns {Promise} Resolves filepath to invest binary */ export function findInvestBinaries(isDevMode) { return new Promise(resolve => { // Binding to the invest server binary: - let serverExe; let investExe; const ext = (process.platform === 'win32') ? '.exe' : ''; @@ -25,59 +24,47 @@ export function findInvestBinaries(isDevMode) { if (fs.existsSync(investRegistryPath)) { const investRegistry = JSON.parse(fs.readFileSync(investRegistryPath)); const activeVersion = investRegistry.active; - serverExe = investRegistry.registry[activeVersion].server; investExe = investRegistry.registry[activeVersion].invest; // B) check for dev mode and an environment variable from dotenv } else if (isDevMode) { // If no dotenv vars are set, default to where this project's // build process places the binaries. - serverExe = `${process.env.SERVER || 'build/invest/server'}${ext}`; investExe = `${process.env.INVEST || 'build/invest/invest'}${ext}`; // C) point to binaries included in this app's installation. } else { const binaryPath = path.join(process.resourcesPath, 'invest'); - serverExe = path.join(binaryPath, `server${ext}`); investExe = path.join(binaryPath, `invest${ext}`); } try { - fs.accessSync(serverExe, fs.constants.X_OK); fs.accessSync(investExe, fs.constants.X_OK); } catch (error) { logger.error(error); throw error; } - logger.info(`Found invest binaries ${investExe} and ${serverExe}`); - resolve({ invest: investExe, server: serverExe }); + logger.info(`Found invest binaries ${investExe}`); + resolve(investExe); }); } /** * Spawn a child process running the Python Flask app. * - * @param {string} serverExe - path to executeable that launches flask app. + * @param {string} investExe - path to executeable that launches flask app. * @param {boolean} isDevMode - a boolean designating dev mode or not. * @returns {undefined} */ -export function createPythonFlaskProcess(serverExe, isDevMode) { - if (serverExe) { - let pythonServerProcess; - if (isDevMode && process.env.PYTHON && serverExe.endsWith('.py')) { - // A special devMode case for launching from the source code - // to facilitate debugging & development of src/server.py - pythonServerProcess = spawn(process.env.PYTHON, [serverExe]); - } else { - // The most reliable, cross-platform way to make sure spawn - // can find the exe is to pass only the command name while - // also putting it's location on the PATH: - pythonServerProcess = spawn(path.basename(serverExe), { - env: { PATH: path.dirname(serverExe) }, - }); - } +export function createPythonFlaskProcess(investExe) { + if (investExe) { + const pythonServerProcess = spawn( + path.basename(investExe), + ['serve'], + { env: { PATH: path.dirname(investExe) } } + ); logger.debug(`Started python process as PID ${pythonServerProcess.pid}`); - logger.debug(serverExe); + logger.debug(investExe); pythonServerProcess.stdout.on('data', (data) => { logger.debug(`${data}`); }); @@ -87,7 +74,7 @@ export function createPythonFlaskProcess(serverExe, isDevMode) { pythonServerProcess.on('error', (err) => { logger.error(err.stack); logger.error( - `The flask app ${serverExe} crashed or failed to start + `The flask app ${investExe} crashed or failed to start so this application must be restarted` ); throw err; diff --git a/tests/binary_tests/flaskapp.test.js b/tests/binary_tests/flaskapp.test.js index 664cc175..74ec1878 100644 --- a/tests/binary_tests/flaskapp.test.js +++ b/tests/binary_tests/flaskapp.test.js @@ -12,8 +12,8 @@ jest.setTimeout(250000); // This test is slow in CI const isDevMode = true; // otherwise need to mock process.resourcesPath beforeAll(async () => { - const binaries = await findInvestBinaries(isDevMode); - createPythonFlaskProcess(binaries.server, isDevMode); + const investExe = await findInvestBinaries(isDevMode); + createPythonFlaskProcess(investExe); // In the CI the flask app takes more than 10x as long to startup. // Especially so on macos. // So, allowing many retries, especially because the error From 07ff3573894d0c89a6bdd1ad8a9fc21221f1bca3 Mon Sep 17 00:00:00 2001 From: davemfish Date: Sat, 21 Nov 2020 15:17:32 -0800 Subject: [PATCH 04/25] fix some npm script commands that had a superflouos npx --- package.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 3c40b769..aa0a0f7b 100644 --- a/package.json +++ b/package.json @@ -6,13 +6,14 @@ "main": "src/main.js", "homepage": "./", "scripts": { - "start": "npx electron . --dev", + "start": "electron . --dev", "lint": "eslint --cache --color --ext .jsx,.js src", "test": "cross-env DEBUG_PRINT_LIMIT=10 jest --testMatch **/tests/*.test.js", "test-flask-app": "jest --runInBand --testMatch **/tests/binary_tests/flaskapp.test.js", "test-electron-app": "jest --runInBand --testMatch **/tests/binary_tests/puppet.test.js", "postinstall": "electron-builder install-app-deps", - "build": "npx ./build.js", + "fetch-invest": "node ./scripts/fetch_invest_binaries.js", + "build": "node ./build.js", "dist": "electron-builder -c.extraMetadata.main=build/main.js", "pack": "electron-builder --dir -c.extraMetadata.main=build/main.js" }, From 8e746485a7a2b60986676a6a0619cf729c14d469 Mon Sep 17 00:00:00 2001 From: davemfish Date: Mon, 23 Nov 2020 14:17:01 -0800 Subject: [PATCH 05/25] let node detect the platform for which invest binaries need to be fetched --- .github/workflows/build-electron.yml | 4 +--- package.json | 4 ++-- scripts/fetch_invest_binaries.js | 16 ++++++---------- 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build-electron.yml b/.github/workflows/build-electron.yml index c0db8d0e..e43f65c9 100644 --- a/.github/workflows/build-electron.yml +++ b/.github/workflows/build-electron.yml @@ -26,9 +26,7 @@ jobs: - name: Fetch InVEST Binaries shell: bash - run: | - npx node ./scripts/fetch_invest_binaries.js ${{matrix.os}} - unzip ./build/binaries.zip -d build/invest/ + run: npm run fetch-invest - name: Build & Release Electron application uses: samuelmeuli/action-electron-builder@v1 diff --git a/package.json b/package.json index aa0a0f7b..6bcbe9fd 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "test-flask-app": "jest --runInBand --testMatch **/tests/binary_tests/flaskapp.test.js", "test-electron-app": "jest --runInBand --testMatch **/tests/binary_tests/puppet.test.js", "postinstall": "electron-builder install-app-deps", - "fetch-invest": "node ./scripts/fetch_invest_binaries.js", + "fetch-invest": "node ./scripts/fetch_invest_binaries.js && unzip ./build/binaries.zip -d build/invest/", "build": "node ./build.js", "dist": "electron-builder -c.extraMetadata.main=build/main.js", "pack": "electron-builder --dir -c.extraMetadata.main=build/main.js" @@ -57,7 +57,7 @@ ] }, "invest": { - "hostname": "https://storage.googleapis.com", + "hostname": "https://storage.googleapis.com", "bucket": "natcap-dev-build-artifacts", "fork": "invest/davemfish", "version": "3.8.0.post1935+gb52992af", diff --git a/scripts/fetch_invest_binaries.js b/scripts/fetch_invest_binaries.js index 29701bad..376ba8a9 100644 --- a/scripts/fetch_invest_binaries.js +++ b/scripts/fetch_invest_binaries.js @@ -14,24 +14,19 @@ const path = require('path') const url = require('url') const package = require('../package') -// TODO: better to just detect the platform with node instead of passing -const args = process.argv.slice(2) -if (args.length !== 1) { - throw new Error('expected exactly 1 argument: the current OS'); -} let fileSuffix; -switch (args[0]) { - case 'windows-latest': +switch (process.platform) { + case 'win32': filePrefix = 'windows' break - case 'macos-latest': + case 'darwin': filePrefix = 'macos' break - case 'ubuntu-latest': + case 'linux': filePrefix = 'linux' break default: - throw new Error("expected argument to be in ['windows-latest, 'macos-latest', 'ubuntu-latest']") + throw new Error("expected platform to be windows, mac, or linux") } const HOSTNAME = package.invest.hostname @@ -44,6 +39,7 @@ const DESTFILE = path.resolve('build/binaries.zip'); const urladdress = url.resolve(HOSTNAME, path.join(BUCKET, FORK, VERSION, SRCFILE)) const download = function(url, dest) { + console.log(`fetching ${url}`) fs.existsSync(path.dirname(dest)) || fs.mkdirSync(path.dirname(dest)); const fileStream = fs.createWriteStream(dest) const request = https.get(url, function(response) { From 075335917d45b1b5ec9d02b18c2d6598dcdd6feb Mon Sep 17 00:00:00 2001 From: davemfish Date: Tue, 24 Nov 2020 05:42:49 -0800 Subject: [PATCH 06/25] removed the server.py module that has moved to invest --- src/server.py | 212 -------------------------------------------------- 1 file changed, 212 deletions(-) delete mode 100755 src/server.py diff --git a/src/server.py b/src/server.py deleted file mode 100755 index 769ac041..00000000 --- a/src/server.py +++ /dev/null @@ -1,212 +0,0 @@ -import codecs -import collections -from datetime import datetime -import importlib -import json -import logging -import pprint -import textwrap - -from flask import Flask -from flask import request -import natcap.invest.cli -import natcap.invest.datastack - -logging.basicConfig(level=logging.DEBUG) -LOGGER = logging.getLogger(__name__) - -app = Flask(__name__) - -# Lookup names to pass to `invest run` based on python module names -_UI_META = collections.namedtuple('UIMeta', ['run_name', 'human_name']) -MODULE_MODELRUN_MAP = { - v.pyname: _UI_META( - run_name=k, - human_name=v.humanname) - for k, v in natcap.invest.cli._MODEL_UIS.items()} - - -def shutdown_server(): - """Shutdown the flask server.""" - func = request.environ.get('werkzeug.server.shutdown') - if func is None: - raise RuntimeError('Not running with the Werkzeug Server') - func() - - -@app.route('/ready', methods=['GET']) -def get_is_ready(): - """Returns something simple to confirm the server is open.""" - return 'Flask ready' - - -@app.route('/shutdown', methods=['GET']) -def shutdown(): - """A request to this endpoint shuts down the server.""" - shutdown_server() - return 'Flask server shutting down...' - - -@app.route('/models', methods=['GET']) -def get_invest_models(): - """Gets a list of available InVEST models. - - Returns: - A JSON string - """ - LOGGER.debug('get model list') - return natcap.invest.cli.build_model_list_json() - - -@app.route('/getspec', methods=['POST']) -def get_invest_getspec(): - """Gets the ARGS_SPEC dict from an InVEST model. - - Body (JSON string): "carbon" - - Returns: - A JSON string. - """ - target_model = request.get_json() - target_module = natcap.invest.cli._MODEL_UIS[target_model].pyname - model_module = importlib.import_module(name=target_module) - LOGGER.debug(model_module.__file__) - spec = model_module.ARGS_SPEC - return json.dumps(spec) - - -@app.route('/validate', methods=['POST']) -def get_invest_validate(): - """Gets the return value of an InVEST model's validate function. - - Body (JSON string): - model_module: string (e.g. natcap.invest.carbon) - args: JSON string of InVEST model args keys and values - - Returns: - A JSON string. - """ - payload = request.get_json() - LOGGER.debug(payload) - target_module = payload['model_module'] - args_dict = json.loads(payload['args']) - LOGGER.debug(args_dict) - try: - limit_to = payload['limit_to'] - except KeyError: - limit_to = None - model_module = importlib.import_module(name=target_module) - results = model_module.validate(args_dict, limit_to=limit_to) - LOGGER.debug(results) - return json.dumps(results) - - -@app.route('/post_datastack_file', methods=['POST']) -def post_datastack_file(): - """Extracts InVEST model args from json, logfiles, or datastacks. - - Body (JSON string): path to file - - Returns: - A JSON string. - """ - filepath = request.get_json() - stack_type, stack_info = natcap.invest.datastack.get_datastack_info( - filepath) - run_name, human_name = MODULE_MODELRUN_MAP[stack_info.model_name] - result_dict = { - 'type': stack_type, - 'args': stack_info.args, - 'module_name': stack_info.model_name, - 'model_run_name': run_name, - 'model_human_name': human_name, - 'invest_version': stack_info.invest_version - } - LOGGER.debug(result_dict) - return json.dumps(result_dict) - - -@app.route('/write_parameter_set_file', methods=['POST']) -def write_parameter_set_file(): - """Writes InVEST model args keys and values to a datastack JSON file. - - Body (JSON string): - parameterSetPath: string - moduleName: string(e.g. natcap.invest.carbon) - args: JSON string of InVEST model args keys and values - relativePaths: boolean - - Returns: - A string. - """ - payload = request.get_json() - filepath = payload['parameterSetPath'] - modulename = payload['moduleName'] - args = json.loads(payload['args']) - relative_paths = payload['relativePaths'] - - natcap.invest.datastack.build_parameter_set( - args, modulename, filepath, relative=relative_paths) - return 'parameter set saved' - - -# Borrowed this function from natcap.invest.model because I assume -# that module won't persist if we eventually deprecate the Qt UI. -# TODO: we could consider refactoring this to javascript, though -# there is one call here to `natcap.invest.cli.__version__` -@app.route('/save_to_python', methods=['POST']) -def save_to_python(): - """Writes a python script with a call to an InVEST model execute function. - - Body (JSON string): - filepath: string - modelname: string (e.g. carbon) - pyname: string (e.g. natcap.invest.carbon) - args_dict: JSON string of InVEST model args keys and values - - Returns: - A string. - """ - payload = request.get_json() - save_filepath = payload['filepath'] - modelname = payload['modelname'] - pyname = payload['pyname'] - args_dict = json.loads(payload['args']) - - script_template = textwrap.dedent("""\ - # coding=UTF-8 - # ----------------------------------------------- - # Generated by InVEST {invest_version} on {today} - # Model: {modelname} - - import {py_model} - - args = {model_args} - - if __name__ == '__main__': - {py_model}.execute(args) - """) - - with codecs.open(save_filepath, 'w', encoding='utf-8') as py_file: - # cast_args = dict((unicode(key), value) for (key, value) - # in args_dict.items()) - args = pprint.pformat(args_dict, indent=4) # 4 spaces - - # Tweak formatting from pprint: - # * Bump parameter inline with starting { to next line - # * add trailing comma to last item item pair - # * add extra space to spacing before first item - args = args.replace('{', '{\n ') - args = args.replace('}', ',\n}') - py_file.write(script_template.format( - invest_version=natcap.invest.cli.__version__, - today=datetime.now().strftime('%c'), - modelname=modelname, - py_model=pyname, - model_args=args)) - - return 'python script saved' - -if __name__ == '__main__': - print('Running Flask App') - app.run() From 5e5a9050dafdb78f7f6447982ce6edf11673c22a Mon Sep 17 00:00:00 2001 From: davemfish Date: Tue, 24 Nov 2020 05:46:57 -0800 Subject: [PATCH 07/25] readme notes --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index 99f4add1..b837b37e 100644 --- a/readme.md +++ b/readme.md @@ -10,7 +10,7 @@ workflows of an InVEST user. * `npm start` but first, * `npm install` -* bind to an `invest` executeable +* bind to an `invest` executeable (see package.json "invest" for a compatible version) In production, the invest exe comes from prebuilt binaries that are an artifact of the `invest` build process. From 36d5444c910f1d478d53c982964341418aa9b4b6 Mon Sep 17 00:00:00 2001 From: davemfish Date: Tue, 24 Nov 2020 06:01:20 -0800 Subject: [PATCH 08/25] removing the pyinstaller script --- invest-flask.spec | 104 ---------------------------------------------- 1 file changed, 104 deletions(-) delete mode 100644 invest-flask.spec diff --git a/invest-flask.spec b/invest-flask.spec deleted file mode 100644 index cac8b52b..00000000 --- a/invest-flask.spec +++ /dev/null @@ -1,104 +0,0 @@ -# coding=UTF-8 -# -*- mode: python -*- -import sys -import os -import itertools -import glob -from PyInstaller.compat import is_win, is_darwin, is_linux - -if not is_win: - # Windows builds on Actions don't use conda - conda_env = os.environ['CONDA_PREFIX'] -workbench_dir = os.getcwd() -invest_dir = os.path.join(workbench_dir, 'invest') -block_cipher = None -invest_exename = 'invest' -server_exename = 'server' - -kwargs = { - 'hookspath': [os.path.join(invest_dir, 'exe', 'hooks')], - 'excludes': None, - 'pathex': sys.path, - 'runtime_hooks': [os.path.join(invest_dir, 'exe', 'hooks', 'rthook.py')], - 'hiddenimports': [ - 'natcap', - 'natcap.invest', - 'yaml', - 'distutils', - 'distutils.dist', - 'rtree', # mac builds aren't picking up rtree by default. - 'pkg_resources.py2_warn' - ], - 'cipher': block_cipher, -} - -cli_file = os.path.join(invest_dir, 'src', 'natcap', 'invest', 'cli.py') -invest_a = Analysis([cli_file], **kwargs) - -# This path matches the directory setup in Makefile (make fetch) -flask_run_file = os.path.join(workbench_dir, 'src', 'server.py') -# All the same kwargs apply because this app also imports natcap.invest -server_a = Analysis([flask_run_file], **kwargs) - -MERGE((invest_a, invest_exename, invest_exename), - (server_a, server_exename, server_exename)) - -# Compress pyc and pyo Files into ZlibArchive Objects -invest_pyz = PYZ(invest_a.pure, invest_a.zipped_data, cipher=block_cipher) -server_pyz = PYZ(server_a.pure, server_a.zipped_data, cipher=block_cipher) - -# Create the executable file. -if not is_win: - # add rtree dependency dynamic libraries from conda environment - invest_a.binaries += [ - (os.path.basename(name), name, 'BINARY') for name in - glob.glob(os.path.join(conda_env, 'lib/libspatialindex*'))] -else: - # Adapted from - # https://shanetully.com/2013/08/cross-platform-deployment-of-python-applications-with-pyinstaller/ - # Supposed to gather the mscvr/p DLLs from the local system before - # packaging. Skirts the issue of us needing to keep them under version - # control. - invest_a.binaries += [ - ('msvcp90.dll', 'C:\\Windows\\System32\\msvcp90.dll', 'BINARY'), - ('msvcr90.dll', 'C:\\Windows\\System32\\msvcr90.dll', 'BINARY') - ] - - # .exe extension is required if we're on windows. - invest_exename += '.exe' - server_exename += '.exe' - -invest_exe = EXE( - invest_pyz, - invest_a.scripts, - name=invest_exename, - exclude_binaries=True, - debug=False, - strip=False, - upx=False, - console=True) - -server_exe = EXE( - server_pyz, - server_a.scripts, - name=server_exename, - exclude_binaries=True, - debug=False, - bootloader_ignore_signals=False, - strip=False, - upx=False, - console=True) - -# Collect Files into Distributable Folder/File -invest_dist = COLLECT( - invest_exe, - invest_a.binaries, - invest_a.zipfiles, - invest_a.datas, - server_exe, - server_a.binaries, - server_a.zipfiles, - server_a.datas, - name="invest", # name of the output folder - strip=False, - upx=False) From 4d217e9815042c0258ae8b43b310e71197271504 Mon Sep 17 00:00:00 2001 From: davemfish Date: Mon, 30 Nov 2020 06:02:24 -0800 Subject: [PATCH 09/25] removing an innacurate comment --- tests/binary_tests/flaskapp.test.js | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/binary_tests/flaskapp.test.js b/tests/binary_tests/flaskapp.test.js index 74ec1878..eee507e8 100644 --- a/tests/binary_tests/flaskapp.test.js +++ b/tests/binary_tests/flaskapp.test.js @@ -4,7 +4,6 @@ import * as server_requests from '../../src/server_requests'; import { findInvestBinaries, createPythonFlaskProcess } from '../../src/main_helpers'; import { argsDictFromObject } from '../../src/utils'; -// If SERVER is defined in .env, this appends it to global process.env const dotenv = require('dotenv'); dotenv.config(); From 11bd67fc9113c7ca695ee8b277a20cf0cefd1a52 Mon Sep 17 00:00:00 2001 From: davemfish Date: Mon, 30 Nov 2020 06:39:25 -0800 Subject: [PATCH 10/25] linting a script --- scripts/fetch_invest_binaries.js | 102 ++++++++++++++++--------------- 1 file changed, 52 insertions(+), 50 deletions(-) diff --git a/scripts/fetch_invest_binaries.js b/scripts/fetch_invest_binaries.js index 376ba8a9..a0764a94 100644 --- a/scripts/fetch_invest_binaries.js +++ b/scripts/fetch_invest_binaries.js @@ -1,59 +1,61 @@ -/** - * 2020-07-30 - the workbench build process no longer depends on - * pre-built binaries, instead building them itself, so this script - * is not being used anymore. - * - * This is a utility script for fetching prebuilt invest binaries - * that include the server.py module and thus are compatible with the - * workbench. - */ - -const https = require('https') -const fs = require('fs') -const path = require('path') -const url = require('url') -const package = require('../package') +const https = require('https'); +const fs = require('fs'); +const path = require('path'); +const url = require('url'); +const pkg = require('../package'); -let fileSuffix; +let filePrefix; +// The invest build process only builds for these OS switch (process.platform) { - case 'win32': - filePrefix = 'windows' - break - case 'darwin': - filePrefix = 'macos' - break - case 'linux': - filePrefix = 'linux' - break - default: - throw new Error("expected platform to be windows, mac, or linux") + case 'win32': + filePrefix = 'windows'; + break; + case 'darwin': + filePrefix = 'macos'; + break; + // case 'linux': + // filePrefix = 'linux'; + // break; + default: + throw new Error( + `No prebuilt invest binaries are available for ${process.platform}` + ); } -const HOSTNAME = package.invest.hostname -const BUCKET = package.invest.bucket -const FORK = package.invest.fork -const VERSION = package.invest.version -const SRCFILE = `${filePrefix}_invest_binaries.zip` +const HOSTNAME = pkg.invest.hostname; +const BUCKET = pkg.invest.bucket; +const FORK = pkg.invest.fork; +const VERSION = pkg.invest.version; +const SRCFILE = `${filePrefix}_invest_binaries.zip`; const DESTFILE = path.resolve('build/binaries.zip'); -const urladdress = url.resolve(HOSTNAME, path.join(BUCKET, FORK, VERSION, SRCFILE)) +const urladdress = url.resolve( + HOSTNAME, path.join(BUCKET, FORK, VERSION, SRCFILE) +); -const download = function(url, dest) { - console.log(`fetching ${url}`) - fs.existsSync(path.dirname(dest)) || fs.mkdirSync(path.dirname(dest)); - const fileStream = fs.createWriteStream(dest) - const request = https.get(url, function(response) { - console.log(response.statusCode) - if (response.statusCode != 200) { - fileStream.close() - console.log(url) - return - } - response.pipe(fileStream) - fileStream.on('finish', function() { - fileStream.close() - }) - }) +/** + * Download a file from src to dest. + * + * @param {string} src - url for a single publicly hosted file + * @param {string} dest - local path for saving the file + */ +function download(src, dest) { + console.log(`downloading ${url}`); + fs.existsSync(path.dirname(dest)) || fs.mkdirSync(path.dirname(dest)); + const fileStream = fs.createWriteStream(dest); + https.get(src, (response) => { + console.log(response.statusCode); + if (response.statusCode !== 200) { + fileStream.close(); + return; + } + response.pipe(fileStream); + fileStream.on('finish', () => { + fileStream.close(); + }); + }).on('error', (e) => { + console.log(e); + }); } -download(urladdress, DESTFILE) +download(urladdress, DESTFILE); From a3468b049da07f2b517f7c09175b35d15a978cd7 Mon Sep 17 00:00:00 2001 From: davemfish Date: Mon, 30 Nov 2020 09:20:56 -0800 Subject: [PATCH 11/25] drop binary builds for ubuntu. fixes #71. --- .github/workflows/build-electron.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-electron.yml b/.github/workflows/build-electron.yml index e43f65c9..eb39ec44 100644 --- a/.github/workflows/build-electron.yml +++ b/.github/workflows/build-electron.yml @@ -9,7 +9,7 @@ jobs: max-parallel: 4 fail-fast: false matrix: - os: [macos-latest, windows-latest, ubuntu-latest] + os: [macos-latest, windows-latest] node-version: [12.x] steps: From eda948e9de0602dd5ae9db457adfb1bc08d7db19 Mon Sep 17 00:00:00 2001 From: davemfish Date: Mon, 30 Nov 2020 09:45:05 -0800 Subject: [PATCH 12/25] bump invest version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 6bcbe9fd..151088b5 100644 --- a/package.json +++ b/package.json @@ -60,7 +60,7 @@ "hostname": "https://storage.googleapis.com", "bucket": "natcap-dev-build-artifacts", "fork": "invest/davemfish", - "version": "3.8.0.post1935+gb52992af", + "version": "3.8.0.post1941+g289301ee", "target": { "macos": "mac_invest_binaries.zip", "windows": "windows_invest_binaries.zip" From 6cf63fc891491c62ca056bc27f51c37c0c54b2ce Mon Sep 17 00:00:00 2001 From: davemfish Date: Mon, 30 Nov 2020 11:25:29 -0800 Subject: [PATCH 13/25] bumping invest version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 151088b5..bb4b601a 100644 --- a/package.json +++ b/package.json @@ -60,7 +60,7 @@ "hostname": "https://storage.googleapis.com", "bucket": "natcap-dev-build-artifacts", "fork": "invest/davemfish", - "version": "3.8.0.post1941+g289301ee", + "version": "3.8.0.post1942+gee99530c", "target": { "macos": "mac_invest_binaries.zip", "windows": "windows_invest_binaries.zip" From 47b572c50c914fe814b1f72c72c884720c6d71bc Mon Sep 17 00:00:00 2001 From: davemfish Date: Mon, 30 Nov 2020 12:26:43 -0800 Subject: [PATCH 14/25] debugging the fetch script for MacOS --- scripts/fetch_invest_binaries.js | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/scripts/fetch_invest_binaries.js b/scripts/fetch_invest_binaries.js index a0764a94..ac1523fa 100644 --- a/scripts/fetch_invest_binaries.js +++ b/scripts/fetch_invest_binaries.js @@ -6,16 +6,14 @@ const pkg = require('../package'); let filePrefix; // The invest build process only builds for these OS +// The prefixes on the zip file are defined by invest's Makefile $OSNAME switch (process.platform) { case 'win32': filePrefix = 'windows'; break; case 'darwin': - filePrefix = 'macos'; + filePrefix = 'mac'; break; - // case 'linux': - // filePrefix = 'linux'; - // break; default: throw new Error( `No prebuilt invest binaries are available for ${process.platform}` @@ -40,11 +38,11 @@ const urladdress = url.resolve( * @param {string} dest - local path for saving the file */ function download(src, dest) { - console.log(`downloading ${url}`); + console.log(`downloading ${src}`); fs.existsSync(path.dirname(dest)) || fs.mkdirSync(path.dirname(dest)); const fileStream = fs.createWriteStream(dest); https.get(src, (response) => { - console.log(response.statusCode); + console.log(`http status: ${response.statusCode}`); if (response.statusCode !== 200) { fileStream.close(); return; From c7e8d8c6c57aa610d69e743bda2c76659d49c328 Mon Sep 17 00:00:00 2001 From: davemfish Date: Mon, 30 Nov 2020 12:52:08 -0800 Subject: [PATCH 15/25] update invest version after pushing invest tags --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index bb4b601a..75c4840c 100644 --- a/package.json +++ b/package.json @@ -60,7 +60,7 @@ "hostname": "https://storage.googleapis.com", "bucket": "natcap-dev-build-artifacts", "fork": "invest/davemfish", - "version": "3.8.0.post1942+gee99530c", + "version": "3.8.9.post1008+gee99530c", "target": { "macos": "mac_invest_binaries.zip", "windows": "windows_invest_binaries.zip" From f00dff369f896d0b7f1f6d02598f6c55352532c5 Mon Sep 17 00:00:00 2001 From: davemfish Date: Mon, 30 Nov 2020 14:53:53 -0800 Subject: [PATCH 16/25] debugging YAML to skip the macos puppet test. --- .github/workflows/build-electron.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-electron.yml b/.github/workflows/build-electron.yml index eb39ec44..e83f5c8c 100644 --- a/.github/workflows/build-electron.yml +++ b/.github/workflows/build-electron.yml @@ -46,7 +46,7 @@ jobs: - name: Test electron app with puppeteer uses: GabrielBB/xvfb-action@v1 - if: ${{ matrix.os }} != 'macos-latest' + if: matrix.os != 'macos-latest' with: run: npm run test-electron-app From 7ea16cd3175b2555cd83d53df02ac701469bdfd2 Mon Sep 17 00:00:00 2001 From: davemfish Date: Tue, 1 Dec 2020 06:26:51 -0800 Subject: [PATCH 17/25] cleaning up readme --- readme.md | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/readme.md b/readme.md index b837b37e..9da36303 100644 --- a/readme.md +++ b/readme.md @@ -8,25 +8,28 @@ workflows of an InVEST user. ## To develop and launch this Application -* `npm start` but first, * `npm install` * bind to an `invest` executeable (see package.json "invest" for a compatible version) In production, the invest exe comes from prebuilt binaries that are an artifact of the `invest` build process. -For development, there are options: -* A. Duplicate the production setup by fetching prebuilt binaries +For development, choose either: +* **A.** Duplicate the production setup by fetching prebuilt binaries `npm run fetch-invest` -* B. use an any other locally installed, compatible, invest CLI (e.g. from an active python environment). To configure this, see `.env-example` +* **B.** Use an any other locally installed, compatible, invest CLI (e.g. from a local python environment). To configure this, see `.env-example` +* `npm start` -`npm run build` -- calls babel to transpile ES6 and jsx code to commonjs +## To package this app for distribution -`npm run dist` -- packages build source into an electron application using electron-builder +`npm run build` - calls babel to transpile ES6 and jsx code to commonjs + - moves other resources (CSS, JSON) to the build directory + +`npm run dist` - packages build source into an electron application using electron-builder ### To run various scripts and local programs -See the "scripts" section of `package.json` and run of them like: +See the "scripts" section of `package.json` and run them like: `npm run lint` `npm run test` From 1c54f78361c6adb46926f37d68365edf0fc8be60 Mon Sep 17 00:00:00 2001 From: davemfish Date: Tue, 1 Dec 2020 06:30:03 -0800 Subject: [PATCH 18/25] cleaning up readme --- readme.md | 1 + 1 file changed, 1 insertion(+) diff --git a/readme.md b/readme.md index 9da36303..b8d89618 100644 --- a/readme.md +++ b/readme.md @@ -18,6 +18,7 @@ For development, choose either: `npm run fetch-invest` * **B.** Use an any other locally installed, compatible, invest CLI (e.g. from a local python environment). To configure this, see `.env-example` + * `npm start` ## To package this app for distribution From cedf89d10831aae15c18b82005714260b603d3fc Mon Sep 17 00:00:00 2001 From: davemfish Date: Tue, 1 Dec 2020 06:31:56 -0800 Subject: [PATCH 19/25] cleaning up readme --- readme.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readme.md b/readme.md index b8d89618..50b9f18e 100644 --- a/readme.md +++ b/readme.md @@ -14,9 +14,9 @@ workflows of an InVEST user. In production, the invest exe comes from prebuilt binaries that are an artifact of the `invest` build process. For development, choose either: -* **A.** Duplicate the production setup by fetching prebuilt binaries ++ **A.** Duplicate the production setup by fetching prebuilt binaries `npm run fetch-invest` -* **B.** Use an any other locally installed, compatible, invest CLI (e.g. from a local python environment). To configure this, see `.env-example` ++ **B.** Use an any other locally installed, compatible, invest CLI (e.g. from a local python environment). To configure this, see `.env-example` * `npm start` From 80ff6d27c3c206f87d97269c15950e28fe72abd9 Mon Sep 17 00:00:00 2001 From: Dave Fisher Date: Tue, 1 Dec 2020 06:35:58 -0800 Subject: [PATCH 20/25] Update readme.md --- readme.md | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/readme.md b/readme.md index 50b9f18e..4478b259 100644 --- a/readme.md +++ b/readme.md @@ -14,17 +14,14 @@ workflows of an InVEST user. In production, the invest exe comes from prebuilt binaries that are an artifact of the `invest` build process. For development, choose either: -+ **A.** Duplicate the production setup by fetching prebuilt binaries -`npm run fetch-invest` -+ **B.** Use an any other locally installed, compatible, invest CLI (e.g. from a local python environment). To configure this, see `.env-example` - + **A.** Duplicate the production setup by fetching prebuilt binaries `npm run fetch-invest` + **B.** Use an any other locally installed, compatible, invest CLI (e.g. from a local python environment). To configure this, see `.env-example` * `npm start` ## To package this app for distribution -`npm run build` - calls babel to transpile ES6 and jsx code to commonjs - - moves other resources (CSS, JSON) to the build directory +`npm run build` - calls babel to transpile ES6 and jsx code to commonjs; moves other resources (CSS, JSON) to the build directory `npm run dist` - packages build source into an electron application using electron-builder From d468d512343fb8b78560c1704bdde413f32327ba Mon Sep 17 00:00:00 2001 From: davemfish Date: Thu, 10 Dec 2020 12:13:30 -0800 Subject: [PATCH 21/25] specify a port number when starting the flask subprocess --- src/main.js | 5 ++++- src/main_helpers.js | 2 +- src/server_requests.js | 17 ++++++++--------- tests/binary_tests/flaskapp.test.js | 4 ++++ 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/main.js b/src/main.js index a1311893..e08716ef 100644 --- a/src/main.js +++ b/src/main.js @@ -20,7 +20,10 @@ const { getLogger } = require('./logger'); const logger = getLogger(__filename.split('/').slice(-1)[0]); -const PORT = (process.env.PORT || '5000').trim(); +// This could be optionally configured already in '.env' +if (!process.env.PORT) { + process.env.PORT = 56789; +} // Keep a global reference of the window object, if you don't, the window will // be closed automatically when the JavaScript object is garbage collected. diff --git a/src/main_helpers.js b/src/main_helpers.js index 02785903..e8c04b65 100644 --- a/src/main_helpers.js +++ b/src/main_helpers.js @@ -59,7 +59,7 @@ export function createPythonFlaskProcess(investExe) { if (investExe) { const pythonServerProcess = spawn( path.basename(investExe), - ['serve'], + ['serve', '--port', process.env.PORT], { env: { PATH: path.dirname(investExe) } } ); diff --git a/src/server_requests.js b/src/server_requests.js index 44344dc2..d8b2056c 100644 --- a/src/server_requests.js +++ b/src/server_requests.js @@ -2,7 +2,6 @@ import fetch from 'node-fetch'; import { getLogger } from './logger'; const logger = getLogger(__filename.split('/').slice(-1)[0]); -const PORT = process.env.PORT || '5000'; const HOSTNAME = 'http://localhost'; /** Find out if the Flask server is online, waiting until it is. @@ -16,7 +15,7 @@ const HOSTNAME = 'http://localhost'; */ export function getFlaskIsReady({ i = 0, retries = 21 } = {}) { return ( - fetch(`${HOSTNAME}:${PORT}/ready`, { + fetch(`${HOSTNAME}:${process.env.PORT}/ready`, { method: 'get', }) .then((response) => response.text()) @@ -47,7 +46,7 @@ export function getFlaskIsReady({ i = 0, retries = 21 } = {}) { */ export function getInvestList() { return ( - fetch(`${HOSTNAME}:${PORT}/models`, { + fetch(`${HOSTNAME}:${process.env.PORT}/models`, { method: 'get', }) .then((response) => response.json()) @@ -63,7 +62,7 @@ export function getInvestList() { */ export function getSpec(payload) { return ( - fetch(`${HOSTNAME}:${PORT}/getspec`, { + fetch(`${HOSTNAME}:${process.env.PORT}/getspec`, { method: 'post', body: JSON.stringify(payload), headers: { 'Content-Type': 'application/json' }, @@ -84,7 +83,7 @@ export function getSpec(payload) { */ export function fetchValidation(payload) { return ( - fetch(`${HOSTNAME}:${PORT}/validate`, { + fetch(`${HOSTNAME}:${process.env.PORT}/validate`, { method: 'post', body: JSON.stringify(payload), headers: { 'Content-Type': 'application/json' }, @@ -102,7 +101,7 @@ export function fetchValidation(payload) { */ export function fetchDatastackFromFile(payload) { return ( - fetch(`${HOSTNAME}:${PORT}/post_datastack_file`, { + fetch(`${HOSTNAME}:${process.env.PORT}/post_datastack_file`, { method: 'post', body: JSON.stringify(payload), headers: { 'Content-Type': 'application/json' }, @@ -125,7 +124,7 @@ export function fetchDatastackFromFile(payload) { */ export function saveToPython(payload) { return ( - fetch(`${HOSTNAME}:${PORT}/save_to_python`, { + fetch(`${HOSTNAME}:${process.env.PORT}/save_to_python`, { method: 'post', body: JSON.stringify(payload), headers: { 'Content-Type': 'application/json' }, @@ -149,7 +148,7 @@ export function saveToPython(payload) { */ export function writeParametersToFile(payload) { return ( - fetch(`${HOSTNAME}:${PORT}/write_parameter_set_file`, { + fetch(`${HOSTNAME}:${process.env.PORT}/write_parameter_set_file`, { method: 'post', body: JSON.stringify(payload), headers: { 'Content-Type': 'application/json' }, @@ -167,7 +166,7 @@ export function writeParametersToFile(payload) { */ export function shutdownPythonProcess() { return ( - fetch(`http://localhost:${PORT}/shutdown`, { + fetch(`http://localhost:${process.env.PORT}/shutdown`, { method: 'get', }) .then((response) => response.text()) diff --git a/tests/binary_tests/flaskapp.test.js b/tests/binary_tests/flaskapp.test.js index eee507e8..e595a1d0 100644 --- a/tests/binary_tests/flaskapp.test.js +++ b/tests/binary_tests/flaskapp.test.js @@ -6,6 +6,10 @@ import { argsDictFromObject } from '../../src/utils'; const dotenv = require('dotenv'); dotenv.config(); +// This could be optionally configured already in '.env' +if (!process.env.PORT) { + process.env.PORT = 56788; +} jest.setTimeout(250000); // This test is slow in CI From 55add4215af8150e0915424a5201f07615c41691 Mon Sep 17 00:00:00 2001 From: davemfish Date: Thu, 10 Dec 2020 12:26:38 -0800 Subject: [PATCH 22/25] bump the invest version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 75c4840c..88d87ca8 100644 --- a/package.json +++ b/package.json @@ -60,7 +60,7 @@ "hostname": "https://storage.googleapis.com", "bucket": "natcap-dev-build-artifacts", "fork": "invest/davemfish", - "version": "3.8.9.post1008+gee99530c", + "version": "3.8.9.post1010+g79ba20e6", "target": { "macos": "mac_invest_binaries.zip", "windows": "windows_invest_binaries.zip" From 10330d42f4a63e979f01e3285290210920eeef2f Mon Sep 17 00:00:00 2001 From: davemfish Date: Mon, 4 Jan 2021 11:10:36 -0800 Subject: [PATCH 23/25] bump invest version after server rename --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 88d87ca8..465a4d68 100644 --- a/package.json +++ b/package.json @@ -60,7 +60,7 @@ "hostname": "https://storage.googleapis.com", "bucket": "natcap-dev-build-artifacts", "fork": "invest/davemfish", - "version": "3.8.9.post1010+g79ba20e6", + "version": "3.8.9.post1011+g17d40ce9", "target": { "macos": "mac_invest_binaries.zip", "windows": "windows_invest_binaries.zip" From 5fe3c2e14eb0565eab41d7c894e77cfadd1d63d3 Mon Sep 17 00:00:00 2001 From: davemfish Date: Tue, 5 Jan 2021 10:26:22 -0800 Subject: [PATCH 24/25] bump invest version to find a successful build --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 465a4d68..141768dd 100644 --- a/package.json +++ b/package.json @@ -60,7 +60,7 @@ "hostname": "https://storage.googleapis.com", "bucket": "natcap-dev-build-artifacts", "fork": "invest/davemfish", - "version": "3.8.9.post1011+g17d40ce9", + "version": "3.8.9.post1123+g025f0689", "target": { "macos": "mac_invest_binaries.zip", "windows": "windows_invest_binaries.zip" From 9e42849be101bc25d74e5f5ba6e8b7b73930e14a Mon Sep 17 00:00:00 2001 From: davemfish Date: Fri, 8 Jan 2021 10:42:02 -0800 Subject: [PATCH 25/25] bump to an invest build with working mac builds --- package.json | 2 +- readme.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 141768dd..e6878b2f 100644 --- a/package.json +++ b/package.json @@ -60,7 +60,7 @@ "hostname": "https://storage.googleapis.com", "bucket": "natcap-dev-build-artifacts", "fork": "invest/davemfish", - "version": "3.8.9.post1123+g025f0689", + "version": "3.8.9.post1130+gdd914a7a", "target": { "macos": "mac_invest_binaries.zip", "windows": "windows_invest_binaries.zip" diff --git a/readme.md b/readme.md index 4478b259..6f41ab19 100644 --- a/readme.md +++ b/readme.md @@ -15,7 +15,7 @@ In production, the invest exe comes from prebuilt binaries that are an artifact For development, choose either: **A.** Duplicate the production setup by fetching prebuilt binaries `npm run fetch-invest` - **B.** Use an any other locally installed, compatible, invest CLI (e.g. from a local python environment). To configure this, see `.env-example` + **B.** Use any other locally installed, compatible, invest CLI (e.g. from a local python environment). To configure this, see `.env-example` * `npm start`