diff --git a/bin/deepforge b/bin/deepforge index 327b90497..7d04d44c7 100755 --- a/bin/deepforge +++ b/bin/deepforge @@ -1,4 +1,6 @@ #!/usr/bin/env node +const childProcess = require('child_process'); +const Conda = require('../utils/conda-utils'); const os = require('os'), IS_WINDOWS = os.type() === 'WINDOWS_NT', SHELL = IS_WINDOWS ? true : '/bin/bash', @@ -8,7 +10,6 @@ const os = require('os'), var Command = require('commander').Command, tcpPortUsed = require('tcp-port-used'), program = new Command(), - childProcess = require('child_process'), rawSpawn = childProcess.spawn, Q = require('q'), execSync = childProcess.execSync, @@ -395,6 +396,63 @@ program program .command('extensions ', 'Manage deepforge extensions'); +program + .command('create-env') + .description('Create conda environment(s) with DeepForge python dependencies') + .option('-n, --name ', 'Name of environment to create') + .option('-s, --server', 'Create environment with server dependencies') + .option('-w, --worker', 'Create environment with worker dependencies') + .option('-f, --force', 'Overwrite any existing environments') + .action(async cmd => { + const createBoth = !cmd.server && !cmd.worker; + if (createBoth) { + cmd.server = cmd.worker = true; + } + + const extender = require('../utils/extender'); + const extensionData = extender.getExtensionsConfig(); + const libraries = Object.values(extensionData.Library); + const dirs = libraries.map(lib => lib.project.root); + const name = typeof cmd.name === 'string' ? cmd.name : 'deepforge'; + + try { + if (cmd.server) { + const serverEnvName = createBoth ? `${name}-server` : name; + await createEnvFromDirs(serverEnvName, dirs, 'server', cmd.force); + } + if (cmd.worker) { + await createEnvFromDirs(name, dirs, 'worker', cmd.force); + } + } catch (errOrExitCode) { + const msg = '\n\nUnable to create environment.'; + if (errOrExitCode instanceof Error) { + console.log(`${msg} An error occurred: ${errOrExitCode}`); + } else { + console.log(`${msg} Conda exited with exit code: ${errOrExitCode}`); + } + } + }); + +async function createEnvFromDirs(name, dirs, type, force=false) { + const envFiles = getCondaEnvFiles(dirs, type); + + const baseEnvFile = `environment.${type}.yml`; + const flags = `--name ${name} --file ${baseEnvFile}${force ? ' --force' : ''}`; + await Conda.spawn(`env create ${flags}`); + for (let i = 0; i < envFiles.length; i++) { + const envFile = envFiles[i]; + await Conda.spawn(`env update -n ${name} --file ${envFile}`); + } +} + +function getCondaEnvFiles(dirs, type) { + const validEnvFilenames = ['environment.yml', `environment.${type}.yml`]; + const envFiles = dirs + .flatMap(dirname => validEnvFilenames.map(file => path.join(dirname, file))) + .filter(filepath => exists.sync(filepath)); + + return envFiles; +} // user-management program.command('users', 'Manage deepforge users.'); diff --git a/package.json b/package.json index bde94000a..5edc2ff73 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ }, "scripts": { "start": "node ./bin/deepforge start", - "postinstall": "node utils/reinstall-extensions.js && node utils/build-job-utils.js", + "postinstall": "node utils/reinstall-extensions.js && node utils/build-job-utils.js && ./bin/deepforge create-env", "start-dev": "NODE_ENV=dev node ./bin/deepforge start", "start-authenticated": "NODE_ENV=production ./bin/deepforge start", "server": "node ./bin/deepforge start --server", diff --git a/test/unit/external-utils/conda-utils.spec.js b/test/unit/external-utils/conda-utils.spec.js index 89b17069f..9ae144278 100644 --- a/test/unit/external-utils/conda-utils.spec.js +++ b/test/unit/external-utils/conda-utils.spec.js @@ -1,24 +1,29 @@ -describe('CondaUtils', function () { - const condaUtils = require('../../../utils/conda-utils'), +describe.only('CondaUtils', function () { + const conda = require('../../../utils/conda-utils'), expect = require('chai').expect, path = require('path'), - ENV_FILE = path.join(__dirname, '..', '..', '..', 'environment.yml'); + ENV_FILE = path.join(__dirname, '..', '..', '..', 'environment.server.yml'); it('should find executable conda', () => { - expect(condaUtils.checkConda).to.not.throw(); + expect(conda.check).to.not.throw(); }); - it('should throw an error when creating from a missing environment file', () => { - const badCreateFunc = () => { - condaUtils.createOrUpdateEnvironment('dummyfile'); - }; - expect(badCreateFunc).to.throw(); + it('should throw an error when creating from a missing environment file', async () => { + const badCreateFunc = () => conda.createOrUpdateEnvironment('dummyfile'); + await shouldThrow(badCreateFunc); }); - it('should not throw an error from a proper environment file', () => { - const createFunc = () => { - condaUtils.createOrUpdateEnvironment(ENV_FILE); - }; - expect(createFunc).to.not.throw(); + it('should not throw an error from a proper environment file', async function() { + this.timeout(5000); + await conda.createOrUpdateEnvironment(ENV_FILE); }); -}); \ No newline at end of file + + async function shouldThrow(fn) { + try { + await fn(); + } catch (err) { + return err; + } + throw new Error('Function did not throw an exception.'); + } +}); diff --git a/utils/conda-utils.js b/utils/conda-utils.js index 41f0bdb83..634811292 100644 --- a/utils/conda-utils.js +++ b/utils/conda-utils.js @@ -1,6 +1,8 @@ /*eslint-env node*/ /*eslint-disable no-console*/ 'use strict'; +const Conda = {}; + const {spawnSync, spawn} = require('child_process'), os = require('os'), path = require('path'), @@ -33,7 +35,7 @@ const dumpYAML = function (environment, envFileName) { return envFileName; }; -const checkConda = function () { +Conda.check = function () { const conda = spawnSyncCondaProcess(['-V']); if (conda.status !== 0) { throw new Error(`Please install conda before continuing. ${conda.stderr.toString()}`); @@ -41,7 +43,7 @@ const checkConda = function () { }; -const createOrUpdateEnvironment = function (envFile, envName) { +Conda.createOrUpdateEnvironment = async function (envFile, envName) { const env = yaml.safeLoad(fs.readFileSync(envFile, 'utf8')); if (envName && envName !== env.name) { env.name = envName; @@ -49,23 +51,25 @@ const createOrUpdateEnvironment = function (envFile, envName) { } const createOrUpdate = envExists(env.name) ? 'update' : 'create'; console.log(`Environment ${env.name} will be ${createOrUpdate}d.`); - spawnCondaProcess(['env', createOrUpdate, '--file', envFile], - `Successfully ${createOrUpdate}d the environment ${env.name}`); - + await Conda.spawn(`env ${createOrUpdate} --file ${envFile}`); + console.log(`Successfully ${createOrUpdate}d the environment ${env.name}`); }; -const spawnCondaProcess = function (args, onCompleteMessage, onErrorMessage) { - const condaProcess = spawn(CONDA_COMMAND, args, { +Conda.spawn = function (command) { + const condaProcess = spawn(CONDA_COMMAND, command.split(' '), { shell: SHELL }); condaProcess.stdout.pipe(process.stdout); condaProcess.stderr.pipe(process.stderr); - condaProcess.on('exit', (code) => { - if(code !== 0){ - throw new Error(onErrorMessage || 'Spawned conda process failed.'); - } - console.log(onCompleteMessage || 'Spawned conda process executed successfully'); + + return new Promise((resolve, reject) => { + condaProcess.on('exit', (code) => { + if(code !== 0){ + return reject(code); + } + resolve(); + }); }); }; @@ -75,6 +79,4 @@ const spawnSyncCondaProcess = function (args) { }); }; -const CondaManager = {checkConda, createOrUpdateEnvironment}; - -module.exports = CondaManager; +module.exports = Conda; diff --git a/utils/extender.js b/utils/extender.js index b7239da73..7be28245f 100644 --- a/utils/extender.js +++ b/utils/extender.js @@ -153,7 +153,8 @@ let updateTemplateFile = (tplPath, type) => { fs.writeFileSync(dstPath, formatsIndex); }; -var makeInstallFor = function(typeCfg) { +function makeInstallFor(typeCfg) { + allExtConfigs[typeCfg.type] = allExtConfigs[typeCfg.type] || {}; var saveExtensions = () => { // regenerate the format.js file from the template var installedExts = values(allExtConfigs[typeCfg.type]), @@ -209,7 +210,7 @@ var makeInstallFor = function(typeCfg) { // Re-generate template file saveExtensions(); }; -}; +} //var PLUGIN_ROOT = path.join(__dirname, '..', 'src', 'plugins', 'Export'); //makeInstallFor({