From 683242f84697184157481dd03bb8e3474a340d92 Mon Sep 17 00:00:00 2001 From: Chengmin Chi Date: Tue, 14 Jan 2020 19:43:33 +0800 Subject: [PATCH 1/4] Allow NaN in metrics data --- src/nni_manager/core/nniDataStore.ts | 5 +++-- src/nni_manager/core/sqlDatabase.ts | 7 ++++--- src/nni_manager/package.json | 2 ++ src/nni_manager/yarn.lock | 16 +++++++++++++--- src/sdk/pynni/nni/msg_dispatcher.py | 2 +- src/sdk/pynni/nni/platform/local.py | 2 +- src/sdk/pynni/nni/trial.py | 5 ++--- src/webui/package.json | 2 ++ .../src/components/trial-detail/TableList.tsx | 10 +++++----- src/webui/src/static/function.ts | 13 +++++++++++-- src/webui/src/static/model/trial.ts | 10 +++++----- src/webui/yarn.lock | 10 ++++++++++ test/config_test/examples/mnist-keras.test.yml | 2 +- 13 files changed, 60 insertions(+), 26 deletions(-) diff --git a/src/nni_manager/core/nniDataStore.ts b/src/nni_manager/core/nniDataStore.ts index 47134430d7..2d0dab8bee 100644 --- a/src/nni_manager/core/nniDataStore.ts +++ b/src/nni_manager/core/nniDataStore.ts @@ -4,6 +4,7 @@ 'use strict'; import * as assert from 'assert'; +import * as JSON5 from 'json5'; import { Deferred } from 'ts-deferred'; import * as component from '../common/component'; @@ -131,7 +132,7 @@ class NNIDataStore implements DataStore { } public async storeMetricData(trialJobId: string, data: string): Promise { - const metrics: MetricData = JSON.parse(data); + const metrics: MetricData = JSON5.parse(data); // REQUEST_PARAMETER is used to request new parameters for multiphase trial job, // it is not metrics, so it is skipped here. if (metrics.type === 'REQUEST_PARAMETER') { @@ -140,7 +141,7 @@ class NNIDataStore implements DataStore { } assert(trialJobId === metrics.trial_job_id); try { - await this.db.storeMetricData(trialJobId, JSON.stringify({ + await this.db.storeMetricData(trialJobId, JSON5.stringify({ trialJobId: metrics.trial_job_id, parameterId: metrics.parameter_id, type: metrics.type, diff --git a/src/nni_manager/core/sqlDatabase.ts b/src/nni_manager/core/sqlDatabase.ts index 125a1aff6d..0ad6fd6bbe 100644 --- a/src/nni_manager/core/sqlDatabase.ts +++ b/src/nni_manager/core/sqlDatabase.ts @@ -5,6 +5,7 @@ import * as assert from 'assert'; import * as fs from 'fs'; +import * as JSON5 from 'json5'; import * as path from 'path'; import * as sqlite3 from 'sqlite3'; import { Deferred } from 'ts-deferred'; @@ -202,10 +203,10 @@ class SqlDB implements Database { public storeMetricData(trialJobId: string, data: string): Promise { const sql: string = 'insert into MetricData values (?,?,?,?,?,?)'; - const json: MetricDataRecord = JSON.parse(data); - const args: any[] = [Date.now(), json.trialJobId, json.parameterId, json.type, json.sequence, JSON.stringify(json.data)]; + const json: MetricDataRecord = JSON5.parse(data); + const args: any[] = [Date.now(), json.trialJobId, json.parameterId, json.type, json.sequence, JSON5.stringify(json.data)]; - this.log.trace(`storeMetricData: SQL: ${sql}, args: ${JSON.stringify(args)}`); + this.log.trace(`storeMetricData: SQL: ${sql}, args: ${JSON5.stringify(args)}`); const deferred: Deferred = new Deferred(); this.db.run(sql, args, (err: Error | null) => { this.resolve(deferred, err); }); diff --git a/src/nni_manager/package.json b/src/nni_manager/package.json index f79a0a4529..93e77cdf48 100644 --- a/src/nni_manager/package.json +++ b/src/nni_manager/package.json @@ -16,6 +16,7 @@ "express": "^4.16.3", "express-joi-validator": "^2.0.0", "js-base64": "^2.4.9", + "json5": "^2.1.1", "kubernetes-client": "^6.5.0", "rx": "^4.1.0", "sqlite3": "^4.0.2", @@ -34,6 +35,7 @@ "@types/express": "^4.16.0", "@types/glob": "^7.1.1", "@types/js-base64": "^2.3.1", + "@types/json5": "^0.0.30", "@types/mocha": "^5.2.5", "@types/node": "10.12.18", "@types/request": "^2.47.1", diff --git a/src/nni_manager/yarn.lock b/src/nni_manager/yarn.lock index deac0b5c89..379af7c4b9 100644 --- a/src/nni_manager/yarn.lock +++ b/src/nni_manager/yarn.lock @@ -157,6 +157,10 @@ version "7.0.3" resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.3.tgz#bdfd69d61e464dcc81b25159c270d75a73c1a636" +"@types/json5@^0.0.30": + version "0.0.30" + resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.30.tgz#44cb52f32a809734ca562e685c6473b5754a7818" + "@types/mime@*": version "2.0.0" resolved "https://registry.yarnpkg.com/@types/mime/-/mime-2.0.0.tgz#5a7306e367c539b9f6543499de8dd519fac37a8b" @@ -1840,9 +1844,9 @@ growl@1.10.5: version "1.10.5" resolved "https://registry.yarnpkg.com/growl/-/growl-1.10.5.tgz#f2735dc2283674fa67478b10181059355c369e5e" -handlebars@^4.0.11, handlebars@^4.3.0: - version "4.5.3" - resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.5.3.tgz#5cf75bd8714f7605713511a56be7c349becb0482" +handlebars@^4.0.11, handlebars@^4.5.3: + version "4.7.2" + resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.7.2.tgz#01127b3840156a0927058779482031afe0e730d7" dependencies: neo-async "^2.6.0" optimist "^0.6.1" @@ -2371,6 +2375,12 @@ json-stringify-safe@~5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb" +json5@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/json5/-/json5-2.1.1.tgz#81b6cb04e9ba496f1c7005d07b4368a2638f90b6" + dependencies: + minimist "^1.2.0" + jsonparse@^1.2.0: version "1.3.1" resolved "https://registry.yarnpkg.com/jsonparse/-/jsonparse-1.3.1.tgz#3f4dae4a91fac315f71062f8521cc239f1366280" diff --git a/src/sdk/pynni/nni/msg_dispatcher.py b/src/sdk/pynni/nni/msg_dispatcher.py index b3aadcaea1..482108c9dd 100644 --- a/src/sdk/pynni/nni/msg_dispatcher.py +++ b/src/sdk/pynni/nni/msg_dispatcher.py @@ -62,7 +62,7 @@ def _pack_parameter(parameter_id, params, customized=False, trial_job_id=None, p ret['parameter_index'] = parameter_index else: ret['parameter_index'] = 0 - return json_tricks.dumps(ret) + return json_tricks.dumps(ret, allow_nan=True) class MsgDispatcher(MsgDispatcherBase): diff --git a/src/sdk/pynni/nni/platform/local.py b/src/sdk/pynni/nni/platform/local.py index 556ff1398d..320cbeac45 100644 --- a/src/sdk/pynni/nni/platform/local.py +++ b/src/sdk/pynni/nni/platform/local.py @@ -35,7 +35,7 @@ def request_next_parameter(): 'type': 'REQUEST_PARAMETER', 'sequence': 0, 'parameter_index': _param_index - }) + }, allow_nan=True) send_metric(metric) def get_next_parameter(): diff --git a/src/sdk/pynni/nni/trial.py b/src/sdk/pynni/nni/trial.py index 6feed9dbef..60da9afe7e 100644 --- a/src/sdk/pynni/nni/trial.py +++ b/src/sdk/pynni/nni/trial.py @@ -116,11 +116,10 @@ def report_intermediate_result(metric): 'type': 'PERIODICAL', 'sequence': _intermediate_seq, 'value': metric - }) + }, allow_nan=True) _intermediate_seq += 1 platform.send_metric(metric) - def report_final_result(metric): """ Reports final result to NNI. @@ -138,5 +137,5 @@ def report_final_result(metric): 'type': 'FINAL', 'sequence': 0, 'value': metric - }) + }, allow_nan=True) platform.send_metric(metric) diff --git a/src/webui/package.json b/src/webui/package.json index 955b0ab4f2..ad202d89f3 100644 --- a/src/webui/package.json +++ b/src/webui/package.json @@ -18,6 +18,7 @@ "fork-ts-checker-webpack-plugin": "^1.5.0", "fs-extra": "^8.1.0", "html-webpack-plugin": "^4.0.0-beta.8", + "json5": "^2.1.1", "less": "^3.9.0", "less-loader": "^5.0.0", "mini-css-extract-plugin": "^0.8.0", @@ -55,6 +56,7 @@ "eslint": "npx eslint ./ --ext .tsx,.ts" }, "devDependencies": { + "@types/json5": "^0.0.30", "@types/node": "^10.14.14", "@types/react": "16.4.17", "@types/react-dom": "^16.0.7", diff --git a/src/webui/src/components/trial-detail/TableList.tsx b/src/webui/src/components/trial-detail/TableList.tsx index 757da666fa..3f76fb8c15 100644 --- a/src/webui/src/components/trial-detail/TableList.tsx +++ b/src/webui/src/components/trial-detail/TableList.tsx @@ -6,7 +6,7 @@ import { ColumnProps } from 'antd/lib/table'; const Option = Select.Option; const CheckboxGroup = Checkbox.Group; import { MANAGER_IP, trialJobStatus, COLUMN_INDEX, COLUMNPro } from '../../static/const'; -import { convertDuration, formatTimestamp, intermediateGraphOption, killJob } from '../../static/function'; +import { convertDuration, formatTimestamp, intermediateGraphOption, killJob, parseMetrics } from '../../static/function'; import { EXPERIMENT, TRIALS } from '../../static/datamodel'; import { TableRecord } from '../../static/interface'; import OpenRow from '../public-child/OpenRow'; @@ -178,11 +178,11 @@ class TableList extends React.Component { // get intermediate result dict keys array let otherkeys: Array = ['default']; if (res.data.length !== 0) { - otherkeys = Object.keys(JSON.parse(res.data[0].data)); + otherkeys = Object.keys(parseMetrics(res.data[0].data)); } // intermediateArr just store default val Object.keys(res.data).map(item => { - const temp = JSON.parse(res.data[item].data); + const temp = parseMetrics(res.data[item].data); if (typeof temp === 'object') { intermediateArr.push(temp.default); } else { @@ -210,7 +210,7 @@ class TableList extends React.Component { // just watch default key-val if (isShowDefault === true) { Object.keys(intermediateData).map(item => { - const temp = JSON.parse(intermediateData[item].data); + const temp = parseMetrics(intermediateData[item].data); if (typeof temp === 'object') { intermediateArr.push(temp[value]); } else { @@ -219,7 +219,7 @@ class TableList extends React.Component { }); } else { Object.keys(intermediateData).map(item => { - const temp = JSON.parse(intermediateData[item].data); + const temp = parseMetrics(intermediateData[item].data); if (typeof temp === 'object') { intermediateArr.push(temp[value]); } diff --git a/src/webui/src/static/function.ts b/src/webui/src/static/function.ts index fd1ccff682..94d59aace2 100644 --- a/src/webui/src/static/function.ts +++ b/src/webui/src/static/function.ts @@ -1,3 +1,4 @@ +import * as JSON5 from 'json5'; import axios from 'axios'; import { message } from 'antd'; import { MANAGER_IP } from './const'; @@ -174,7 +175,7 @@ function formatTimestamp(timestamp?: number, placeholder?: string = 'N/A'): stri } function metricAccuracy(metric: MetricDataRecord): number { - const data = JSON.parse(metric.data); + const data = parseMetrics(metric.data); return typeof data === 'number' ? data : NaN; } @@ -183,8 +184,16 @@ function formatAccuracy(accuracy: number): string { return accuracy.toFixed(6).replace(/0+$/, '').replace(/\.$/, ''); } +function parseMetrics(metricData: string): any { + if (metricData.includes('NaN')) { + return JSON5.parse(metricData) + } else { + return JSON.parse(metricData) + } +} + export { convertTime, convertDuration, getFinalResult, getFinal, downFile, intermediateGraphOption, killJob, filterByStatus, filterDuration, - formatAccuracy, formatTimestamp, metricAccuracy + formatAccuracy, formatTimestamp, metricAccuracy, parseMetrics }; diff --git a/src/webui/src/static/model/trial.ts b/src/webui/src/static/model/trial.ts index 2a91450ebc..2ab0e4afa8 100644 --- a/src/webui/src/static/model/trial.ts +++ b/src/webui/src/static/model/trial.ts @@ -1,5 +1,5 @@ import { MetricDataRecord, TrialJobInfo, TableObj, TableRecord, Parameters, FinalType } from '../interface'; -import { getFinal, formatAccuracy, metricAccuracy } from '../function'; +import { getFinal, formatAccuracy, metricAccuracy, parseMetrics } from '../function'; class Trial implements TableObj { private metricsInitialized: boolean = false; @@ -56,7 +56,7 @@ class Trial implements TableObj { // TODO: support intermeidate result is dict const temp = this.intermediates[this.intermediates.length - 1]; if (temp !== undefined) { - return JSON.parse(temp.data); + return parseMetrics(temp.data); } else { return undefined; } @@ -138,10 +138,10 @@ class Trial implements TableObj { const mediate: number[] = [ ]; for (const items of this.intermediateMetrics) { - if (typeof JSON.parse(items.data) === 'object') { - mediate.push(JSON.parse(items.data).default); + if (typeof parseMetrics(items.data) === 'object') { + mediate.push(parseMetrics(items.data).default); } else { - mediate.push(JSON.parse(items.data)); + mediate.push(parseMetrics(items.data)); } } ret.intermediate = mediate; diff --git a/src/webui/yarn.lock b/src/webui/yarn.lock index 0c8fbd6672..1d7548df58 100644 --- a/src/webui/yarn.lock +++ b/src/webui/yarn.lock @@ -80,6 +80,10 @@ version "0.0.29" resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.29.tgz#ee28707ae94e11d2b827bcbe5270bcea7f3e71ee" +"@types/json5@^0.0.30": + version "0.0.30" + resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.30.tgz#44cb52f32a809734ca562e685c6473b5754a7818" + "@types/minimatch@*": version "3.0.3" resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.3.tgz#3dca0e3f33b200fc7d1139c0cd96c1268cadfd9d" @@ -3988,6 +3992,12 @@ json5@^1.0.1: dependencies: minimist "^1.2.0" +json5@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/json5/-/json5-2.1.1.tgz#81b6cb04e9ba496f1c7005d07b4368a2638f90b6" + dependencies: + minimist "^1.2.0" + jsonfile@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-4.0.0.tgz#8771aae0799b64076b76640fca058f9c10e33ecb" diff --git a/test/config_test/examples/mnist-keras.test.yml b/test/config_test/examples/mnist-keras.test.yml index 3743be301e..cfb90fb0e7 100644 --- a/test/config_test/examples/mnist-keras.test.yml +++ b/test/config_test/examples/mnist-keras.test.yml @@ -19,5 +19,5 @@ trial: useAnnotation: false multiPhase: false multiThread: false - +debug: true trainingServicePlatform: local From 6f9678f2611399a7dbbfcf859ddfc44e90d1df35 Mon Sep 17 00:00:00 2001 From: Chengmin Chi Date: Tue, 14 Jan 2020 19:48:48 +0800 Subject: [PATCH 2/4] updates --- test/config_test/examples/mnist-keras.test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/config_test/examples/mnist-keras.test.yml b/test/config_test/examples/mnist-keras.test.yml index cfb90fb0e7..3743be301e 100644 --- a/test/config_test/examples/mnist-keras.test.yml +++ b/test/config_test/examples/mnist-keras.test.yml @@ -19,5 +19,5 @@ trial: useAnnotation: false multiPhase: false multiThread: false -debug: true + trainingServicePlatform: local From d84d65b6da444e99639cefb8df3f212077d385f1 Mon Sep 17 00:00:00 2001 From: Chengmin Chi Date: Wed, 15 Jan 2020 00:34:53 +0800 Subject: [PATCH 3/4] updates --- src/webui/src/static/function.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/webui/src/static/function.ts b/src/webui/src/static/function.ts index 94d59aace2..33193f2bf4 100644 --- a/src/webui/src/static/function.ts +++ b/src/webui/src/static/function.ts @@ -174,6 +174,14 @@ function formatTimestamp(timestamp?: number, placeholder?: string = 'N/A'): stri return timestamp ? new Date(timestamp).toLocaleString('en-US') : placeholder; } +function parseMetrics(metricData: string): any { + if (metricData.includes('NaN')) { + return JSON5.parse(metricData) + } else { + return JSON.parse(metricData) + } +} + function metricAccuracy(metric: MetricDataRecord): number { const data = parseMetrics(metric.data); return typeof data === 'number' ? data : NaN; @@ -184,14 +192,6 @@ function formatAccuracy(accuracy: number): string { return accuracy.toFixed(6).replace(/0+$/, '').replace(/\.$/, ''); } -function parseMetrics(metricData: string): any { - if (metricData.includes('NaN')) { - return JSON5.parse(metricData) - } else { - return JSON.parse(metricData) - } -} - export { convertTime, convertDuration, getFinalResult, getFinal, downFile, intermediateGraphOption, killJob, filterByStatus, filterDuration, From d64e2f40cc60c9a74c2ac03a10d359ddc54fa297 Mon Sep 17 00:00:00 2001 From: Chengmin Chi Date: Wed, 15 Jan 2020 13:51:59 +0800 Subject: [PATCH 4/4] updates --- src/sdk/pynni/nni/msg_dispatcher.py | 4 ++-- src/sdk/pynni/nni/platform/local.py | 6 +++--- src/sdk/pynni/nni/trial.py | 11 +++++------ src/sdk/pynni/nni/utils.py | 8 ++++---- 4 files changed, 14 insertions(+), 15 deletions(-) diff --git a/src/sdk/pynni/nni/msg_dispatcher.py b/src/sdk/pynni/nni/msg_dispatcher.py index 482108c9dd..739687fa0b 100644 --- a/src/sdk/pynni/nni/msg_dispatcher.py +++ b/src/sdk/pynni/nni/msg_dispatcher.py @@ -11,7 +11,7 @@ from .assessor import AssessResult from .common import multi_thread_enabled, multi_phase_enabled from .env_vars import dispatcher_env_vars -from .utils import MetricType +from .utils import MetricType, to_json _logger = logging.getLogger(__name__) @@ -62,7 +62,7 @@ def _pack_parameter(parameter_id, params, customized=False, trial_job_id=None, p ret['parameter_index'] = parameter_index else: ret['parameter_index'] = 0 - return json_tricks.dumps(ret, allow_nan=True) + return to_json(ret) class MsgDispatcher(MsgDispatcherBase): diff --git a/src/sdk/pynni/nni/platform/local.py b/src/sdk/pynni/nni/platform/local.py index 320cbeac45..1b4bc081c5 100644 --- a/src/sdk/pynni/nni/platform/local.py +++ b/src/sdk/pynni/nni/platform/local.py @@ -6,10 +6,10 @@ import json import time import subprocess -import json_tricks from ..common import init_logger from ..env_vars import trial_env_vars +from ..utils import to_json _sysdir = trial_env_vars.NNI_SYS_DIR if not os.path.exists(os.path.join(_sysdir, '.nni')): @@ -30,12 +30,12 @@ _param_index = 0 def request_next_parameter(): - metric = json_tricks.dumps({ + metric = to_json({ 'trial_job_id': trial_env_vars.NNI_TRIAL_JOB_ID, 'type': 'REQUEST_PARAMETER', 'sequence': 0, 'parameter_index': _param_index - }, allow_nan=True) + }) send_metric(metric) def get_next_parameter(): diff --git a/src/sdk/pynni/nni/trial.py b/src/sdk/pynni/nni/trial.py index 60da9afe7e..70127330fa 100644 --- a/src/sdk/pynni/nni/trial.py +++ b/src/sdk/pynni/nni/trial.py @@ -1,8 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -import json_tricks - +from .utils import to_json from .env_vars import trial_env_vars from . import platform @@ -110,13 +109,13 @@ def report_intermediate_result(metric): global _intermediate_seq assert _params or trial_env_vars.NNI_PLATFORM is None, \ 'nni.get_next_parameter() needs to be called before report_intermediate_result' - metric = json_tricks.dumps({ + metric = to_json({ 'parameter_id': _params['parameter_id'] if _params else None, 'trial_job_id': trial_env_vars.NNI_TRIAL_JOB_ID, 'type': 'PERIODICAL', 'sequence': _intermediate_seq, 'value': metric - }, allow_nan=True) + }) _intermediate_seq += 1 platform.send_metric(metric) @@ -131,11 +130,11 @@ def report_final_result(metric): """ assert _params or trial_env_vars.NNI_PLATFORM is None, \ 'nni.get_next_parameter() needs to be called before report_final_result' - metric = json_tricks.dumps({ + metric = to_json({ 'parameter_id': _params['parameter_id'] if _params else None, 'trial_job_id': trial_env_vars.NNI_TRIAL_JOB_ID, 'type': 'FINAL', 'sequence': 0, 'value': metric - }, allow_nan=True) + }) platform.send_metric(metric) diff --git a/src/sdk/pynni/nni/utils.py b/src/sdk/pynni/nni/utils.py index 99b5017107..bb7602a4b1 100644 --- a/src/sdk/pynni/nni/utils.py +++ b/src/sdk/pynni/nni/utils.py @@ -1,16 +1,16 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -""" -utils.py -""" - import os +import functools from enum import Enum, unique +import json_tricks from .common import init_logger from .env_vars import dispatcher_env_vars +to_json = functools.partial(json_tricks.dumps, allow_nan=True) + @unique class OptimizeMode(Enum): """Optimize Mode class