diff --git a/src/nni_manager/common/datastore.ts b/src/nni_manager/common/datastore.ts index 43b0532020..0b2847f7f1 100644 --- a/src/nni_manager/common/datastore.ts +++ b/src/nni_manager/common/datastore.ts @@ -38,6 +38,7 @@ interface TrialJobEventRecord { readonly event: TrialJobEvent; readonly data?: string; readonly logPath?: string; + readonly sequenceId?: number; } interface MetricData { @@ -59,6 +60,7 @@ interface MetricDataRecord { interface TrialJobInfo { id: string; + sequenceId?: number; status: TrialJobStatus; startTime?: number; endTime?: number; @@ -73,7 +75,8 @@ abstract class DataStore { public abstract close(): Promise; public abstract storeExperimentProfile(experimentProfile: ExperimentProfile): Promise; public abstract getExperimentProfile(experimentId: string): Promise; - public abstract storeTrialJobEvent(event: TrialJobEvent, trialJobId: string, data?: string, logPath?: string): Promise; + public abstract storeTrialJobEvent( + event: TrialJobEvent, trialJobId: string, hyperParameter?: string, jobDetail?: TrialJobDetail): Promise; public abstract getTrialJobStatistics(): Promise; public abstract listTrialJobs(status?: TrialJobStatus): Promise; public abstract getTrialJob(trialJobId: string): Promise; @@ -87,7 +90,8 @@ abstract class Database { public abstract storeExperimentProfile(experimentProfile: ExperimentProfile): Promise; public abstract queryExperimentProfile(experimentId: string, revision?: number): Promise; public abstract queryLatestExperimentProfile(experimentId: string): Promise; - public abstract storeTrialJobEvent(event: TrialJobEvent, trialJobId: string, data?: string, logPath?: string): Promise; + public abstract storeTrialJobEvent( + event: TrialJobEvent, trialJobId: string, hyperParameter?: string, jobDetail?: TrialJobDetail): Promise; public abstract queryTrialJobEvent(trialJobId?: string, event?: TrialJobEvent): Promise; public abstract storeMetricData(trialJobId: string, data: string): Promise; public abstract queryMetricData(trialJobId?: string, type?: MetricType): Promise; diff --git a/src/nni_manager/core/nniDataStore.ts b/src/nni_manager/core/nniDataStore.ts index 0ccfdd9bb6..09f846de3d 100644 --- a/src/nni_manager/core/nniDataStore.ts +++ b/src/nni_manager/core/nniDataStore.ts @@ -29,7 +29,7 @@ import { NNIError } from '../common/errors'; import { getExperimentId, isNewExperiment } from '../common/experimentStartupInfo'; import { getLogger, Logger } from '../common/log'; import { ExperimentProfile, TrialJobStatistics } from '../common/manager'; -import { TrialJobStatus } from '../common/trainingService'; +import { TrialJobDetail, TrialJobStatus } from '../common/trainingService'; import { getDefaultDatabaseDir, mkDirP } from '../common/utils'; class NNIDataStore implements DataStore { @@ -83,10 +83,11 @@ class NNIDataStore implements DataStore { return this.db.queryLatestExperimentProfile(experimentId); } - public storeTrialJobEvent(event: TrialJobEvent, trialJobId: string, data?: string, logPath?: string): Promise { - this.log.debug(`storeTrialJobEvent: event: ${event}, data: ${data}, logpath: ${logPath}`); + public storeTrialJobEvent( + event: TrialJobEvent, trialJobId: string, hyperParameter?: string, jobDetail?: TrialJobDetail): Promise { + this.log.debug(`storeTrialJobEvent: event: ${event}, data: ${hyperParameter}, jobDetail: ${JSON.stringify(jobDetail)}`); - return this.db.storeTrialJobEvent(event, trialJobId, data, logPath).catch( + return this.db.storeTrialJobEvent(event, trialJobId, hyperParameter, jobDetail).catch( (err: Error) => { throw new NNIError('Datastore error', `Datastore error: ${err.message}`, err); } @@ -281,6 +282,9 @@ class NNIDataStore implements DataStore { assert(false, 'jobInfo.hyperParameters is undefined'); } } + if (record.sequenceId !== undefined && jobInfo.sequenceId === undefined) { + jobInfo.sequenceId = record.sequenceId; + } map.set(record.trialJobId, jobInfo); } diff --git a/src/nni_manager/core/nnimanager.ts b/src/nni_manager/core/nnimanager.ts index 46c0e9088c..8083809397 100644 --- a/src/nni_manager/core/nnimanager.ts +++ b/src/nni_manager/core/nnimanager.ts @@ -355,7 +355,7 @@ class NNIManager implements Manager { const oldTrialJobDetail: TrialJobDetail | undefined = this.trialJobs.get(trialJobId); if (oldTrialJobDetail !== undefined && oldTrialJobDetail.status !== trialJobDetail.status) { this.trialJobs.set(trialJobId, Object.assign({}, trialJobDetail)); - await this.dataStore.storeTrialJobEvent(trialJobDetail.status, trialJobDetail.id, undefined, trialJobDetail.url); + await this.dataStore.storeTrialJobEvent(trialJobDetail.status, trialJobDetail.id, undefined, trialJobDetail); } switch (trialJobDetail.status) { case 'SUCCEEDED': @@ -461,7 +461,7 @@ class NNIManager implements Manager { const trialJobDetailSnapshot: TrialJobDetail | undefined = this.trialJobs.get(trialJobDetail.id); if (trialJobDetailSnapshot != undefined) { await this.dataStore.storeTrialJobEvent( - trialJobDetailSnapshot.status, trialJobDetailSnapshot.id, hyperParams, trialJobDetailSnapshot.url); + trialJobDetailSnapshot.status, trialJobDetailSnapshot.id, hyperParams, trialJobDetailSnapshot); } else { assert(false, `undefined trialJobDetail in trialJobs: ${trialJobDetail.id}`); } diff --git a/src/nni_manager/core/sqlDatabase.ts b/src/nni_manager/core/sqlDatabase.ts index 4f4755d331..3e55be25e7 100644 --- a/src/nni_manager/core/sqlDatabase.ts +++ b/src/nni_manager/core/sqlDatabase.ts @@ -33,11 +33,12 @@ import { TrialJobEventRecord } from '../common/datastore'; import { ExperimentProfile } from '../common/manager'; +import { TrialJobDetail } from '../common/trainingService'; /* tslint:disable:no-any */ const createTables: string = ` -create table TrialJobEvent (timestamp integer, trialJobId text, event text, data text, logPath text); +create table TrialJobEvent (timestamp integer, trialJobId text, event text, data text, logPath text, sequenceId integer); create index TrialJobEvent_trialJobId on TrialJobEvent(trialJobId); create index TrialJobEvent_event on TrialJobEvent(event); @@ -51,6 +52,7 @@ create table ExperimentProfile ( execDuration integer, startTime integer, endTime integer, + logDir text, revision integer); create index ExperimentProfile_id on ExperimentProfile(id); `; @@ -62,6 +64,7 @@ function loadExperimentProfile(row: any): ExperimentProfile { execDuration: row.execDuration, startTime: row.startTime === null ? undefined : row.startTime, endTime: row.endTime === null ? undefined : row.endTime, + logDir: row.logDir === null ? undefined : row.logDir, revision: row.revision }; } @@ -72,7 +75,8 @@ function loadTrialJobEvent(row: any): TrialJobEventRecord { trialJobId: row.trialJobId, event: row.event, data: row.data === null ? undefined : row.data, - logPath: row.logPath === null ? undefined : row.logPath + logPath: row.logPath === null ? undefined : row.logPath, + sequenceId: row.sequenceId === null ? undefined : row.sequenceId }; } @@ -127,13 +131,14 @@ class SqlDB implements Database { } public storeExperimentProfile(exp: ExperimentProfile): Promise { - const sql: string = 'insert into ExperimentProfile values (?,?,?,?,?,?)'; + const sql: string = 'insert into ExperimentProfile values (?,?,?,?,?,?,?)'; const args: any[] = [ JSON.stringify(exp.params), exp.id, exp.execDuration, exp.startTime === undefined ? null : exp.startTime, exp.endTime === undefined ? null : exp.endTime, + exp.logDir === undefined ? null : exp.logDir, exp.revision ]; @@ -168,9 +173,12 @@ class SqlDB implements Database { return profiles[0]; } - public storeTrialJobEvent(event: TrialJobEvent, trialJobId: string, data?: string, logPath?: string): Promise { - const sql: string = 'insert into TrialJobEvent values (?,?,?,?,?)'; - const args: any[] = [Date.now(), trialJobId, event, data, logPath]; + public storeTrialJobEvent( + event: TrialJobEvent, trialJobId: string, hyperParameter?: string, jobDetail?: TrialJobDetail): Promise { + const sql: string = 'insert into TrialJobEvent values (?,?,?,?,?,?)'; + const logPath: string | undefined = jobDetail === undefined ? undefined : jobDetail.url; + const sequenceId: number | undefined = jobDetail === undefined ? undefined : jobDetail.sequenceId; + const args: any[] = [Date.now(), trialJobId, event, hyperParameter, logPath, sequenceId]; const deferred: Deferred = new Deferred(); this.db.run(sql, args, (err: Error | null) => { this.resolve(deferred, err); });