Skip to content
This repository has been archived by the owner on Sep 18, 2024. It is now read-only.

Commit

Permalink
Quick fix resume logic (#285)
Browse files Browse the repository at this point in the history
  • Loading branch information
SparkSnail authored Oct 31, 2018
1 parent b702854 commit 44ab774
Showing 1 changed file with 9 additions and 25 deletions.
34 changes: 9 additions & 25 deletions tools/nnicmd/launcher.py
Original file line number Diff line number Diff line change
Expand Up @@ -110,8 +110,9 @@ def set_trial_config(experiment_config, port, config_file_name):
else:
print('Error message is {}'.format(response.text))
_, stderr_full_path = get_log_path(config_file_name)
with open(stderr_full_path, 'a+') as fout:
fout.write(json.dumps(json.loads(response.text), indent=4, sort_keys=True, separators=(',', ':')))
if response:
with open(stderr_full_path, 'a+') as fout:
fout.write(json.dumps(json.loads(response.text), indent=4, sort_keys=True, separators=(',', ':')))
return False

def set_local_config(experiment_config, port, config_file_name):
Expand Down Expand Up @@ -212,8 +213,9 @@ def set_experiment(experiment_config, mode, port, config_file_name):
return response
else:
_, stderr_full_path = get_log_path(config_file_name)
with open(stderr_full_path, 'a+') as fout:
fout.write(json.dumps(json.loads(response.text), indent=4, sort_keys=True, separators=(',', ':')))
if response:
with open(stderr_full_path, 'a+') as fout:
fout.write(json.dumps(json.loads(response.text), indent=4, sort_keys=True, separators=(',', ':')))
print_error('Setting experiment error, error message is {}'.format(response.text))
return None

Expand Down Expand Up @@ -324,15 +326,6 @@ def launch_experiment(args, experiment_config, mode, config_file_name, experimen

print_normal(EXPERIMENT_SUCCESS_INFO % (experiment_id, ' '.join(web_ui_url_list)))

def cmp_time(time1, time2):
'''compare the time'''
try:
time1 = time.strptime(time1,'%Y-%m-%d %H:%M:%S')
time2 = time.strptime(time2,'%Y-%m-%d %H:%M:%S')
return int(time1) - int(time2)
except:
return 0

def resume_experiment(args):
'''resume an experiment'''
experiment_config = Experiments()
Expand All @@ -341,18 +334,9 @@ def resume_experiment(args):
experiment_endTime = None
#find the latest stopped experiment
if not args.id:
for key in experiment_dict.keys():
if experiment_dict[key]['status'] == 'stopped':
if experiment_id is None:
experiment_id = key
experiment_endTime = experiment_dict[key]['endTime']
else:
if cmp_time(experiment_dict[key]['endTime'], experiment_endTime) > 0:
experiment_id = key
experiment_endTime = experiment_dict[key]['endTime']
if experiment_id is None:
print_error('There is no experiment stopped!')
exit(1)
print_error('Please set experiment id! \nYou could use \'nnictl resume {id}\' to resume a stopped experiment!\n' \
'You could use \'nnictl experiment list all\' to show all of stopped experiments!')
exit(1)
else:
if experiment_dict.get(args.id) is None:
print_error('Id %s not exist!' % args.id)
Expand Down

0 comments on commit 44ab774

Please sign in to comment.