Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

profile-battery: address dependencies #2182

Merged
merged 3 commits into from
Mar 2, 2017
Merged

profile-battery: address dependencies #2182

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5d321d7
--help should show even if cylc is not a git repo.
oliver-sanders Mar 1, 2017
d2bddbc
Remove OrderedDict dependency.
oliver-sanders Mar 1, 2017
b0c9977
pep8 fix.
oliver-sanders Mar 1, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 41 additions & 36 deletions bin/cylc-profile-battery
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -419,8 +419,8 @@ def print_manual_scheme(versions, experiments, all_versions=None):
for version in versions:
print '\t$ git checkout ' + version['id']
print '\t$ cylc profile-battery --experiments ' + exp
print ('\t$ cylc profile-battery --versions {versions} --experiments '
'{experiments}'.format(versions=ver, experiments=exp))
print('\t$ cylc profile-battery --versions {versions} --experiments '
'{experiments}'.format(versions=ver, experiments=exp))


def determine_action(schedule, versions, experiments, non_interactive=False):
Expand Down Expand Up @@ -480,25 +480,25 @@ def determine_action(schedule, versions, experiments, non_interactive=False):
else:
automatic_only_versions.append(version)

print ('To perform profiling different cylc versions will need to be '
'checked out. I can checkout and profile versions '
'automatically.')
print ('If using the automatic checkout system ensure that there are '
'no un-commited changes before proceeding and do not make '
'any changes to the local repository whist the profiling is '
'running\n')
print('To perform profiling different cylc versions will need to be '
'checked out. I can checkout and profile versions '
'automatically.')
print('If using the automatic checkout system ensure that there are '
'no un-commited changes before proceeding and do not make '
'any changes to the local repository whist the profiling is '
'running\n')

if automatic_only_versions:
print ('These versions can only be profiled '
'automatically:\n\t{0}'.format(
' '.join([version['name'] for version in
automatic_only_versions])
))
print('These versions can only be profiled '
'automatically:\n\t{0}'.format(
' '.join([version['name'] for version in
automatic_only_versions])
))
if manual_versions:
print ('These versions you can profile manually if you '
'prefer:\n\t{0}'.format(
' '.join([version['name'] for version in
manual_versions])))
print('These versions you can profile manually if you '
'prefer:\n\t{0}'.format(
' '.join([version['name'] for version in
manual_versions])))

print

Expand All @@ -508,8 +508,8 @@ def determine_action(schedule, versions, experiments, non_interactive=False):
response = raw_input('Do you want to checkout these versions '
'automatically? (y/n): ')
if response == 'n':
print ('You can perform this profiling manually by doing '
'something like:')
print('You can perform this profiling manually by doing '
'something like:')
print_manual_scheme(manual_versions, experiments,
all_versions=versions)
sys.exit('Profiling aborted by user.')
Expand All @@ -529,8 +529,8 @@ def determine_action(schedule, versions, experiments, non_interactive=False):
all_versions=versions)
to_checkout = automatic_only_versions
if response == 'none':
print ('Some versions can be profiled manually by doing '
'something like:')
print('Some versions can be profiled manually by doing '
'something like:')
print_manual_scheme(manual_versions, experiments,
all_versions=manual_versions)
sys.exit('Profiling aborted by user.')
Expand Down Expand Up @@ -701,13 +701,13 @@ def run_schedule(schedule, experiments, versions, exps_to_run,
raise GitCheckoutError()
checkout(r'@{-%d}' % checkout_count, delete_pyc=True)
except GitCheckoutError:
print ('ERROR: Could not checkout git repo to original location. '
r'\n\t$ git checkout @{-%d}' % checkout_count)
print('ERROR: Could not checkout git repo to original location. '
r'\n\t$ git checkout @{-%d}' % checkout_count)

# Stop here if profiling was un-successfull.
if not success:
print ('ERROR: Some experiments failed to run, no plotting will be '
'attempted.')
print('ERROR: Some experiments failed to run, no plotting will be '
'attempted.')

return success

Expand Down Expand Up @@ -760,8 +760,8 @@ def run_analysis(experiments, versions, interactive=False,
plot_results(full_results, versions, experiment, plt_dir,
quick_analysis=quick_analysis, lobf_order=lobf_order)
if plt_dir:
print ('Results for experiment "{exp}" have been written out to '
'"{dir}"'.format(exp=experiment['name'], dir=plt_dir))
print('Results for experiment "{exp}" have been written out to '
'"{dir}"'.format(exp=experiment['name'], dir=plt_dir))


def ls(exp_names, ver_names, delete=False):
Expand Down Expand Up @@ -869,17 +869,17 @@ def promote(experiment_id, yes=False):
candidate_versions = [version for version in candidate_versions if
version not in target_versions]
print
print ('Only the results for cylc versions not already profiled in '
'the current experiment version will be promoted.')
print('Only the results for cylc versions not already profiled in '
'the current experiment version will be promoted.')
git.order_identifiers_by_date(candidate_versions)

print
print ('Promote the following results for experiment "{name}" at version '
'"{candidate}" to the current version "{target}":'.format(
name=experiment_name,
candidate=experiment_version,
target=cur_exp_id.rsplit('@', 1)[1]
))
print('Promote the following results for experiment "{name}" at version '
'"{candidate}" to the current version "{target}":'.format(
name=experiment_name,
candidate=experiment_version,
target=cur_exp_id.rsplit('@', 1)[1]
))
print '\t', ' '.join(candidate_versions)

if not yes:
Expand All @@ -906,6 +906,11 @@ def main():
"""cylc profile-battery"""
opts = parse_args()

if not prof.IS_GIT_REPO:
print >> sys.stderr, ('ERROR: profiling requires cylc to be a git '
'repository.')
sys.exit(2)

# Promote mode.
if opts.promote:
promote(opts.promote, opts.yes)
Expand Down
4 changes: 1 addition & 3 deletions lib/cylc/profiling/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,7 @@ def get_cylc_directory():
# Ensure that the cylc directory is a git repository.
CYLC_DIR = get_cylc_directory()
os.chdir(CYLC_DIR)
if not is_git_repo():
print 'ERROR: profiling requires cylc to be a git repository.'
sys.exit(2)
IS_GIT_REPO = is_git_repo()

# Files and directories
PROFILE_DIR_NAME = '.profiling' # Path to profiling directory.
Expand Down
19 changes: 9 additions & 10 deletions lib/cylc/profiling/analysis.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""Module for performing analysis on profiling results and generating plots."""

from collections import OrderedDict
import os
import re
import sys
Expand Down Expand Up @@ -171,7 +170,7 @@ def process_out_file(file_name, suite_start_time, validate=False):
lines[0][len(SUITE_STARTUP_STRING):])

# Scan through log entries.
ret['memory'] = OrderedDict()
ret['memory'] = []
loop_mem_entries = []
for line in lines:
# Profile summary.
Expand All @@ -186,7 +185,7 @@ def process_out_file(file_name, suite_start_time, validate=False):
match = MEMORY_LINE_REGEX.search(line)
if match:
memory, module, checkpoint = tuple(match.groups())
ret['memory'][(module, checkpoint,)] = int(memory)
ret['memory'].append((module, checkpoint, int(memory),))

# Main loop memory info.
if not validate:
Expand All @@ -213,7 +212,7 @@ def process_out_file(file_name, suite_start_time, validate=False):
loop_mem_entries[-1][0])

# Maximum memory usage.
ret['mxmem'] = max([ret['memory'][key] for key in ret['memory']])
ret['mxmem'] = max([entry[2] for entry in ret['memory']])

# Startup time (time from running cmd to reaching the end of the first
# loop).
Expand Down Expand Up @@ -302,8 +301,8 @@ def make_table(results, versions, experiment, quick_analysis=False):
[data[run_name][metric] for metric in
sorted(metrics)])
except ValueError:
print ('ERROR: Data is not complete. Try removing results and '
're-running any experiments')
print('ERROR: Data is not complete. Try removing results and '
're-running any experiments')

return table

Expand Down Expand Up @@ -391,8 +390,8 @@ def plot_scale(results, run_names, versions, metric, experiment,
# Compute and plot line of best fit.
if lobf_order >= 1:
if lobf_order > 8:
print ('WARNING: Line of best fit order too high (' +
lobf_order + '). Order has been set to 3.')
print('WARNING: Line of best fit order too high (' +
lobf_order + '). Order has been set to 3.')
lobf_order = 3
lobf = numpy.polyfit(x_data, y_data, lobf_order)
line = numpy.linspace(x_data[0], x_data[-1], 100)
Expand Down Expand Up @@ -427,8 +426,8 @@ def plot_results(results, versions, experiment, plt_dir=None,
"""
# Are we able to plot?
if not CAN_PLOT:
print ('\nWarning: Plotting requires numpy and maplotlib so cannot be '
'run.')
print('\nWarning: Plotting requires numpy and maplotlib so cannot be '
'run.')
return

versions = remove_profile_from_versions(versions)
Expand Down