From 43d170187140632d2a017568e99880af1f2f3e41 Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Tue, 24 Jul 2018 08:40:39 -0400 Subject: [PATCH 1/5] Use the about:home: key as the github download source for all packages This commit replaces the https url for the private packages with github ssh and changes the logic in ska_builder.py to just use whatever is in that url (instead of guessing a github ssh string from the package name). --- ska_conda/pkg_defs/Chandra.Maneuver/meta.yaml | 3 +- ska_conda/pkg_defs/annie/meta.yaml | 2 +- ska_conda/pkg_defs/maude/meta.yaml | 2 +- ska_conda/pkg_defs/parse_cm/meta.yaml | 3 +- ska_conda/ska_builder.py | 35 ++++++------------- 5 files changed, 14 insertions(+), 31 deletions(-) diff --git a/ska_conda/pkg_defs/Chandra.Maneuver/meta.yaml b/ska_conda/pkg_defs/Chandra.Maneuver/meta.yaml index 11154c90..40eefce5 100644 --- a/ska_conda/pkg_defs/Chandra.Maneuver/meta.yaml +++ b/ska_conda/pkg_defs/Chandra.Maneuver/meta.yaml @@ -45,5 +45,4 @@ test: about: - home: https://github.com/sot/Chandra.Maneuver - + home: git@github.com:sot/Chandra.Maneuver diff --git a/ska_conda/pkg_defs/annie/meta.yaml b/ska_conda/pkg_defs/annie/meta.yaml index b06d6a7d..20117034 100644 --- a/ska_conda/pkg_defs/annie/meta.yaml +++ b/ska_conda/pkg_defs/annie/meta.yaml @@ -47,5 +47,5 @@ test: about: - home: https://github.com/sot/annie + home: git@github.com:sot/annie diff --git a/ska_conda/pkg_defs/maude/meta.yaml b/ska_conda/pkg_defs/maude/meta.yaml index ebf90143..af9648a8 100644 --- a/ska_conda/pkg_defs/maude/meta.yaml +++ b/ska_conda/pkg_defs/maude/meta.yaml @@ -46,4 +46,4 @@ test: about: - home: https://github.com/sot/maude + home: git@github.com:sot/maude diff --git a/ska_conda/pkg_defs/parse_cm/meta.yaml b/ska_conda/pkg_defs/parse_cm/meta.yaml index b846645c..67f4d8e3 100644 --- a/ska_conda/pkg_defs/parse_cm/meta.yaml +++ b/ska_conda/pkg_defs/parse_cm/meta.yaml @@ -35,7 +35,6 @@ test: imports: - parse_cm - about: - home: https://github.com/sot/parse_cm + home: git@github.com:sot/parse_cm diff --git a/ska_conda/ska_builder.py b/ska_conda/ska_builder.py index 412e41f1..785fb76a 100644 --- a/ska_conda/ska_builder.py +++ b/ska_conda/ska_builder.py @@ -1,7 +1,7 @@ import os import subprocess import git -import yaml +import re ska_conda_path = os.path.abspath(os.path.dirname(__file__)) pkg_defs_path = os.path.join(ska_conda_path, "pkg_defs") @@ -10,9 +10,7 @@ class SkaBuilder(object): - def __init__(self, build_root='.', user='sot', git_repo_path='git@github.com:{user}/{name}.git'): - self.user = user - self.git_repo_path = git_repo_path + def __init__(self, build_root='.'): self.build_dir = os.path.abspath(os.path.join(build_root, "builds")) self.src_dir = os.path.abspath(os.path.join(build_root, "src")) os.environ["SKA_TOP_SRC_DIR"] = self.src_dir @@ -20,33 +18,20 @@ def __init__(self, build_root='.', user='sot', git_repo_path='git@github.com:{us def _clone_repo(self, name, tag=None): if name in no_source_pkgs: return - print("Cloning source %s." % name) + print("Cloning or updating source source %s." % name) clone_path = os.path.join(self.src_dir, name) if not os.path.exists(clone_path): - # Try ssh first to avoid needing passwords for the private repos - # We could add these ssh strings to the meta.yaml for convenience - # The default ssh is not going to work if the package name doesn't match the - # top-level git repo name (cmd_states, eng_archive) - try: - repo = git.Repo.clone_from(self.git_repo_path.format(user=self.user, name=name), clone_path) - assert not repo.bare - print("Cloned via default ssh git") - except: - yml = os.path.join(pkg_defs_path, name, "meta.yaml") - with open(yml) as f: - requires = False - while not requires: - line = f.readline().strip() - if line.startswith("path:"): - requires = True - data = yaml.load(f) - url = data['about']['home'] - repo = git.Repo.clone_from(url, clone_path) - print("Cloned from url {}".format(url)) + metayml = os.path.join(pkg_defs_path, name, "meta.yaml") + # It isn't clean yaml at this point, so just extract the string we want after "home:" + meta = open(metayml).read() + url = re.search("home:\s?(\S+)", meta).group(1) + repo = git.Repo.clone_from(url, clone_path) + print("Cloned from url {}".format(url)) else: repo = git.Repo(clone_path) repo.remotes.origin.fetch() repo.remotes.origin.fetch("--tags") + print("Updated repo in {}".format(clone_path)) assert not repo.is_dirty() # I think we want the commit/tag with the most recent date, though # if we actually want the most recently created tag, that would probably be From 08f55517586de139441bd1da15c58af1ae116d92 Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Tue, 24 Jul 2018 08:43:02 -0400 Subject: [PATCH 2/5] Add try/except on each package when building all packages --- ska_conda/ska_builder.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/ska_conda/ska_builder.py b/ska_conda/ska_builder.py index 785fb76a..350c7430 100644 --- a/ska_conda/ska_builder.py +++ b/ska_conda/ska_builder.py @@ -70,11 +70,20 @@ def build_one_package(self, name, tag=None): self._build_package(name) def build_list_packages(self): + failures = [] with open(build_list, "r") as f: for line in f.readlines(): pkg_name = line.strip() if not pkg_name.startswith("#"): - self.build_one_package(pkg_name) + try: + self.build_one_package(pkg_name) + # Just try to build the rest if there's a failure, but record the name + except: + failures.append(pkg_name) + continue + if len(failures): + raise ValueError("Packages {} failed".format(",".join(failures))) + def build_all_packages(self): self.build_list_packages() From c599c97336e12851349eb44d13725f337ca436ed Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Wed, 25 Jul 2018 11:40:22 -0400 Subject: [PATCH 3/5] Add a 'continue y/n' on package build or clone fails --- ska_conda/ska_builder.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/ska_conda/ska_builder.py b/ska_conda/ska_builder.py index 350c7430..b4f9654c 100644 --- a/ska_conda/ska_builder.py +++ b/ska_conda/ska_builder.py @@ -77,10 +77,14 @@ def build_list_packages(self): if not pkg_name.startswith("#"): try: self.build_one_package(pkg_name) - # Just try to build the rest if there's a failure, but record the name + # If there's a failure, confirm before continuing except: - failures.append(pkg_name) - continue + print(f'{pkg_name} failed, continue anyway (y/n)?') + if input().lower().strip().startswith('y'): + failures.append(pkg_name) + continue + else: + raise ValueError(f"{pkg_name} failed") if len(failures): raise ValueError("Packages {} failed".format(",".join(failures))) From 9859fae6d4c3301f912983dedd2a37aa199535dd Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Wed, 25 Jul 2018 11:41:45 -0400 Subject: [PATCH 4/5] Allow multiple spaces in regex --- ska_conda/ska_builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ska_conda/ska_builder.py b/ska_conda/ska_builder.py index b4f9654c..8ad7e307 100644 --- a/ska_conda/ska_builder.py +++ b/ska_conda/ska_builder.py @@ -24,7 +24,7 @@ def _clone_repo(self, name, tag=None): metayml = os.path.join(pkg_defs_path, name, "meta.yaml") # It isn't clean yaml at this point, so just extract the string we want after "home:" meta = open(metayml).read() - url = re.search("home:\s?(\S+)", meta).group(1) + url = re.search("home:\s*(\S+)", meta).group(1) repo = git.Repo.clone_from(url, clone_path) print("Cloned from url {}".format(url)) else: From 43cd08af6f0a64df9f3a173d32e79daca9a841b5 Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Wed, 25 Jul 2018 11:44:01 -0400 Subject: [PATCH 5/5] Don't actually need yaml to build packages now --- setup.py | 2 +- ska_conda/pkg_defs/ska3_builder/meta.yaml | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/setup.py b/setup.py index db891e74..1a52a53d 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ download_url=url, scripts=scripts, include_package_data=True, - install_requires=["gitpython","pyyaml"], + install_requires=["gitpython"], classifiers=[ 'Intended Audience :: Science/Research', 'Operating System :: OS Independent', diff --git a/ska_conda/pkg_defs/ska3_builder/meta.yaml b/ska_conda/pkg_defs/ska3_builder/meta.yaml index 2b409f69..999a629e 100644 --- a/ska_conda/pkg_defs/ska3_builder/meta.yaml +++ b/ska_conda/pkg_defs/ska3_builder/meta.yaml @@ -17,13 +17,11 @@ requirements: build: - python - pip - - pyyaml - gitpython - git run: - python - ipython - - pyyaml - gitpython - git - conda-build