Pull request for #68: Remove Google Drive from default release (#72)

* issue #68

- Fix broken imports in test
- Move `read_yaml_value` to `misc` from `config`
- Fix appropriate tests broken because of above move

* #68 accommodate atexit implementation of state_of_repo

* #68 fix numargs

* #68 - instead of _ for user-config.yaml

* #68 fix regex error in _release_tools

* #68 respond to PR

* #68 move check_and_expand_cache_path to misc.py, rename check_and_expand_path

* #68 response to PR about release paths

* #68 remove prompt for github token

* #68 respond to PR

* #68 add logical gate to use getpass.getpass for github_token

gslab_scons/README.md

@@ -19,3 +19,10 @@ from being zipped before their release to Google Drive.
 This release procedure will warn the user when a versioned file
 is larger than 2MB and when the directory's versioned content
 is larger than 500MB in total.  
+
+Instead of entering the GitHub token as a password when using `release`,
+the user can store it `user-config.yaml` in the relevant template as
+
+```
+github_token: <YOUR_TOKEN_HERE>
+```

gslab_scons/_release_tools.py

@@ -10,33 +10,34 @@
 
 from _exception_classes import ReleaseError
 
-
-def release(vers, org, repo, 
+def release(vers, org, repo,
             DriveReleaseFiles = [],  
             local_release     = '',  
             target_commitish  = '', 
-            zip_release       = True):
+            zip_release       = True,
+            github_token      = None):
     '''Publish a release
 
     Parameters
     ----------
     env: an SCons environment object
     vers: the version of the release
     DriveReleaseFiles a optional list of files to be included in a
-        release to Google Drive.
+        release to drive (e.g. DropBox or Google Drive).
     local_release: The path of the release directory on the user's computer.
     org: The GtHub organisaton hosting the repository specified by `repo`.
     repo: The name of the GitHub repository from which the user is making
         the release.
     '''
     # Check the argument types
 
-
-    token    = getpass.getpass("Enter a GitHub token and then press enter: ") 
+    if bool(github_token) is False:
+        github_token = getpass.getpass("Enter a GitHub token and then press enter: ") 
+
     tag_name = vers
 
     releases_path = 'https://%s:@api.github.com/repos/%s/%s/releases' \
-                    % (token, org, repo)
+                    % (github_token, org, repo)
     session       = requests.session()
 
     # Create release
@@ -49,11 +50,20 @@ def release(vers, org, repo,
 
     json_dump = json.dumps(payload)
     json_dump = re.sub('"FALSE"', 'false', json_dump)
-    posting = session.post(releases_path, data = json_dump)
+    posting   = session.post(releases_path, data = json_dump)
     # Check that the GitHub release was successful
-    posting.raise_for_status()
-
-    # Release to Google Drive
+    try:
+        posting.raise_for_status()
+    except requests.exceptions.HTTPError:
+        message  = "We could not post the following json to the releases path \n" 
+        message  = message + ("https://YOURTOKEN:@api.github.com/repos/%s/%s/releases \n" % (org, repo))
+        message  = message + "The json looks like this:"
+        print(message)
+        for (k,v) in payload.items():
+            print(" '%s' : '%s' " % (k,v))
+        raise requests.exceptions.HTTPError
+
+    # Release to drive
     if bool(DriveReleaseFiles):
         # Delay
         time.sleep(1)
@@ -73,24 +83,15 @@ def release(vers, org, repo,
         tag_name_index = json_split.index('"tag_name":"%s"' % tag_name)
         release_id     = json_split[tag_name_index - 1].split(':')[1]
 
-        # Get root directory name on Drive
-        path     = local_release.split('/')
-        layers   = len(path)
-        dir_name = None
-
-        for i in range(layers):
-            if path[i] == 'release' and i + 1 < layers:
-                dir_name = path[i + 1]
-                break
-
-        if dir_name is None:
-            raise ReleaseError("No /release/ superdirectory found "
-                               "in path given by local_release")
+        # Get root directory name on drive
+        path       = local_release.split('/')
+        drive_name = path[-2]
+        dir_name   = path[-1]
 
         if not os.path.isdir(local_release):
             os.makedirs(local_release)
 
-        # If the files released to Google Drive are to be zipped,
+        # If the files released to drive are to be zipped,
         # specify their copy destination as an intermediate directory
         if zip_release:
             archive_files = 'release_content'
@@ -99,13 +100,13 @@ def release(vers, org, repo,
             os.makedirs(archive_files)
 
             destination_base = archive_files
-            drive_header = 'Google Drive: release/%s/%s/release_content.zip' % \
-                            (dir_name, vers)
+            drive_header = '%s: release/%s/%s/release_content.zip' % \
+                            (drive_name, dir_name, vers)
         # Otherwise, send the release files directly to the local release
-        # Google Drive directory
+        # drive directory
         else:
             destination_base = local_release
-            drive_header = 'Google Drive:'
+            drive_header = '%s:' % drive_name
 
         for path in DriveReleaseFiles:
             file_name   = os.path.basename(path)
@@ -125,25 +126,25 @@ def release(vers, org, repo,
             make_paths = lambda s: 'release/%s/%s/%s' % (dir_name, vers, s)
             DriveReleaseFiles = map(make_paths, DriveReleaseFiles)
 
-        with open('gdrive_assets.txt', 'wb') as f:
+        with open('drive_assets.txt', 'wb') as f:
             f.write('\n'.join([drive_header] + DriveReleaseFiles))
 
-        upload_asset(token      = token, 
-                     org        = org, 
-                     repo       = repo, 
-                     release_id = release_id, 
-                     file_name  = 'gdrive_assets.txt')
+        upload_asset(github_token = github_token, 
+                     org          = org, 
+                     repo         = repo, 
+                     release_id   = release_id, 
+                     file_name    = 'drive_assets.txt')
 
-        os.remove('gdrive_assets.txt')
+        os.remove('drive_assets.txt')
 
 
-def upload_asset(token, org, repo, release_id, file_name, 
+def upload_asset(github_token, org, repo, release_id, file_name, 
                  content_type = 'text/markdown'):
     '''
     This function uploads a release asset to GitHub.
 
     --Parameters--
-    token: a GitHub token
+    github_token: a GitHub token
     org: the GitHub organisation to which the repository associated
         with the release belongs
     repo: the GitHub repository associated with the release
@@ -158,7 +159,7 @@ def upload_asset(token, org, repo, release_id, file_name,
         raise ReleaseError('upload_asset() cannot find file_name')
 
     files  = {'file' : open(file_name, 'rU')}
-    header = {'Authorization': 'token %s' % token, 
+    header = {'Authorization': 'token %s' % github_token, 
               'Content-Type':  content_type}
     path_base   = 'https://uploads.github.com/repos'
     upload_path = '%s/%s/%s/releases/%s/assets?name=%s' % \
@@ -285,6 +286,6 @@ def extract_dot_git(path = '.git'):
 
     # Next, find the branch's name
     branch_info = open('%s/HEAD' % path, 'rU').readlines()
-    branch = re.findall('ref: refs/heads/([\w-]+)', branch_info[0])[0]
+    branch = re.findall('ref: refs/heads/(.+)\\n', branch_info[0])[0]
 
     return repo, organisation, branch

gslab_scons/configuration_tests.py

@@ -4,7 +4,7 @@
 import importlib
 import subprocess
 import pkg_resources
-from misc import is_in_path, get_stata_executable, get_stata_command, is_unix
+from misc import is_in_path, get_stata_executable, get_stata_command, is_unix, load_yaml_value, check_and_expand_path
 from _exception_classes import PrerequisiteError
 
 
@@ -114,19 +114,6 @@ def check_lfs():
                               'git lfs install --force' if prompted above.''')
 
 
-def check_and_expand_cache_path(cache):
-    error_message = " Cache directory, '%s', is not created. " % cache + \
-                    "Please manually create before running.\n\t\t" + \
-                    "    Or fix the path in user-config.yaml.\n"
-    try:
-        cache = os.path.expanduser(cache)
-        if not os.path.isdir(cache):
-            raise PrerequisiteError(error_message)
-        return cache
-    except:
-        raise PrerequisiteError(error_message)
-
-
 def check_stata(packages = ["yaml"], user_yaml = "user-config.yaml"):
     '''
     Check that a valid Stata executable is in the path and that the specified
@@ -146,50 +133,6 @@ def check_stata(packages = ["yaml"], user_yaml = "user-config.yaml"):
     return stata_executable
 
 
-def load_yaml_value(path, key):
-    '''
-    Load the yaml value indexed by the key argument in the file
-    specified by the path argument.
-    '''
-    import yaml
-
-    if key == "stata_executable":
-        prompt = "Enter %s value or None to search for defaults: "
-    else:
-        prompt = "Enter %s value: "
-
-    # Check if file exists and is not corrupted. If so, load yaml contents.
-    yaml_contents = None
-    if os.path.isfile(path):
-        try:
-            yaml_contents = yaml.load(open(path, 'rU'))
-            if not isinstance(yaml_contents, dict):
-                raise yaml.scanner.ScannerError()
-
-        except yaml.scanner.ScannerError:
-            message  = "%s is a corrupted yaml file. Delete file and recreate? (y/n) "
-            response = str(raw_input(message % path))
-            if response.lower() == 'y':
-                os.remove(path)
-                yaml_contents = None
-            else:
-                message = "%s is a corrupted yaml file. Please fix." % path
-                raise PrerequisiteError(message)
-
-    # If key exists, return value. Otherwise, add key-value to file.
-    try:
-        if yaml_contents[key] == "None":
-            return None
-        else:
-            return yaml_contents[key]
-    except:
-        with open(path, 'ab') as f:        
-            val = str(raw_input(prompt % key))
-            if re.sub('"', '', re.sub('\'', '', val.lower())) == "none":
-                val = None
-            f.write('%s: %s\n' % (key, val))
-        return val
-
 
 def check_stata_packages(command, packages):
     '''Check that the specified Stata packages are installed'''
@@ -215,7 +158,8 @@ def check_stata_packages(command, packages):
             if re.search('command %s not found' % pkg, log):
                 raise PrerequisiteError('Stata package %s is not installed' % pkg)
 
-    except subprocess.CalledProcessError:
+    except subprocess.CalledProcessError:        
         raise PrerequisiteError("Stata command, '%s', failed.\n" % command.split(' ')[0] + \
                                 "\t\t   Please supply a correct stata_executable" + \
-                                " value in user_config.yaml.\n" )
+                                " value in user-config.yaml.\n" )
+

gslab_scons/misc.py

@@ -5,12 +5,13 @@
 import shutil
 import subprocess
 import datetime
+import yaml
+import getpass
 # Import gslab_scons modules
 import _exception_classes
 from size_warning import issue_size_warnings
 
-
-def scons_debrief(target, source, env):
+def scons_debrief(target, env):
     '''Execute functions after SCons has built all targets'''
     # Log the state of the repo
     env['CL_ARG'] = env['MAXIT']
@@ -23,7 +24,7 @@ def scons_debrief(target, source, env):
     file_MB_limit = float(env['file_MB_limit'])
     total_MB_limit = float(env['total_MB_limit'])
     issue_size_warnings(look_in, file_MB_limit, total_MB_limit)
-
+    return None
 
 def state_of_repo(maxit):
     outfile = 'state_of_repo.log'
@@ -237,6 +238,68 @@ def lyx_scan(node, env, path):
     return SOURCE
 
 
+def load_yaml_value(path, key):
+    '''
+    Load the yaml value indexed by the key argument in the file
+    specified by the path argument.
+    '''
+    if key == "stata_executable":
+        prompt = "Enter %s or None to search for defaults: "
+    elif key == "github_token":
+        prompt = "(Optional) Enter %s to be stored in user_config.yaml.\n" 
+        prompt = prompt + "Github token can also be entered without storing to file later:" 
+    else:
+        prompt = "Enter %s: "
+
+    # Check if file exists and is not corrupted. If so, load yaml contents.
+    yaml_contents = None
+    if os.path.isfile(path):
+        try:
+            yaml_contents = yaml.load(open(path, 'rU'))
+            if not isinstance(yaml_contents, dict):
+                raise yaml.scanner.ScannerError()
+
+        except yaml.scanner.ScannerError:
+            message  = "%s is a corrupted yaml file. Delete file and recreate? (y/n) "
+            response = str(raw_input(message % path))
+            if response.lower() == 'y':
+                os.remove(path)
+                yaml_contents = None
+            else:
+                message = "%s is a corrupted yaml file. Please fix." % path
+                raise _exception_classes.PrerequisiteError(message)
+
+    # If key exists, return value. Otherwise, add key-value to file.
+    try:
+        if yaml_contents[key] == "None":
+            return None
+        else:
+            return yaml_contents[key]
+    except:
+        with open(path, 'ab') as f:  
+            if key == "github_token":
+                val = getpass.getpass(prompt = (prompt % key))
+            else:
+                val = str(raw_input(prompt % key))
+            if re.sub('"', '', re.sub('\'', '', val.lower())) == "none":
+                val = None
+            f.write('%s: %s\n' % (key, val))
+        return val
+
+
+def check_and_expand_path(path):
+    error_message = " The directory provided, '%s', cannot be found. " % path + \
+                    "Please manually create before running\n" + \
+                    "or fix the path in user-config.yaml.\n"
+    try:
+        path = os.path.expanduser(path)
+        if not os.path.isdir(path):
+            raise _exception_classes.PrerequisiteError(error_message)
+        return path
+    except:
+        raise _exception_classes.PrerequisiteError(error_message)
+
+
 def get_directory(path):
     '''
     Determine the directory of a file. This function returns