feat: Initialize a project from non-cookiecutter github repo or zip f…

…ile (aws#1595)

samcli/commands/deploy/command.py

@@ -7,7 +7,7 @@
 import click
 from click.types import FuncParamType
 
-from samcli.lib.utils import temp_file_utils
+from samcli.lib.utils import osutils
 from samcli.cli.cli_config_file import configuration_option, TomlProvider
 from samcli.cli.context import get_cmd_names
 from samcli.cli.main import pass_context, common_options, aws_creds_options
@@ -256,7 +256,7 @@ def do_cli(
         confirm_changeset=changeset_decision if guided else confirm_changeset,
     )
 
-    with temp_file_utils.tempfile_platform_independent() as output_template_file:
+    with osutils.tempfile_platform_independent() as output_template_file:
 
         with PackageContext(
             template_file=template_file,

samcli/commands/init/init_generator.py

@@ -5,11 +5,11 @@
 
 from samcli.commands.exceptions import UserException
 from samcli.local.init import generate_project
-from samcli.local.init.exceptions import GenerateProjectFailedError
+from samcli.local.init.exceptions import GenerateProjectFailedError, ArbitraryProjectDownloadFailed
 
 
 def do_generate(location, runtime, dependency_manager, output_dir, name, no_input, extra_context):
     try:
         generate_project(location, runtime, dependency_manager, output_dir, name, no_input, extra_context)
-    except GenerateProjectFailedError as e:
+    except (GenerateProjectFailedError, ArbitraryProjectDownloadFailed) as e:
         raise UserException(str(e))

samcli/lib/utils/osutils.py

@@ -6,10 +6,15 @@
 import os
 import shutil
 import tempfile
+import logging
+import contextlib
 
 from contextlib import contextmanager
 
 
+LOG = logging.getLogger(__name__)
+
+
 @contextmanager
 def mkdir_temp(mode=0o755):
     """
@@ -62,3 +67,71 @@ def stderr():
         Byte stream of stderr
     """
     return sys.stderr.buffer
+
+
+def remove(path):
+    if path:
+        try:
+            os.remove(path)
+        except OSError:
+            pass
+
+
+@contextlib.contextmanager
+def tempfile_platform_independent():
+    # NOTE(TheSriram): Setting delete=False is specific to windows.
+    # https://docs.python.org/3/library/tempfile.html#tempfile.NamedTemporaryFile
+    _tempfile = tempfile.NamedTemporaryFile(delete=False)
+    try:
+        yield _tempfile
+    finally:
+        _tempfile.close()
+        remove(_tempfile.name)
+
+
+# NOTE: Py3.8 or higher has a ``dir_exist_ok=True`` parameter to provide this functionality.
+#       This method can be removed if we stop supporting Py37
+def copytree(source, destination, ignore=None):
+    """
+    Similar to shutil.copytree except that it removes the limitation that the destination directory should
+    be present.
+    :type source: str
+    :param source:
+        Path to the source folder to copy
+    :type destination: str
+    :param destination:
+        Path to destination folder
+    :type ignore: function
+    :param ignore:
+        A function that returns a set of file names to ignore, given a list of available file names. Similar to the
+        ``ignore`` property of ``shutils.copytree`` method
+    """
+
+    if not os.path.exists(destination):
+        os.makedirs(destination)
+
+        try:
+            # Let's try to copy the directory metadata from source to destination
+            shutil.copystat(source, destination)
+        except OSError as ex:
+            # Can't copy file access times in Windows
+            LOG.debug("Unable to copy file access times from %s to %s", source, destination, exc_info=ex)
+
+    names = os.listdir(source)
+    if ignore is not None:
+        ignored_names = ignore(source, names)
+    else:
+        ignored_names = set()
+
+    for name in names:
+        # Skip ignored names
+        if name in ignored_names:
+            continue
+
+        new_source = os.path.join(source, name)
+        new_destination = os.path.join(destination, name)
+
+        if os.path.isdir(new_source):
+            copytree(new_source, new_destination, ignore=ignore)
+        else:
+            shutil.copy2(new_source, new_destination)

samcli/lib/utils/temp_file_utils.py

@@ -2,25 +2,4 @@
 Helper functions for temporary files
 """
 import os
-import contextlib
 import tempfile
-
-
-def remove(path):
-    if path:
-        try:
-            os.remove(path)
-        except OSError:
-            pass
-
-
-@contextlib.contextmanager
-def tempfile_platform_independent():
-    # NOTE(TheSriram): Setting delete=False is specific to windows.
-    # https://docs.python.org/3/library/tempfile.html#tempfile.NamedTemporaryFile
-    _tempfile = tempfile.NamedTemporaryFile(delete=False)
-    try:
-        yield _tempfile
-    finally:
-        _tempfile.close()
-        remove(_tempfile.name)

samcli/local/init/__init__.py

@@ -4,11 +4,14 @@
 import itertools
 import logging
 
-from cookiecutter.exceptions import CookiecutterException
+from pathlib import Path
+
+from cookiecutter.exceptions import CookiecutterException, RepositoryNotFound
 from cookiecutter.main import cookiecutter
 
 from samcli.local.common.runtime_template import RUNTIME_DEP_TEMPLATE_MAPPING
 from samcli.local.init.exceptions import GenerateProjectFailedError
+from .arbitrary_project import generate_non_cookiecutter_project
 
 LOG = logging.getLogger(__name__)
 
@@ -76,5 +79,15 @@ def generate_project(
     try:
         LOG.debug("Baking a new template with cookiecutter with all parameters")
         cookiecutter(**params)
+    except RepositoryNotFound as e:
+        # cookiecutter.json is not found in the template. Let's just clone it directly without using cookiecutter
+        # and call it done.
+        LOG.debug(
+            "Unable to find cookiecutter.json in the project. Downloading it directly without treating "
+            "it as a cookiecutter template"
+        )
+        project_output_dir = str(Path(output_dir, name)) if name else output_dir
+        generate_non_cookiecutter_project(location=params["template"], output_dir=project_output_dir)
+
     except CookiecutterException as e:
         raise GenerateProjectFailedError(project=name, provider_error=e)

samcli/local/init/arbitrary_project.py

@@ -0,0 +1,94 @@
+"""
+Initialize an arbitrary project
+"""
+
+import functools
+import logging
+
+from cookiecutter import repository
+from cookiecutter import config
+
+from samcli.lib.utils import osutils
+from .exceptions import ArbitraryProjectDownloadFailed
+
+
+LOG = logging.getLogger(__name__)
+
+
+def generate_non_cookiecutter_project(location, output_dir):
+    """
+    Uses Cookiecutter APIs to download a project at given ``location`` to the ``output_dir``.
+    This does *not* run cookiecutter on the downloaded project.
+
+    Parameters
+    ----------
+    location : str
+        Path to where the project is. This supports all formats of location cookiecutter supports
+        (ex: zip, git, ssh, hg, local zipfile)
+
+        NOTE: This value *cannot* be a local directory. We didn't see a value in simply copying the directory
+        contents to ``output_dir`` without any processing.
+
+    output_dir : str
+        Directory where the project should be downloaded to
+
+    Returns
+    -------
+    str
+        Name of the directory where the project was downloaded to.
+
+    Raises
+    ------
+    cookiecutter.exception.CookiecutterException if download failed for some reason
+    """
+
+    LOG.debug("Downloading project from %s to %s", location, output_dir)
+
+    # Don't prompt ever
+    no_input = True
+
+    # Expand abbreviations in URL such as gh:awslabs/aws-sam-cli
+    location = repository.expand_abbreviations(location, config.BUILTIN_ABBREVIATIONS)
+
+    # If this is a zip file, download and unzip into output directory
+    if repository.is_zip_file(location):
+        LOG.debug("%s location is a zip file", location)
+        download_fn = functools.partial(
+            repository.unzip, zip_uri=location, is_url=repository.is_repo_url(location), no_input=no_input
+        )
+
+    # Else, treat it as a git/hg/ssh URL and try to clone
+    elif repository.is_repo_url(location):
+        LOG.debug("%s location is a source control repository", location)
+        download_fn = functools.partial(repository.clone, repo_url=location, no_input=no_input)
+
+    else:
+        raise ArbitraryProjectDownloadFailed(msg="Unsupported location {location}".format(location=location))
+
+    return _download_and_copy(download_fn, output_dir)
+
+
+def _download_and_copy(download_fn, output_dir):
+    """
+    Runs the download function to download files into a temporary directory and then copy the files over to
+    the ``output_dir``
+
+    Parameters
+    ----------
+    download_fn : function
+        Method to be called to download. It needs to accept a parameter called `clone_to_dir`. This will be
+        set to the temporary directory
+
+    output_dir : str
+        Path to the directory where files will be copied to
+
+    Returns
+    -------
+    output_dir
+    """
+
+    with osutils.mkdir_temp() as tempdir:
+        downloaded_dir = download_fn(clone_to_dir=tempdir)
+        osutils.copytree(downloaded_dir, output_dir)
+
+    return output_dir

samcli/local/init/exceptions.py

@@ -13,4 +13,8 @@ def __init__(self, **kwargs):
 
 
 class GenerateProjectFailedError(InitErrorException):
-    fmt = "An error occurred while generating this {project}: {provider_error}"
+    fmt = "An error occurred while generating this project {project}: {provider_error}"
+
+
+class ArbitraryProjectDownloadFailed(InitErrorException):
+    fmt = "An error occurred when downloading this project: {msg}"