Add --readonly-app-data option (#2009)

docs/changelog/2009.feature.rst

@@ -0,0 +1,17 @@
+
+Add ``--read-only-app-data`` option to allow for creation based on an existing
+app data cache which is non-writable.  This may be useful (for example) to
+produce a docker image where the app-data is pre-populated.
+
+.. code-block:: dockerfile
+
+    ENV \
+        VIRTUALENV_OVERRIDE_APP_DATA=/opt/virtualenv/cache \
+        VIRTUALENV_SYMLINK_APP_DATA=1
+    RUN virtualenv venv && rm -rf venv
+    ENV VIRTUALENV_READ_ONLY_APP_DATA=1
+    USER nobody
+    # this virtualenv has symlinks into the read-only app-data cache
+    RUN virtualenv /tmp/venv
+
+Patch by :user:`asottile`.

src/virtualenv/app_data/__init__.py

@@ -5,58 +5,53 @@
 
 import logging
 import os
-from argparse import Action, ArgumentError
 
 from appdirs import user_data_dir
 
 from .na import AppDataDisabled
+from .read_only import ReadOnlyAppData
 from .via_disk_folder import AppDataDiskFolder
 from .via_tempdir import TempAppData
 
 
-class AppDataAction(Action):
-    def __call__(self, parser, namespace, values, option_string=None):
-        folder = self._check_folder(values)
-        if folder is None:
-            raise ArgumentError("app data path {} is not valid".format(values))
-        setattr(namespace, self.dest, AppDataDiskFolder(folder))
-
-    @staticmethod
-    def _check_folder(folder):
-        folder = os.path.abspath(folder)
-        if not os.path.isdir(folder):
-            try:
-                os.makedirs(folder)
-                logging.debug("created app data folder %s", folder)
-            except OSError as exception:
-                logging.info("could not create app data folder %s due to %r", folder, exception)
-                return None
-        write_enabled = os.access(folder, os.W_OK)
-        if write_enabled:
-            return folder
-        logging.debug("app data folder %s has no write access", folder)
-        return None
+def _default_app_data_dir():  # type: () -> str
+    key = str("VIRTUALENV_OVERRIDE_APP_DATA")
+    if key in os.environ:
+        return os.environ[key]
+    else:
+        return user_data_dir(appname="virtualenv", appauthor="pypa")
+
+
+def make_app_data(folder, **kwargs):
+    read_only = kwargs.pop("read_only")
+    if kwargs:  # py3+ kwonly
+        raise TypeError("unexpected keywords: {}")
+
+    if folder is None:
+        folder = _default_app_data_dir()
+    folder = os.path.abspath(folder)
 
-    @staticmethod
-    def default():
-        for folder in AppDataAction._app_data_candidates():
-            folder = AppDataAction._check_folder(folder)
-            if folder is not None:
-                return AppDataDiskFolder(folder)
-        return AppDataDisabled()
+    if read_only:
+        return ReadOnlyAppData(folder)
 
-    @staticmethod
-    def _app_data_candidates():
-        key = str("VIRTUALENV_OVERRIDE_APP_DATA")
-        if key in os.environ:
-            yield os.environ[key]
-        else:
-            yield user_data_dir(appname="virtualenv", appauthor="pypa")
+    if not os.path.isdir(folder):
+        try:
+            os.makedirs(folder)
+            logging.debug("created app data folder %s", folder)
+        except OSError as exception:
+            logging.info("could not create app data folder %s due to %r", folder, exception)
+
+    if os.access(folder, os.W_OK):
+        return AppDataDiskFolder(folder)
+    else:
+        logging.debug("app data folder %s has no write access", folder)
+        return TempAppData()
 
 
 __all__ = (
+    "AppDataDisabled",
     "AppDataDiskFolder",
+    "ReadOnlyAppData",
     "TempAppData",
-    "AppDataAction",
-    "AppDataDisabled",
+    "make_app_data",
 )

src/virtualenv/app_data/base.py

@@ -31,6 +31,10 @@ def py_info(self, path):
     def py_info_clear(self):
         raise NotImplementedError
 
+    @property
+    def can_update(self):
+        raise NotImplementedError
+
     @abstractmethod
     def embed_update_log(self, distribution, for_py_version):
         raise NotImplementedError

src/virtualenv/app_data/na.py

@@ -8,6 +8,9 @@
 class AppDataDisabled(AppData):
     """No application cache available (most likely as we don't have write permissions)"""
 
+    transient = True
+    can_update = False
+
     def __init__(self):
         pass
 
@@ -40,10 +43,6 @@ def house(self):
     def wheel_image(self, for_py_version, name):
         raise self.error
 
-    @property
-    def transient(self):
-        return True
-
     def py_info_clear(self):
         """"""
 

src/virtualenv/app_data/read_only.py

@@ -0,0 +1,34 @@
+import os.path
+
+from virtualenv.util.lock import NoOpFileLock
+
+from .via_disk_folder import AppDataDiskFolder, PyInfoStoreDisk
+
+
+class ReadOnlyAppData(AppDataDiskFolder):
+    can_update = False
+
+    def __init__(self, folder):  # type: (str) -> None
+        if not os.path.isdir(folder):
+            raise RuntimeError("read-only app data directory {} does not exist".format(folder))
+        self.lock = NoOpFileLock(folder)
+
+    def reset(self):  # type: () -> None
+        raise RuntimeError("read-only app data does not support reset")
+
+    def py_info_clear(self):  # type: () -> None
+        raise NotImplementedError
+
+    def py_info(self, path):
+        return _PyInfoStoreDiskReadOnly(self.py_info_at, path)
+
+    def embed_update_log(self, distribution, for_py_version):
+        raise NotImplementedError
+
+
+class _PyInfoStoreDiskReadOnly(PyInfoStoreDisk):
+    def write(self, content):
+        raise RuntimeError("read-only app data python info cannot be updated")
+
+
+__all__ = ("ReadOnlyAppData",)

src/virtualenv/app_data/via_disk_folder.py

@@ -46,15 +46,17 @@ class AppDataDiskFolder(AppData):
     Store the application data on the disk within a folder layout.
     """
 
+    transient = False
+    can_update = True
+
     def __init__(self, folder):
         self.lock = ReentrantFileLock(folder)
 
     def __repr__(self):
-        return "{}".format(self.lock.path)
+        return "{}({})".format(type(self).__name__, self.lock.path)
 
-    @property
-    def transient(self):
-        return False
+    def __str__(self):
+        return str(self.lock.path)
 
     def reset(self):
         logging.debug("reset app data folder %s", self.lock.path)

src/virtualenv/app_data/via_tempdir.py

@@ -9,6 +9,9 @@
 
 
 class TempAppData(AppDataDiskFolder):
+    transient = True
+    can_update = False
+
     def __init__(self):
         super(TempAppData, self).__init__(folder=mkdtemp())
         logging.debug("created temporary app data folder %s", self.lock.path)
@@ -21,8 +24,4 @@ def close(self):
         safe_delete(self.lock.path)
 
     def embed_update_log(self, distribution, for_py_version):
-        return None
-
-    @property
-    def transient(self):
-        return True
+        raise NotImplementedError

src/virtualenv/run/__init__.py

@@ -1,8 +1,9 @@
 from __future__ import absolute_import, unicode_literals
 
 import logging
+from functools import partial
 
-from ..app_data import AppDataAction, AppDataDisabled, TempAppData
+from ..app_data import make_app_data
 from ..config.cli.parser import VirtualEnvConfigParser
 from ..report import LEVELS, setup_report
 from ..run.session import Session
@@ -88,32 +89,31 @@ def handle_extra_commands(options):
 
 
 def load_app_data(args, parser, options):
+    parser.add_argument(
+        "--read-only-app-data",
+        action="store_true",
+        help="use app data folder in read-only mode (write operations will fail with error)",
+    )
+    options, _ = parser.parse_known_args(args, namespace=options)
+
     # here we need a write-able application data (e.g. the zipapp might need this for discovery cache)
-    default_app_data = AppDataAction.default()
     parser.add_argument(
         "--app-data",
-        dest="app_data",
-        action=AppDataAction,
-        default="<temp folder>" if isinstance(default_app_data, AppDataDisabled) else default_app_data,
         help="a data folder used as cache by the virtualenv",
+        type=partial(make_app_data, read_only=options.read_only_app_data),
+        default=make_app_data(None, read_only=options.read_only_app_data),
     )
     parser.add_argument(
         "--reset-app-data",
-        dest="reset_app_data",
         action="store_true",
         help="start with empty app data folder",
-        default=False,
     )
     parser.add_argument(
         "--upgrade-embed-wheels",
-        dest="upgrade_embed_wheels",
         action="store_true",
         help="trigger a manual update of the embedded wheels",
-        default=False,
     )
     options, _ = parser.parse_known_args(args, namespace=options)
-    if options.app_data == "<temp folder>":
-        options.app_data = TempAppData()
     if options.reset_app_data:
         options.app_data.reset()
     return options

src/virtualenv/seed/embed/via_app_data/via_app_data.py

@@ -11,9 +11,7 @@
 from virtualenv.info import fs_supports_symlink
 from virtualenv.seed.embed.base_embed import BaseEmbed
 from virtualenv.seed.wheels import get_wheel
-from virtualenv.util.lock import _CountedFileLock
 from virtualenv.util.path import Path
-from virtualenv.util.six import ensure_text
 
 from .pip_install.copy import CopyPipInstall
 from .pip_install.symlink import SymlinkPipInstall
@@ -52,7 +50,8 @@ def _install(name, wheel):
                     key = Path(installer_class.__name__) / wheel.path.stem
                     wheel_img = self.app_data.wheel_image(creator.interpreter.version_release_str, key)
                     installer = installer_class(wheel.path, creator, wheel_img)
-                    with _CountedFileLock(ensure_text(str(wheel_img.parent / "{}.lock".format(wheel_img.name)))):
+                    parent = self.app_data.lock / wheel_img.parent
+                    with parent.non_reentrant_lock_for_key(wheel_img.name):
                         if not installer.has_image():
                             installer.build_image()
                     installer.install(creator.interpreter.version_info)

src/virtualenv/seed/wheels/bundle.py

@@ -1,7 +1,5 @@
 from __future__ import absolute_import, unicode_literals
 
-from virtualenv.app_data import AppDataDiskFolder, TempAppData
-
 from ..wheels.embed import get_embed_wheel
 from .periodic_update import periodic_update
 from .util import Version, Wheel, discover_wheels
@@ -16,7 +14,7 @@ def from_bundle(distribution, version, for_py_version, search_dirs, app_data, do
 
     if version != Version.embed:
         # 2. check if we have upgraded embed
-        if isinstance(app_data, AppDataDiskFolder) and not isinstance(app_data, TempAppData):
+        if app_data.can_update:
             wheel = periodic_update(distribution, for_py_version, wheel, search_dirs, app_data, do_periodic_update)
 
         # 3. acquire from extra search dir