[PR196 3/3] Adding support for partial workspace cleaning

.travis.yml

@@ -35,8 +35,7 @@ matrix:
 before_install:
   # Install catkin_tools dependencies
   - source .travis.before_install.bash
-  - pip install setuptools argparse catkin-pkg distribute PyYAML psutil trollius
-  - pip install git+https://github.com/osrf/osrf_pycommon.git
+  - pip install setuptools argparse catkin-pkg distribute PyYAML psutil trollius osrf_pycommon
 install:
   # Install catkin_tools
   - python setup.py develop

catkin_tools/commands/catkin.py

@@ -172,7 +172,7 @@ def catkin_main(sysargs):
 
     # Get colors config
     no_color = False
-    force_color = False
+    force_color = os.environ.get('CATKIN_TOOLS_FORCE_COLOR', False)
     for arg in sysargs:
         if arg == '--no-color':
             no_color = True

catkin_tools/common.py

@@ -327,6 +327,35 @@ def get_recursive_build_dependants_in_workspace(package_name, ordered_packages):
     return recursive_dependants
 
 
+def get_recursive_run_dependants_in_workspace(package_name, ordered_packages):
+    """Calculates the recursive run dependants of a package which are also in
+    the ordered_packages
+
+    :param package: package for which the recursive depends should be calculated
+    :type package: :py:class:`catkin_pkg.package.Package`
+    :param ordered_packages: packages in the workspace, ordered topologically
+    :type ordered_packages: list(tuple(package path,
+        :py:class:`catkin_pkg.package.Package`))
+    :returns: list of package path, package object tuples which are the
+        recursive run depends for the given package
+    :rtype: list(tuple(package path, :py:class:`catkin_pkg.package.Package`))
+    """
+    recursive_dependants = list()
+
+    for pth, pkg in reversed(ordered_packages):
+        # Break if this is one to check
+        if pkg.name == package_name:
+            break
+
+        # Check if this package depends on the target package
+        deps = get_recursive_run_depends_in_workspace([pkg], ordered_packages)
+        deps_names = [p.name for _, p in deps]
+        if package_name in deps_names:
+            recursive_dependants.insert(0, (pth, pkg))
+
+    return recursive_dependants
+
+
 def is_tty(stream):
     """Returns True if the given stream is a tty, else False"""
     return hasattr(stream, 'isatty') and stream.isatty()
@@ -555,22 +584,29 @@ def get_build_type(package):
 
 
 def find_enclosing_package(search_start_path=None, ws_path=None, warnings=None, symlinks=True):
-    """Get the package containing the current directory."""
+    """Get the package containing a specific directory.
 
-    search_start_path = search_start_path or getcwd(symlinks=symlinks)
-    child_path = ''
+    :param search_start_path: The path to crawl upward to find a package, CWD if None
+    :param ws_path: The path at which the search should stop
+    """
+
+    search_path = search_start_path or getcwd(symlinks=symlinks)
+    stop_path = ws_path or '/'
+    child_path = '.'
 
-    while True:
-        pkgs = find_packages(search_start_path, warnings=warnings)
+    while search_path != stop_path:
+        # Find packages under the search path
+        try:
+            pkgs = find_packages(search_path, warnings=warnings)
+        except:
+            return None
 
-        # Check if the previous directory is a catkin package
+        # Check if the directory is a catkin package
         if child_path in pkgs:
             return pkgs[child_path].name
 
-        # Update search path or end
-        (search_start_path, child_path) = os.path.split(search_start_path)
-        if len(child_path) == 0 or search_start_path == ws_path:
-            break
+        # Update search path
+        (search_path, child_path) = os.path.split(search_path)
 
     return None
 

catkin_tools/context.py

@@ -49,6 +49,7 @@ class Context(object):
     This context can be locked, so that changing the members is prevented.
     """
 
+    DEFAULT_LOG_SPACE = 'logs'
     DEFAULT_SOURCE_SPACE = 'src'
     DEFAULT_BUILD_SPACE = 'build'
     DEFAULT_DEVEL_SPACE = 'devel'
@@ -57,6 +58,7 @@ class Context(object):
     STORED_KEYS = [
         'extend_path',
         'source_space',
+        'log_space',
         'build_space',
         'devel_space',
         'install_space',
@@ -193,6 +195,7 @@ def __init__(
         profile=None,
         extend_path=None,
         source_space=None,
+        log_space=None,
         build_space=None,
         devel_space=None,
         install_space=None,
@@ -220,6 +223,8 @@ def __init__(
         :type extend_path: str
         :param source_space: relative location of source space, defaults to '<workspace>/src'
         :type source_space: str
+        :param log_space: relative location of log space, defaults to '<workspace>/logs'
+        :type log_space: str
         :param build_space: relativetarget location of build space, defaults to '<workspace>/build'
         :type build_space: str
         :param devel_space: relative target location of devel space, defaults to '<workspace>/devel'
@@ -268,6 +273,7 @@ def __init__(
         self.profile = profile
 
         self.source_space = Context.DEFAULT_SOURCE_SPACE if source_space is None else source_space
+        self.log_space = Context.DEFAULT_LOG_SPACE + ss if ss or log_space is None else log_space
         self.build_space = Context.DEFAULT_BUILD_SPACE + ss if ss or build_space is None else build_space
         self.devel_space = Context.DEFAULT_DEVEL_SPACE + ss if ss or devel_space is None else devel_space
         self.install_space = Context.DEFAULT_INSTALL_SPACE + ss if ss or install_space is None else install_space
@@ -278,7 +284,7 @@ def __init__(
         self.blacklist = blacklist or []
 
         # Handle build options
-        self.devel_layout = devel_layout if devel_layout else 'merged'
+        self.devel_layout = devel_layout if devel_layout else 'linked'
         self.install = install
         self.isolate_install = isolate_install
 
@@ -401,16 +407,18 @@ def summary(self, notes=[]):
                 clr("@{cf}Profile:@|                     @{yf}{profile}@|"),
                 clr("@{cf}Extending:@|        {extend_mode} @{yf}{extend}@|"),
                 clr("@{cf}Workspace:@|                   @{yf}{_Context__workspace}@|"),
+            ],
+            [
                 clr("@{cf}Source Space:@|      {source_missing} @{yf}{_Context__source_space_abs}@|"),
+                clr("@{cf}Log Space:@|         {log_missing} @{yf}{_Context__log_space_abs}@|"),
                 clr("@{cf}Build Space:@|       {build_missing} @{yf}{_Context__build_space_abs}@|"),
                 clr("@{cf}Devel Space:@|       {devel_missing} @{yf}{_Context__devel_space_abs}@|"),
                 clr("@{cf}Install Space:@|     {install_missing} @{yf}{_Context__install_space_abs}@|"),
-                clr("@{cf}DESTDIR:@|                     @{yf}{_Context__destdir}@|"),
+                clr("@{cf}DESTDIR:@|           {destdir_missing} @{yf}{_Context__destdir}@|")
             ],
             [
                 clr("@{cf}Devel Space Layout:@|          @{yf}{_Context__devel_layout}@|"),
-                clr("@{cf}Install Packages:@|            @{yf}{_Context__install}@|"),
-                clr("@{cf}Isolate Installs:@|            @{yf}{_Context__isolate_install}@|"),
+                clr("@{cf}Install Space Layout:@|        @{yf}{install_layout}@|"),
             ],
             [
                 clr("@{cf}Additional CMake Args:@|       @{yf}{cmake_args}@|"),
@@ -441,21 +449,31 @@ def summary(self, notes=[]):
             extend_value = 'None'
             extend_mode = clr('          ')
 
-        def existence_str(path):
-            return clr(' @{gf}[exists]@|' if os.path.exists(path) else '@{rf}[missing]@|')
+        def existence_str(path, used=True):
+            if used:
+                return clr(' @{gf}[exists]@|' if os.path.exists(path) else '@{rf}[missing]@|')
+            else:
+                return clr(' @{bf}[unused]@|')
+
+        install_layout = 'None'
+        if self.__install:
+            install_layout = 'merged' if not self.__isolate_install else 'isolated'
 
         subs = {
             'profile': self.profile,
             'extend_mode': extend_mode,
             'extend': extend_value,
+            'install_layout': install_layout,
             'cmake_prefix_path': (self.cmake_prefix_path or ['Empty']),
             'cmake_args': ' '.join(self.__cmake_args or ['None']),
             'make_args': ' '.join(self.__make_args + self.__jobs_args or ['None']),
             'catkin_make_args': ', '.join(self.__catkin_make_args or ['None']),
             'source_missing': existence_str(self.source_space_abs),
+            'log_missing': existence_str(self.log_space_abs),
             'build_missing': existence_str(self.build_space_abs),
             'devel_missing': existence_str(self.devel_space_abs),
-            'install_missing': existence_str(self.install_space_abs),
+            'install_missing': existence_str(self.install_space_abs, used=self.__install),
+            'destdir_missing': existence_str(self.destdir, used=self.destdir),
             'whitelisted_packages': ' '.join(self.__whitelist or ['None']),
             'blacklisted_packages': ' '.join(self.__blacklist or ['None']),
         }
@@ -539,6 +557,21 @@ def initialized(self):
         """Check if this context is initialized."""
         return self.workspace == find_enclosing_workspace(self.workspace)
 
+    @property
+    def log_space_abs(self):
+        return self.__log_space_abs
+
+    @property
+    def log_space(self):
+        return self.__log_space
+
+    @log_space.setter
+    def log_space(self, value):
+        if self.__locked:
+            raise RuntimeError("Setting of context members is not allowed while locked.")
+        self.__log_space = value
+        self.__log_space_abs = os.path.join(self.__workspace, value)
+
     @property
     def build_space_abs(self):
         return self.__build_space_abs
@@ -551,7 +584,6 @@ def build_space(self):
     def build_space(self, value):
         if self.__locked:
             raise RuntimeError("Setting of context members is not allowed while locked.")
-        # TODO: check that build space was not run with a different context before
         self.__build_space = value
         self.__build_space_abs = os.path.join(self.__workspace, value)
 
@@ -567,7 +599,6 @@ def devel_space(self):
     def devel_space(self, value):
         if self.__locked:
             raise RuntimeError("Setting of context members is not allowed while locked.")
-        # TODO: check that devel space was not run with a different context before
         self.__devel_space = value
         self.__devel_space_abs = os.path.join(self.__workspace, value)
 
@@ -583,7 +614,6 @@ def install_space(self):
     def install_space(self, value):
         if self.__locked:
             raise RuntimeError("Setting of context members is not allowed while locked.")
-        # TODO: check that install space was not run with a different context before
         self.__install_space = value
         self.__install_space_abs = os.path.join(self.__workspace, value)
 
@@ -730,14 +760,14 @@ def blacklist(self, value):
         self.__blacklist = value
 
     @property
-    def linked_devel_path(self):
+    def private_devel_path(self):
         """The path to the hidden directory in the develspace that
         contains the symbollically-linked isolated develspaces."""
-        return os.path.join(self.devel_space_abs, '.catkin_tools')
+        return os.path.join(self.devel_space_abs, '.private')
 
-    def package_linked_devel_path(self, package):
+    def package_private_devel_path(self, package):
         """The path to the linked devel space for a given package."""
-        return os.path.join(self.linked_devel_path, package.name)
+        return os.path.join(self.private_devel_path, package.name)
 
     def package_build_space(self, package):
         """Get the build directory for a specific package."""
@@ -752,7 +782,7 @@ def package_devel_space(self, package):
         elif self.isolate_devel:
             return os.path.join(self.devel_space_abs, package.name)
         elif self.link_devel:
-            return os.path.join(self.linked_devel_path, package.name)
+            return os.path.join(self.private_devel_path, package.name)
         else:
             raise ValueError('Unkown devel space layout: {}'.format(self.devel_layout))
 
@@ -788,3 +818,15 @@ def package_final_path(self, package):
                 return self.devel_space_abs
             else:
                 return self.package_devel_space(package)
+
+    def metadata_path(self):
+        """Get the path to the metadata directory for this profile."""
+        profile_path, _ = metadata.get_paths(self.workspace, self.profile)
+        return profile_path
+
+    def package_metadata_path(self, package=None):
+        """Get the workspace and profile-specific metadata path for a package"""
+        profile_path, _ = metadata.get_paths(self.workspace, self.profile)
+        if package is None:
+            return os.path.join(profile_path, 'packages')
+        return os.path.join(profile_path, 'packages', package.name)

catkin_tools/execution/controllers.py

@@ -20,16 +20,13 @@
     from Queue import Empty
 
 import math
-import os
-import operator
 import sys
 import threading
 import time
 
 from catkin_tools.common import disable_wide_log
 from catkin_tools.common import format_time_delta
 from catkin_tools.common import format_time_delta_short
-from catkin_tools.common import log
 from catkin_tools.common import remove_ansi_escape
 from catkin_tools.common import terminal_width
 from catkin_tools.common import wide_log

catkin_tools/execution/executor.py

@@ -15,8 +15,6 @@
 from __future__ import print_function
 
 import os
-import time
-import threading
 import traceback
 
 from itertools import tee
@@ -147,6 +145,11 @@ def async_job(verb, job, threadpool, locks, event_queue, log_path):
                         logger,
                         event_queue))
                 except:
+                    logger.err('Stage `{}` failed with arguments:'.format(stage.label))
+                    for arg_val in stage.args:
+                        logger.err('  {}'.format(arg_val))
+                    for arg_name, arg_val in stage.kwargs.items():
+                        logger.err('  {}: {}'.format(arg_name, arg_val))
                     logger.err(str(traceback.format_exc()))
                     retcode = 3
             else:
@@ -215,6 +218,10 @@ def execute_jobs(
     # List of jobs whose deps failed
     abandoned_jobs = []
 
+    # Make sure job server has been initialized
+    if not job_server.initialized():
+        raise RuntimeError('JobServer has not been initialized.')
+
     # Create a thread pool executor for blocking python stages in the asynchronous jobs
     threadpool = ThreadPoolExecutor(max_workers=job_server.max_jobs())
 

catkin_tools/execution/job_server.py

@@ -248,6 +248,11 @@ def _running_jobs(cls):
         return cls._max_jobs
 
 
+def initialized():
+    """Return True if the job server has been initialized."""
+    return JobServer._initialized
+
+
 def initialize(max_jobs=None, max_load=None, max_mem=None, gnu_make_enabled=False):
     """
     Initialize the global GNU Make jobserver.

catkin_tools/execution/stages.py

@@ -13,7 +13,6 @@
 # limitations under the License.
 
 import os
-import sys
 
 from catkin_tools.common import string_type
 
@@ -166,6 +165,9 @@ def __init__(
             raise ValueError('Function stage must be callable.')
         super(FunctionStage, self).__init__(label, logger_factory, occupy_job, locked_resource)
 
+        self.args = args
+        self.kwargs = kwargs
+
         def function_proxy(logger, event_queue):
             return function(logger, event_queue, *args, **kwargs)
         self.function = function_proxy