From 5608ea840ccf480c7808074ac382527d97aa7e15 Mon Sep 17 00:00:00 2001 From: Jonathan Bohren Date: Fri, 12 Feb 2016 01:48:05 -0500 Subject: [PATCH] clean: Adding support for partial workspace cleaning based on #196 --- catkin_tools/context.py | 20 +- catkin_tools/execution/executor.py | 4 + catkin_tools/execution/job_server.py | 5 + catkin_tools/jobs/catkin.py | 185 ++++++---- catkin_tools/jobs/cmake.py | 120 +++---- catkin_tools/jobs/commands/cmake.py | 15 + catkin_tools/jobs/utils.py | 94 ++++- catkin_tools/verbs/catkin_build/build.py | 7 +- catkin_tools/verbs/catkin_clean/clean.py | 203 +++++++++++ catkin_tools/verbs/catkin_clean/cli.py | 333 ++++++++++++------ catkin_tools/verbs/catkin_clean/color.py | 69 ++++ tests/system/verbs/catkin_build/test_build.py | 2 +- 12 files changed, 777 insertions(+), 280 deletions(-) create mode 100644 catkin_tools/verbs/catkin_clean/clean.py create mode 100644 catkin_tools/verbs/catkin_clean/color.py diff --git a/catkin_tools/context.py b/catkin_tools/context.py index 44380068..bce641a9 100644 --- a/catkin_tools/context.py +++ b/catkin_tools/context.py @@ -730,14 +730,14 @@ def blacklist(self, value): self.__blacklist = value @property - def linked_devel_path(self): + def private_devel_path(self): """The path to the hidden directory in the develspace that contains the symbollically-linked isolated develspaces.""" - return os.path.join(self.devel_space_abs, '.catkin_tools') + return os.path.join(self.devel_space_abs, '.private') - def package_linked_devel_path(self, package): + def package_private_devel_path(self, package): """The path to the linked devel space for a given package.""" - return os.path.join(self.linked_devel_path, package.name) + return os.path.join(self.private_devel_path, package.name) def package_build_space(self, package): """Get the build directory for a specific package.""" @@ -752,7 +752,7 @@ def package_devel_space(self, package): elif self.isolate_devel: return os.path.join(self.devel_space_abs, package.name) elif self.link_devel: - return os.path.join(self.linked_devel_path, package.name) + return os.path.join(self.private_devel_path, package.name) else: raise ValueError('Unkown devel space layout: {}'.format(self.devel_layout)) @@ -788,3 +788,13 @@ def package_final_path(self, package): return self.devel_space_abs else: return self.package_devel_space(package) + + def metadata_path(self): + """Get the path to the metadata directory for this profile.""" + profile_path, _ = metadata.get_paths(self.workspace, self.profile) + return profile_path + + def package_metadata_path(self, package): + """Get the workspace and profile-specific metadata path for a package""" + profile_path, _ = metadata.get_paths(self.workspace, self.profile) + return os.path.join(profile_path, 'packages', package.name) diff --git a/catkin_tools/execution/executor.py b/catkin_tools/execution/executor.py index f0cf4a4d..a294f3e0 100644 --- a/catkin_tools/execution/executor.py +++ b/catkin_tools/execution/executor.py @@ -215,6 +215,10 @@ def execute_jobs( # List of jobs whose deps failed abandoned_jobs = [] + # Make sure job server has been initialized + if not job_server.initialized(): + raise RuntimeError('JobServer has not been initialized.') + # Create a thread pool executor for blocking python stages in the asynchronous jobs threadpool = ThreadPoolExecutor(max_workers=job_server.max_jobs()) diff --git a/catkin_tools/execution/job_server.py b/catkin_tools/execution/job_server.py index f23b05f5..a63077e3 100644 --- a/catkin_tools/execution/job_server.py +++ b/catkin_tools/execution/job_server.py @@ -248,6 +248,11 @@ def _running_jobs(cls): return cls._max_jobs +def initialized(): + """Return True if the job server has been initialized.""" + return JobServer._initialized + + def initialize(max_jobs=None, max_load=None, max_mem=None, gnu_make_enabled=False): """ Initialize the global GNU Make jobserver. diff --git a/catkin_tools/jobs/catkin.py b/catkin_tools/jobs/catkin.py index d185795f..005b65ed 100644 --- a/catkin_tools/jobs/catkin.py +++ b/catkin_tools/jobs/catkin.py @@ -30,13 +30,15 @@ from catkin_tools.execution.stages import FunctionStage from .commands.cmake import CMAKE_EXEC +from .commands.cmake import CMAKE_INSTALL_MANIFEST_FILENAME from .commands.cmake import CMakeIOBufferProtocol from .commands.cmake import CMakeMakeIOBufferProtocol +from .commands.cmake import get_installed_files from .commands.make import MAKE_EXEC from .utils import get_env_loader from .utils import makedirs -from .utils import rmdirs +from .utils import rmfiles def generate_prebuild_package(build_space_abs, devel_space_abs, force): @@ -50,7 +52,7 @@ def generate_prebuild_package(build_space_abs, devel_space_abs, force): """ # Get the path to the prebuild package - prebuild_path = os.path.join(devel_space_abs, '.catkin_tools', 'catkin_tools_prebuild') + prebuild_path = os.path.join(devel_space_abs, '.private', 'catkin_tools_prebuild') if not os.path.exists(prebuild_path): mkdir_p(prebuild_path) @@ -75,9 +77,10 @@ def generate_prebuild_package(build_space_abs, devel_space_abs, force): def clean_linked_files( logger, event_queue, - devel_space_abs, + metadata_path, files_that_collide, - files_to_clean): + files_to_clean, + dry_run): """Removes a list of files and adjusts collison counts for colliding files. This function synchronizes access to the devel collisions file. @@ -88,7 +91,7 @@ def clean_linked_files( """ # Get paths - devel_collisions_file_path = os.path.join(devel_space_abs, 'devel_collisions.txt') + devel_collisions_file_path = os.path.join(metadata_path, 'devel_collisions.txt') # Map from dest files to number of collisions dest_collisions = dict() @@ -115,12 +118,13 @@ def clean_linked_files( if n_collisions == 0: logger.out('Unlinking %s' % (dest_file)) # Remove this link - os.unlink(dest_file) - # Remove any non-empty directories containing this file - try: - os.removedirs(os.path.split(dest_file)[0]) - except OSError: - pass + if not dry_run: + os.unlink(dest_file) + # Remove any non-empty directories containing this file + try: + os.removedirs(os.path.split(dest_file)[0]) + except OSError: + pass # Update collisions if n_collisions > 1: @@ -131,32 +135,41 @@ def clean_linked_files( del dest_collisions[dest_file] # Load destination collisions file - with open(devel_collisions_file_path, 'w') as collisions_file: - collisions_writer = csv.writer(collisions_file, delimiter=' ', quotechar='"') - for dest_file, count in dest_collisions.items(): - collisions_writer.writerow([dest_file, count]) + if not dry_run: + with open(devel_collisions_file_path, 'w') as collisions_file: + collisions_writer = csv.writer(collisions_file, delimiter=' ', quotechar='"') + for dest_file, count in dest_collisions.items(): + collisions_writer.writerow([dest_file, count]) def unlink_devel_products( logger, event_queue, devel_space_abs, - package_name): + private_devel_path, + metadata_path, + package_metadata_path, + dry_run): """ Remove all files listed in the devel manifest for the given package, as well as any empty directories containing those files. :param devel_space_abs: Path to a merged devel space. - :param package_name: Name of the package whose files should be unlinked. + :param private_devel_path: Path to the private devel space + :param devel_manifest_path: Path to the directory containing the package's + catkin_tools metadata """ - # Get paths - linked_devel_path = get_linked_devel_package_path(devel_space_abs, package_name) - devel_manifest_path = get_linked_devel_manifest_path(devel_space_abs, package_name) - - if not os.path.exists(linked_devel_path) or not os.path.exists(devel_manifest_path): + # Check paths + if not os.path.exists(private_devel_path): + logger.err('Warning: No private devel path found at `{}`'.format(private_devel_path)) return 0 + devel_manifest_file_path = os.path.join(package_metadata_path, DEVEL_MANIFEST_FILENAME) + if not os.path.exists(devel_manifest_file_path): + logger.err('Error: No devel manifest found at `{}`'.format(devel_manifest_file_path)) + return 1 + # List of files to clean files_to_clean = [] @@ -181,7 +194,7 @@ def unlink_devel_products( # Remove all listed symli and empty directories which have been removed # after this build, and update the collision file - clean_linked_files(logger, event_queue, devel_space_abs, [], files_to_clean) + clean_linked_files(logger, event_queue, metadata_path, [], files_to_clean, dry_run) return 0 @@ -192,13 +205,17 @@ def link_devel_products( package_path, devel_manifest_path, source_devel_path, - dest_devel_path): + dest_devel_path, + metadata_path): """Link files from an isolated devel space into a merged one. This creates directories and symlinks in a merged devel space to a package's linked devel space. """ + # Create the devel manifest path if necessary + mkdir_p(devel_manifest_path) + # Construct manifest file path devel_manifest_file_path = os.path.join(devel_manifest_path, DEVEL_MANIFEST_FILENAME) @@ -274,7 +291,7 @@ def link_devel_products( # Remove all listed symlinks and empty directories which have been removed # after this build, and update the collision file - clean_linked_files(logger, event_queue, dest_devel_path, files_that_collide, files_to_clean) + clean_linked_files(logger, event_queue, metadata_path, files_that_collide, files_to_clean, dry_run=False) # Save the list of symlinked files with open(devel_manifest_file_path, 'w') as devel_manifest: @@ -408,9 +425,10 @@ def create_catkin_build_job(context, package, package_path, dependencies, force_ locked_resource='symlink-collisions-file', package=package, package_path=package_path, - devel_manifest_path=context.package_linked_devel_path(package), + devel_manifest_path=context.package_metadata_path(package), source_devel_path=context.package_devel_space(package), - dest_devel_path=context.devel_space_abs + dest_devel_path=context.devel_space_abs, + metadata_path=context.metadata_path() )) # Make install command, if installing @@ -430,73 +448,88 @@ def create_catkin_build_job(context, package, package_path, dependencies, force_ stages=stages) -def create_catkin_clean_job(context, package, package_path, dependencies): +def create_catkin_clean_job( + context, + package, + package_path, + dependencies, + dry_run, + clean_build, + clean_devel, + clean_install): """Generate a Job that cleans a catkin package""" stages = [] - # If the build space doesn't exist, do nothing build_space = context.package_build_space(package) - if not os.path.exists(build_space): - return Job(jid=package_name, deps=dependencies, stages=[]) - - # For isolated devel space, remove it entirely - if context.isolate_devel: - devel_space = os.path.join(context.devel_space_abs, package_name) - - stages.append(FunctionStage('clean', rmdirs, path=devel_space)) - return Job( - jid=package_name, - deps=dependencies, - stages=stages) - - elif context.link_devel: - devel_space = os.path.join(build_space, 'devel') - else: - devel_space = context.devel_space_abs - - # For isolated install space, remove it entirely - if context.isolate_install: - install_space = os.path.join(context.install_space_abs, package_name) - - stages.append(FunctionStage('clean', rmdirs, path=install_space)) - - return Job( - jid=package_name, - deps=dependencies, - stages=stages) - else: - install_space = context.install_space_abs - - if context.link_devel: - # Remove symlinked products + # Remove installed files + if clean_install: + installed_files = get_installed_files(context.package_metadata_path(package)) stages.append(FunctionStage( - 'unlink', - unlink_devel_products, - devel_space_abs=context.devel_space_abs, - package_name=package_name, - locked_resource='symlink-collisions-file' - )) - - # Remove devel space - stages.append(FunctionStage( - 'rmdevel', rmdirs, - path=context.package_devel_space(package))) + 'cleaninstall', + rmfiles, + paths=sorted(installed_files), + remove_empty=True, + empty_root=context.install_space_abs, + dry_run=dry_run)) + + # Remove products in develspace + if clean_devel: + if context.merge_devel: + # Remove build targets from devel space + stages.append(CommandStage( + 'clean', + [MAKE_EXEC, 'clean'] + make_args, + cwd=build_space, + )) + elif context.link_devel: + # Remove symlinked products + stages.append(FunctionStage( + 'unlink', + unlink_devel_products, + locked_resource='symlink-collisions-file', + devel_space_abs=context.devel_space_abs, + private_devel_path=context.package_private_devel_path(package), + metadata_path=context.metadata_path(), + package_metadata_path=context.package_metadata_path(package), + dry_run=dry_run + )) + + # Remove devel space + stages.append(FunctionStage( + 'rmdevel', + rmfiles, + paths=[context.package_private_devel_path(package)], + dry_run=dry_run)) + elif context.isolate_devel: + # Remove devel space + stages.append(FunctionStage( + 'rmdevel', + rmfiles, + paths=[context.package_devel_space(package)], + dry_run=dry_run)) # Remove build space - stages.append(FunctionStage('rmbuild', rmdirs, path=build_space)) + if clean_build: + stages.append(FunctionStage( + 'rmbuild', + rmfiles, + paths=[build_space], + dry_run=dry_run)) return Job( - jid=package_name, + jid=package.name, deps=dependencies, + env_loader=get_env_loader(package, context), stages=stages) description = dict( build_type='catkin', description="Builds a catkin package.", - create_build_job=create_catkin_build_job + create_build_job=create_catkin_build_job, + create_clean_job=create_catkin_clean_job ) diff --git a/catkin_tools/jobs/cmake.py b/catkin_tools/jobs/cmake.py index 3d9be82a..d4786a70 100644 --- a/catkin_tools/jobs/cmake.py +++ b/catkin_tools/jobs/cmake.py @@ -24,14 +24,16 @@ from catkin_tools.common import mkdir_p from .commands.cmake import CMAKE_EXEC +from .commands.cmake import CMAKE_INSTALL_MANIFEST_FILENAME from .commands.cmake import CMakeIOBufferProtocol from .commands.cmake import CMakeMakeIOBufferProtocol +from .commands.cmake import get_installed_files from .commands.make import MAKE_EXEC from .utils import get_env_loader from .utils import makedirs -from .utils import rmdirs from .utils import rmfile +from .utils import rmfiles from catkin_tools.execution.jobs import Job from catkin_tools.execution.stages import CommandStage @@ -50,9 +52,6 @@ class FileNotFoundError(OSError): pass -INSTALL_MANIFEST_FILENAME = 'install_manifest.txt' - - def copy_install_manifest( logger, event_queue, src_install_manifest_path, @@ -60,8 +59,8 @@ def copy_install_manifest( """Copy the install manifest file from one path to another,""" # Get file paths - src_install_manifest_file_path = os.path.join(src_install_manifest_path, INSTALL_MANIFEST_FILENAME) - dst_install_manifest_file_path = os.path.join(dst_install_manifest_path, INSTALL_MANIFEST_FILENAME) + src_install_manifest_file_path = os.path.join(src_install_manifest_path, CMAKE_INSTALL_MANIFEST_FILENAME) + dst_install_manifest_file_path = os.path.join(dst_install_manifest_path, CMAKE_INSTALL_MANIFEST_FILENAME) # Create the directory for the manifest if it doesn't exist mkdir_p(dst_install_manifest_path) @@ -288,7 +287,7 @@ def create_cmake_build_job(context, package, package_path, dependencies, force_c 'register', copy_install_manifest, src_install_manifest_path=build_space, - dst_install_manifest_path=context.package_linked_devel_path(package) + dst_install_manifest_path=context.package_metadata_path(package) )) # Determine the location where the setup.sh file should be created @@ -313,84 +312,59 @@ def create_cmake_build_job(context, package, package_path, dependencies, force_c stages=stages) -def create_cmake_clean_job(context, package_name, dependencies): - """Factory for a Job to clean cmake packages""" - - # Determine install target - install_target = context.install_space_abs if context.install else context.devel_space_abs - - # Setup build variables - build_space = get_package_build_space_path(context.build_space_abs, package_name) +def create_cmake_clean_job( + context, + package, + package_path, + dependencies, + dry_run, + clean_build, + clean_devel, + clean_install): + """Generate a Job to clean a cmake package""" - # Read install manifest - install_manifest_path = os.path.join( - context.package_linked_devel_path(package), - INSTALL_MANIFEST_FILENAME) - installed_files = set() - if os.path.exists(install_manifest_path): - with open(install_manifest_path) as f: - installed_files = set([line.strip() for line in f.readlines()]) - - # List of directories to check for removed files - dirs_to_check = set() - - # Stages for this clean job stages = [] - for installed_file in installed_files: - # Make sure the file is given by an absolute path and it exists - if not os.path.isabs(installed_file) or not os.path.exists(installed_file): - continue - - # Add stages to remove the file or directory - if os.path.isdir(installed_file): - stages.append(FunctionStage('rmdir', rmdirs, path=installed_file)) - else: - stages.append(FunctionStage('rm', rmfile, path=installed_file)) - - # Check if directories that contain this file will be empty once it's removed - path = installed_file - # Only look in the devel space - while path != self.context.devel_space_abs: - # Pop up a directory - path, dirname = os.path.split(path) - - # Skip if this path isn't a directory - if not os.path.isdir(path): - continue - - dirs_to_check.add(path) - - # For each directory which may be empty after cleaning, visit them depth-first and count their descendants - dir_descendants = dict() - dirs_to_remove = set() - for path in sorted(dirs_to_check, key=lambda k: -len(k.split(os.path.sep))): - # Get the absolute path to all the files currently in this directory - files = [os.path.join(path, f) for f in os.listdir(path)] - # Filter out the files which we intend to remove - files = [f for f in files if f not in installed_files] - # Compute the minimum number of files potentially contained in this path - dir_descendants[path] = sum([(dir_descendants.get(f, 1) if os.path.isdir(f) else 1) for f in files]) - - # Schedule the directory for removal if removal of the given files will make it empty - if dir_descendants[path] == 0: - dirs_to_remove.add(path) - - for generated_dir in dirs_to_remove: - stages.append(FunctionStage('rmdir', rmdirs, path=generated_dir)) - - stages.append(FunctionStage('rmbuild', rmdirs, path=build_space)) + if clean_install and context.install: + installed_files = get_installed_files(context.package_metadata_path(package)) + stages.append(FunctionStage( + 'cleaninstall', + rmfiles, + paths=sorted(installed_files), + remove_empty=True, + empty_root=context.install_space_abs, + dry_run=dry_run)) + + if clean_devel and not context.install: + installed_files = get_installed_files(context.package_metadata_path(package)) + stages.append(FunctionStage( + 'cleandevel', + rmfiles, + paths=sorted(installed_files), + remove_empty=True, + empty_root=context.devel_space_abs, + dry_run=dry_run)) + + if clean_build: + build_space = context.package_build_space(package) + stages.append(FunctionStage( + 'rmbuild', + rmfiles, + paths=[build_space], + dry_run=dry_run)) return Job( - jid=package_name, + jid=package.name, deps=dependencies, + env_loader=get_env_loader(package, context), stages=stages) description = dict( build_type='cmake', description="Builds a plain CMake package.", - create_build_job=create_cmake_build_job + create_build_job=create_cmake_build_job, + create_clean_job=create_cmake_clean_job ) diff --git a/catkin_tools/jobs/commands/cmake.py b/catkin_tools/jobs/commands/cmake.py index 85cd5070..baf2dd69 100644 --- a/catkin_tools/jobs/commands/cmake.py +++ b/catkin_tools/jobs/commands/cmake.py @@ -26,6 +26,7 @@ from catkin_tools.utils import which CMAKE_EXEC = which('cmake') +CMAKE_INSTALL_MANIFEST_FILENAME = 'install_manifest.txt' def split_to_last_line_break(data): @@ -171,3 +172,17 @@ def on_stdout_received(self, data): job_id=self.job_id, stage_label=self.stage_label, percent=str(progress_matches.groups()[0]))) + + +def get_installed_files(path): + """Get a set of files installed by a CMake package as specified by an + install_manifest.txt in a given directory.""" + + install_manifest_path = os.path.join( + path, + CMAKE_INSTALL_MANIFEST_FILENAME) + installed_files = set() + if os.path.exists(install_manifest_path): + with open(install_manifest_path) as f: + installed_files = set([line.strip() for line in f.readlines()]) + return installed_files diff --git a/catkin_tools/jobs/utils.py b/catkin_tools/jobs/utils.py index 9f8c2459..cb5cf727 100644 --- a/catkin_tools/jobs/utils.py +++ b/catkin_tools/jobs/utils.py @@ -15,6 +15,7 @@ from __future__ import print_function import os +import shutil import stat from catkin_tools.common import mkdir_p @@ -24,6 +25,7 @@ from catkin_tools.execution.jobs import Job from catkin_tools.execution.stages import CommandStage +from catkin_tools.execution.events import ExecutionEvent from .commands.cmake import CMAKE_EXEC @@ -94,25 +96,79 @@ def rmfile(logger, event_queue, path): return 0 -def rmdirs(logger, event_queue, path): +def rmdirs(logger, event_queue, paths): """FunctionStage functor that removes a directory tree.""" - if os.path.exists(path): - shutil.rmtree(path) - return 0 - - -def create_clean_buildspace_job(context, package, dependencies): - """Create a job to remove a buildspace only.""" - build_space = context.package_build_space(package) - if not os.path.exists(build_space): - # No-op - return Job(jid=package.name, deps=dependencies, stages=[]) - - stages = [] + return rmfiles(logger, event_queue, paths, remove_empty=False) + + +def rmfiles(logger, event_queue, paths, dry_run, remove_empty=False, empty_root='/'): + """FunctionStage functor that removes a list of files and directories. + + If remove_empty is True, then this will also remove directories which + become emprt after deleting the files in `paths`. It will delete files up + to the path specified by `empty_root`. + """ + + # Determine empty directories + if remove_empty: + # First get a list of directories to check + dirs_to_check = set() + + for path in paths: + # Make sure the file is given by an absolute path and it exists + if not os.path.isabs(path) or not os.path.exists(path): + continue + + # Only look in the devel space + while empty_root.find(path) != 0: + # Pop up a directory + path, dirname = os.path.split(path) + + # Skip if this path isn't a directory + if not os.path.isdir(path): + continue + + dirs_to_check.add(path) + + # For each directory which may be empty after cleaning, visit them + # depth-first and count their descendants + dir_descendants = dict() + for path in sorted(dirs_to_check, key=lambda k: -len(k.split(os.path.sep))): + # Get the absolute path to all the files currently in this directory + files = [os.path.join(path, f) for f in os.listdir(path)] + # Filter out the files which we intend to remove + files = [f for f in files if f not in paths] + # Compute the minimum number of files potentially contained in this path + dir_descendants[path] = sum([ + (dir_descendants.get(f, 1) if os.path.isdir(f) else 1) + for f in files + ]) + + # Schedule the directory for removal if removal of the given files will make it empty + if dir_descendants[path] == 0: + paths.append(path) + + # REmove the paths + for index, path in enumerate(paths): + + # Remove the path + if os.path.exists(path): + if os.path.isdir(path): + logger.out('Removing directory: {}'.format(path)) + if not dry_run: + shutil.rmtree(path) + else: + logger.out(' Removing file: {}'.format(path)) + if not dry_run: + os.remove(path) + else: + logger.err('Warning: File {} could not be deleted because it does not exist.'.format(path)) - stages.append(FunctionStage('rmbuild', rmdirs, path=build_space)) + # Report progress + event_queue.put(ExecutionEvent( + 'STAGE_PROGRESS', + job_id=logger.job_id, + stage_label=logger.stage_label, + percent=str(index / float(len(paths))))) - return Job( - jid=package.name, - deps=dependencies, - stages=stages) + return 0 diff --git a/catkin_tools/verbs/catkin_build/build.py b/catkin_tools/verbs/catkin_build/build.py index 7bac4074..5af7f945 100644 --- a/catkin_tools/verbs/catkin_build/build.py +++ b/catkin_tools/verbs/catkin_build/build.py @@ -378,12 +378,13 @@ def build_isolated_workspace( # Generate prebuild jobs, if necessary prebuild_jobs = {} setup_util_exists = os.path.exists(os.path.join(context.devel_space_abs, '_setup_util.py')) - if context.link_devel and (not setup_util_exists or (force_cmake and len(packages) == 0)): + build_catkin_package = 'catkin' in packages_to_be_built_names + packages_to_be_built_deps_names + if context.link_devel and (not setup_util_exists or (force_cmake and len(packages) == 0) or build_catkin_package): wide_log('[build] Preparing linked develspace...') pkg_dict = dict([(pkg.name, (pth, pkg)) for pth, pkg in all_packages]) - if 'catkin' in packages_to_be_built_names + packages_to_be_built_deps_names: + if build_catkin_package: # Use catkin as the prebuild package prebuild_pkg_path, prebuild_pkg = pkg_dict['catkin'] else: @@ -504,7 +505,7 @@ def build_isolated_workspace( jobs, locks, event_queue, - os.path.join(context.build_space_abs, '_logs'), + os.path.join(context.metadata_path(), 'logs'), max_toplevel_jobs=n_jobs, continue_on_failure=continue_on_failure, continue_without_deps=False)) diff --git a/catkin_tools/verbs/catkin_clean/clean.py b/catkin_tools/verbs/catkin_clean/clean.py new file mode 100644 index 00000000..7bf53746 --- /dev/null +++ b/catkin_tools/verbs/catkin_clean/clean.py @@ -0,0 +1,203 @@ +# Copyright 2014 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This modules implements the engine for cleaning packages in parallel""" + +import os +import pkg_resources +import sys +import time +import traceback + +try: + # Python3 + from queue import Queue +except ImportError: + # Python2 + from Queue import Queue + +try: + from catkin_pkg.packages import find_packages + from catkin_pkg.topological_order import topological_order_packages +except ImportError as e: + sys.exit( + 'ImportError: "from catkin_pkg.topological_order import ' + 'topological_order" failed: %s\nMake sure that you have installed ' + '"catkin_pkg", and that it is up to date and on the PYTHONPATH.' % e + ) + +from catkin_tools.execution.controllers import ConsoleStatusController +from catkin_tools.execution.executor import execute_jobs +from catkin_tools.execution.executor import run_until_complete + +from catkin_tools.common import get_build_type +from catkin_tools.common import get_recursive_build_dependants_in_workspace +from catkin_tools.common import log +from catkin_tools.common import wide_log + + +def determine_packages_to_be_cleaned(context, include_dependants, packages): + """Returns list of packages which should be cleaned, and those packages' deps. + + :param context: Workspace context + :type context: :py:class:`catkin_tools.verbs.catkin_build.context.Context` + :param packages: list of package names to be cleaned + :type packages: list + :returns: full list of package names to be cleaned + :rtype: list + """ + start = time.time() + + # Get all the packages in the context source space + workspace_packages = find_packages(context.source_space_abs, exclude_subspaces=True, warnings=[]) + # Order the packages by topology + ordered_packages = topological_order_packages(workspace_packages) + + # Create a dict of all packages in the workspace by name + workspace_packages_by_name = dict([(pkg.name, (path, pkg)) for path, pkg in ordered_packages]) + + # Initialize empty output + packages_to_be_cleaned = set() + + # Expand metapackages into their constituents + for package_name in packages: + # This is ok if it's orphaned + if package_name not in workspace_packages_by_name: + packages_to_be_cleaned.add(package_name) + else: + # Get the package object + package = workspace_packages_by_name[package_name][1] + # If metapackage, include run depends which are in the workspace + if 'metapackage' in [e.tagname for e in package.exports]: + for rdep in package.run_depends: + if rdep.name in workspace_packages_by_name: + packages_to_be_cleaned.add(rdep.name) + else: + packages_to_be_cleaned.add(package_name) + + # Determine the packages that depend on the given packages + if include_dependants: + for package_name in list(packages_to_be_cleaned): + # Get the packages that depend on the packages to be cleaned + dependants = get_recursive_build_dependants_in_workspace(package_name, ordered_packages) + packages_to_be_cleaned.update([pkg.name for _, pkg in dependants]) + + return [workspace_packages_by_name[n] for n in packages_to_be_cleaned] + + +def clean_packages( + context, + names_of_packages_to_be_cleaned, + clean_dependants, + verbose, + dry_run): + + pre_start_time = time.time() + + # Update the names of packages to be cleaned with dependants + packages_to_be_cleaned = determine_packages_to_be_cleaned( + context, + clean_dependants, + names_of_packages_to_be_cleaned) + + # print(packages_to_be_cleaned) + # for path, pkg in packages_to_be_cleaned: + # if os.path.exists(os.path.join(context.build_space_abs, pkg.name)): + # print("[clean] Cleaning package: %s" % pkg.name) + + # Construct jobs + jobs = [] + for path, pkg in packages_to_be_cleaned: + + # Get all build type plugins + clean_job_creators = { + ep.name: ep.load()['create_clean_job'] + for ep in pkg_resources.iter_entry_points(group='catkin_tools.jobs') + } + + # It's a problem if there aren't any build types available + if len(clean_job_creators) == 0: + sys.exit('Error: No build types availalbe. Please check your catkin_tools installation.') + + # Determine the job parameters + clean_job_kwargs = dict( + context=context, + package=pkg, + package_path=path, + dependencies=[], # Unused because clean jobs are not parallelized + dry_run=dry_run, + clean_build=True, + clean_devel=True, + clean_install=True) + + # Create the job based on the build type + build_type = get_build_type(pkg) + + if build_type in clean_job_creators: + jobs.append(clean_job_creators[build_type](**clean_job_kwargs)) + + if len(jobs) == 0: + print("[clean] There are no products from the given packages to clean.") + return False + + # Queue for communicating status + event_queue = Queue() + + # Spin up status output thread + status_thread = ConsoleStatusController( + 'clean', + ['package', 'packages'], + jobs, + 1, + [pkg.name for _, pkg in context.packages], + [p for p in context.whitelist], + [p for p in context.blacklist], + event_queue, + show_notifications=False, + show_active_status=False, + show_buffered_stdout=True, + show_buffered_stderr=True, + show_live_stdout=False, + show_live_stderr=False, + show_stage_events=False, + show_full_summary=False, + pre_start_time=pre_start_time, + active_status_rate=10.0) + status_thread.start() + + # Initialize locks (none need to be configured here) + locks = { + } + + # Block while running N jobs asynchronously + try: + ej = execute_jobs( + 'clean', + jobs, + locks, + event_queue, + os.path.join(context.metadata_path(), 'logs'), + max_toplevel_jobs=1, + continue_on_failure=True, + continue_without_deps=False) + all_succeeded = run_until_complete(ej) + except Exception: + status_thread.keep_running = False + all_succeeded = False + status_thread.join(1.0) + wide_log(str(traceback.format_exc())) + + status_thread.join(1.0) + + return True diff --git a/catkin_tools/verbs/catkin_clean/cli.py b/catkin_tools/verbs/catkin_clean/cli.py index 04bc9f94..5ce9ed3e 100644 --- a/catkin_tools/verbs/catkin_clean/cli.py +++ b/catkin_tools/verbs/catkin_clean/cli.py @@ -14,6 +14,11 @@ from __future__ import print_function +try: + raw_input +except NameError: + raw_input = input + import os import shutil @@ -23,169 +28,291 @@ from catkin_tools.context import Context +from catkin_tools.common import log + +import catkin_tools.execution.job_server as job_server + from catkin_tools.metadata import update_metadata +from catkin_tools.metadata import METADATA_DIR_NAME from catkin_tools.terminal_color import ColorMapper +from .clean import clean_packages + color_mapper = ColorMapper() clr = color_mapper.clr # Exempt build directories # See https://github.com/catkin/catkin_tools/issues/82 -exempt_build_files = ['build_logs', '.built_by', '.catkin_tools.yaml'] +exempt_build_files = ['.catkin_tools.yaml', 'catkin_tools_prebuild'] setup_files = ['.catkin', 'env.sh', 'setup.bash', 'setup.sh', 'setup.zsh', '_setup_util.py'] +def yes_no_loop(question): + while True: + resp = str(raw_input(question + " [yN]: ")) + if resp.lower() in ['n', 'no'] or len(resp) == 0: + return False + elif resp.lower() in ['y', 'yes']: + return True + log(clr("[clean] Please answer either \"yes\" or \"no\".")) + + def prepare_arguments(parser): # Workspace / profile args add_context_args(parser) + add = parser.add_argument + add('--dry-run', '-n', action='store_true', default=False, + help='Show the effects of the clean action without modifying the workspace.') + add('--verbose', '-v', action='store_true', default=False, + help='Verbose status output.') + add('--force', '-f', action='store_true', default=False, + help='Skip all interactive checks.') + # Basic group basic_group = parser.add_argument_group('Basic', 'Clean workspace subdirectories.') add = basic_group.add_argument add('-a', '--all', action='store_true', default=False, - help='Remove all of the *spaces associated with the given or active' - ' profile. This will remove everything but the source space and the' - ' hidden .catkin_tools directory.') + help='Remove all of the generated spaces associated with the given or ' + 'active profile. This will remove everything but the source space and ' + 'the hidden .catkin_tools directory.') add('-b', '--build', action='store_true', default=False, - help='Remove the buildspace.') + help='Remove the entire buildspace.') add('-d', '--devel', action='store_true', default=False, - help='Remove the develspace.') + help='Remove the entire develspace.') add('-i', '--install', action='store_true', default=False, - help='Remove the installspace.') + help='Remove the entire installspace.') + add('--deinit', action='store_true', default=False, + help='De-initialize the workspace, delete all build profiles and configuration.') + + # Packages group + packages_group = parser.add_argument_group( + 'Packages', + "Clean products from specific packages in the workspace. These options will " + "automatically enable the --force-cmake option for the next build " + "invocation.") + add = packages_group.add_argument + add('packages', metavar='PKGNAME', nargs='*', + help='Explicilty specify a list of specific packages to clean from the build, devel, and install space.') + add('--deps', action='store_true', default=False, + help='Clean the packages which depend on other packages to be cleaned.') + add('--orphans', action='store_true', default=False, + help='Remove products from packages are no longer in the source space. ' + 'Note that this also removes packages which are ' + 'blacklisted or which contain `CATKIN_INGORE` marker files.') # Advanced group advanced_group = parser.add_argument_group( 'Advanced', - "Clean only specific parts of the workspace. These options will " + "Clean only specific parts of the workspace for specified packages. These options will " "automatically enable the --force-cmake option for the next build " "invocation.") add = advanced_group.add_argument - add('-c', '--cmake-cache', action='store_true', default=False, - help='Clear the CMakeCache for each package, but leave build and devel spaces.') - - add('-s', '--setup-files', action='store_true', default=False, - help='Clear the catkin-generated files in order to rebase onto another workspace.') - - add('-o', '--orphans', action='store_true', default=False, - help='Remove only build directories whose source packages are no' - ' longer enabled or in the source space. This might require' - ' --force-cmake on the next build.') + add('-l', '--logs', action='store_true', default=False, + help='Only clear the catkin-generated logfiles.') + add('--setup-files', action='store_true', default=False, + help='Clear the catkin-generated setup files from the devel and install spaces.') return parser def main(opts): - actions = ['all', 'build', 'devel', 'install', 'cmake_cache', 'orphans', 'setup_files'] - if not any([v for (k, v) in vars(opts).items() if k in actions]): - print("[clean] No actions performed. See `catkin clean -h` for usage.") - return 0 - - needs_force = False - # Load the context ctx = Context.load(opts.workspace, opts.profile, opts, strict=True, load_env=False) if not ctx: if not opts.workspace: print( - "catkin clean: error: The current or desired workspace could not be " + "[clean] Error: The current or desired workspace could not be " "determined. Please run `catkin clean` from within a catkin " "workspace or specify the workspace explicitly with the " "`--workspace` option.") else: print( - "catkin clean: error: Could not clean workspace \"%s\" because it " + "[clean] Error: Could not clean workspace \"%s\" because it " "either does not exist or it has no catkin_tools metadata." % opts.workspace) return 1 + # Initialize job server + job_server.initialize( + max_jobs=1, + max_load=None, + gnu_make_enabled=False) + + # Check if the user wants to do something explicit + actions = ['all', 'build', 'devel', 'install', 'deinit', 'orphans', + 'setup_files', 'packages', 'logs'] + + build_exists = os.path.exists(ctx.build_space_abs) + devel_exists = os.path.exists(ctx.devel_space_abs) + install_exists = os.path.exists(ctx.install_space_abs) + + if not any([build_exists, devel_exists, install_exists]) and not opts.deinit: + log("[clean] Neither the build, devel, or install spaces for this " + "profile exist. Nothing to be done.") + return 0 + + # Default is to clean all products for this profile + if not any([v for (k, v) in vars(opts).items() if k in actions]): + opts.all = True + + # Make sure the user intends to clena everything + if opts.all and not (opts.force or opts.dry_run): + log(clr("[clean] @!@{yf}Warning:@| This will completely remove the " + "existing build, devel, and install spaces for this profile. " + "Use `--force` to skip this check.")) + if build_exists: + log(clr("[clean] Build Space: @{yf}{}").format(ctx.build_space_abs)) + if devel_exists: + log(clr("[clean] Devel Space: @{yf}{}").format(ctx.devel_space_abs)) + if install_exists: + log(clr("[clean] Install Space: @{yf}{}").format(ctx.install_space_abs)) + + try: + opts.all = yes_no_loop("\n[clean] Are you sure you want to completely remove the directories listed above?") + if not opts.all: + log(clr("[clean] Not removing build, devel, or install spaces for this profile.")) + except KeyboardInterrupt: + log("\n[clean] No actions performed.") + return 0 + + # Warn before nuking .catkin_tools + if opts.deinit and not opts.force: + log(clr("[clean] @!@{yf}Warning:@| If you deinitialize this workspace " + "you will lose all profiles and all saved build configuration. " + "Use `--force` to skip this check.")) + try: + opts.deinit = yes_no_loop("\n[clean] Are you sure you want to deinitialize this workspace?") + if not opts.deinit: + log(clr("[clean] Not deinitializing workspace.")) + except KeyboardInterrupt: + log("\n[clean] No actions performed.") + return 0 + + # Initialize flag to be used on the next invocation + needs_force = False + # Remove the requested spaces if opts.all: opts.build = opts.devel = opts.install = True - if opts.build: - if os.path.exists(ctx.build_space_abs): - print("[clean] Removing buildspace: %s" % ctx.build_space_abs) - shutil.rmtree(ctx.build_space_abs) - else: - # Orphan removal - if opts.orphans: + try: + # Remove all installspace files + if opts.install: + if os.path.exists(ctx.install_space_abs): + print("[clean] Removing installspace: %s" % ctx.install_space_abs) + if not opts.dry_run: + shutil.rmtree(ctx.install_space_abs) + + # Remove all develspace files + if opts.devel: + if os.path.exists(ctx.devel_space_abs): + print("[clean] Removing develspace: %s" % ctx.devel_space_abs) + if not opts.dry_run: + shutil.rmtree(ctx.devel_space_abs) + + # Remove all buildspace files + if opts.build: if os.path.exists(ctx.build_space_abs): - # TODO: Check for merged build and report error - - # Get all enabled packages in source space - # Suppress warnings since this is looking for packages which no longer exist - found_source_packages = [ - pkg.name for (path, pkg) in find_packages(ctx.source_space_abs, warnings=[]).items()] - - # Iterate over all packages with build dirs - print("[clean] Removing orphaned build directories from %s" % ctx.build_space_abs) - no_orphans = True - for pkg_build_name in os.listdir(ctx.build_space_abs): - if pkg_build_name not in exempt_build_files: - pkg_build_path = os.path.join(ctx.build_space_abs, pkg_build_name) - # Remove package build dir if not found - if pkg_build_name not in found_source_packages: - no_orphans = False - print(" - Removing %s" % pkg_build_path) - shutil.rmtree(pkg_build_path) - - if no_orphans: - print("[clean] No orphans found, nothing removed from buildspace.") + print("[clean] Removing buildspace: %s" % ctx.build_space_abs) + if not opts.dry_run: + shutil.rmtree(ctx.build_space_abs) + + # Find orphaned packages + if ctx.link_devel and not any([opts.build, opts.devel]): + if opts.orphans: + if os.path.exists(ctx.build_space_abs): + # Initialize orphan list + orphans = set() + + # Get all enabled packages in source space + # Suppress warnings since this is looking for packages which no longer exist + found_source_packages = [ + pkg.name for (path, pkg) in find_packages(ctx.source_space_abs, warnings=[]).items()] + + # Look for orphaned products in the build space + print("[clean] Determining orphaned packages...") + for pkg_build_name in os.listdir(ctx.build_space_abs): + if pkg_build_name not in exempt_build_files: + if pkg_build_name not in found_source_packages: + orphans.add(pkg_build_name) + print("[clean] Orphaned build products: %s" % pkg_build_name) + + # Look for orphaned products in the develspace + for pkg_devel_name in ctx.private_devel_path(): + if os.path.isdir(pkg_devel_name): + if pkg_devel_name not in found_source_packages: + orphans.add(pkg_devel_name) + print("[clean] Orphaned devel products: %s" % pkg_devel_name) + + if len(orphans) > 0: + opts.packages.extend(list(orphans)) + else: + print("[clean] No orphans in the workspace.") else: - # Remove the develspace - # TODO: For isolated devel, this could just remove individual packages - if os.path.exists(ctx.devel_space_abs): - print("Removing develspace: %s" % ctx.devel_space_abs) - shutil.rmtree(ctx.devel_space_abs) - needs_force = True + print("[clean] No buildspace exists, no potential for orphans.") + + # Remove specific packages + if len(opts.packages) > 0: + + try: + # Clean the packages + needs_force = clean_packages( + ctx, + opts.packages, + opts.deps, + opts.verbose, + opts.dry_run) + except KeyboardInterrupt: + wide_log("[build] User interrupted!") + event_queue.put(None) + + elif opts.orphans or len(opts.packages) > 0: + print("[clean] Error: Individual packages can only be cleaned from " + "workspaces with symbolically-linked develspaces (`catkin " + "config --link-devel`).") + + # Clean log files + if opts.logs: + log_dir = os.path.join(ctx.metadata_path(), 'logs') + if os.path.exists(log_dir): + print("[clean] Removing log files from: {}".format(log_dir)) + if not opts.dry_run: + shutil.rmtree(log_dir) else: - print("[clean] No buildspace exists, no potential for orphans.") - return 0 + print("[clean] Log file directory does not exist: {}".format(log_dir)) - # CMake Cache removal - if opts.cmake_cache: - # Clear the CMakeCache for each package - if os.path.exists(ctx.build_space_abs): - # Remove CMakeCaches - print("[clean] Removing CMakeCache.txt files from %s" % ctx.build_space_abs) - for pkg_build_name in os.listdir(ctx.build_space_abs): - if pkg_build_name not in exempt_build_files: - pkg_build_path = os.path.join(ctx.build_space_abs, pkg_build_name) - ccache_path = os.path.join(pkg_build_path, 'CMakeCache.txt') - - if os.path.exists(ccache_path): - print(" - Removing %s" % ccache_path) - os.remove(ccache_path) - needs_force = True - else: - print("[clean] No buildspace exists, no CMake caches to clear.") + # Nuke .catkin_tools + if opts.deinit: + metadata_dir = os.path.join(ctx.workspace, METADATA_DIR_NAME) + print("[clean] Deinitializing workspace by removing catkin_tools config: %s" % metadata_dir) + if not opts.dry_run: + shutil.rmtree(metadata_dir) - if opts.devel: - if os.path.exists(ctx.devel_space_abs): - print("[clean] Removing develspace: %s" % ctx.devel_space_abs) - shutil.rmtree(ctx.devel_space_abs) - else: + # Setup file removal if opts.setup_files: - print("[clean] Removing setup files from develspace: %s" % ctx.devel_space_abs) - for filename in setup_files: - full_path = os.path.join(ctx.devel_space_abs, filename) - if os.path.exists(full_path): - print(" - Removing %s" % full_path) - os.remove(full_path) - needs_force = True - - if opts.install: - if os.path.exists(ctx.install_space_abs): - print("[clean] Removing installspace: %s" % ctx.install_space_abs) - shutil.rmtree(ctx.install_space_abs) - - if needs_force: - print( - "NOTE: Parts of the workspace have been cleaned which will " - "necessitate re-configuring CMake on the next build.") - update_metadata(ctx.workspace, ctx.profile, 'build', {'needs_force': True}) + if os.path.exists(ctx.devel_space_abs): + print("[clean] Removing setup files from develspace: %s" % ctx.devel_space_abs) + for filename in setup_files: + full_path = os.path.join(ctx.devel_space_abs, filename) + if os.path.exists(full_path): + print(" - Removing %s" % full_path) + os.remove(full_path) + needs_force = True + else: + print("[clean] No develspace exists, no setup files to clean.") + except: + needs_force = True + raise + + finally: + if needs_force: + print(clr( + "[clean] @/@!Note:@| @/Parts of the workspace have been cleaned which will " + "necessitate re-configuring CMake on the next build.@|")) + update_metadata(ctx.workspace, ctx.profile, 'build', {'needs_force': True}) return 0 diff --git a/catkin_tools/verbs/catkin_clean/color.py b/catkin_tools/verbs/catkin_clean/color.py new file mode 100644 index 00000000..3ab39dbc --- /dev/null +++ b/catkin_tools/verbs/catkin_clean/color.py @@ -0,0 +1,69 @@ +# Copyright 2014 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This module implements many of the colorization functions used by catkin clean""" + +from catkin_tools.terminal_color import ansi +from catkin_tools.terminal_color import fmt +from catkin_tools.terminal_color import sanitize +from catkin_tools.terminal_color import ColorMapper + +# This map translates more human reable format strings into colorized versions +_color_translation_map = { + # 'output': 'colorized_output' + '': fmt('@!' + sanitize('') + '@|'), + + "[{package}] ==> '{cmd.cmd_str}' in '{location}'": + fmt("[@{cf}{package}@|] @!@{bf}==>@| '@!{cmd.cmd_str}@|' @{kf}@!in@| '@!{location}@|'"), + + "Starting ==> {package}": + fmt("Starting @!@{gf}==>@| @!@{cf}{package}@|"), + + "[{package}] {msg}": + fmt("[@{cf}{package}@|] {msg}"), + + "[{package}] <== '{cmd.cmd_str}' failed with return code '{retcode}'": + fmt("[@{cf}{package}@|] @!@{rf}<==@| '@!{cmd.cmd_str}@|' @{rf}failed with return code@| '@!{retcode}@|'"), + + "[{package}] <== '{cmd.cmd_str}' finished with return code '{retcode}'": + fmt("[@{cf}{package}@|] @{gf}<==@| '@!{cmd.cmd_str}@|' finished with return code '@!{retcode}@|'"), + + "Finished <== {package:<": + fmt("@!@{kf}Finished@| @{gf}<==@| @{cf}{package:<").rstrip(ansi('reset')), + + "Failed <== {package:<": + fmt("@!@{rf}Failed@| @{gf}<==@| @{cf}{package:<").rstrip(ansi('reset')), + + "} [ {time} ]": + fmt("}@| [ @{yf}{time}@| ]"), + + "[build - {run_time}] ": + fmt("[@{pf}build@| - @{yf}{run_time}@|] "), + + "[{name} - {run_time}] ": + fmt("[@{cf}{name}@| - @{yf}{run_time}@|] "), + + "[{0}/{1} Active | {2}/{3} Completed]": + fmt("[@!@{gf}{0}@|/@{gf}{1}@| Active | @!@{gf}{2}@|/@{gf}{3}@| Completed]"), + + "[{0}/{1} Jobs | {2}/{3} Active | {4}/{5} Completed]": + fmt("[@!@{gf}{0}@|/@{gf}{1}@| Jobs | @!@{gf}{2}@|/@{gf}{3}@| Active | @!@{gf}{4}@|/@{gf}{5}@| Completed]"), + + "[!{package}] ": + fmt("[@!@{rf}!@|@{cf}{package}@|] "), +} + +color_mapper = ColorMapper(_color_translation_map) + +clr = color_mapper.clr diff --git a/tests/system/verbs/catkin_build/test_build.py b/tests/system/verbs/catkin_build/test_build.py index ab8d54a2..a3da9bc1 100644 --- a/tests/system/verbs/catkin_build/test_build.py +++ b/tests/system/verbs/catkin_build/test_build.py @@ -21,7 +21,7 @@ RESOURCES_DIR = os.path.join(os.path.dirname(__file__), '..', '..', 'resources') BUILD = ['build', '--no-notify', '--no-status'] -CLEAN = ['clean', '--all'] # , '--force'] # , '--no-notify', '--no-color', '--no-status'] +CLEAN = ['clean', '--all', '--force'] BUILD_TYPES = ['cmake', 'catkin']