Add support for custom_rosdep_urls in build files (#1055)

* Implement custom_rosdep_urls for ci jobs

Add support to ci_build_files in order to support custom rosdep rules in the same way that they exists for devel and source jobs.
---------

Signed-off-by: Jose Luis Rivero <jrivero@osrfoundation.org>

doc/configuration_options.rst

@@ -597,3 +597,15 @@ The following options are valid in version ``1`` (beside the generic options):
   to, if desired.
   By default, the resulting archives are only available to other jobs within
   Jenkins.
+
+The following options are valid as keys in the ``_config`` dict under
+``targets``:
+
+* ``custom_rosdep_urls``: a list of URLs containing rosdep sources.list.d entry
+  files that are downloaded into /etc/ros/rosdep/sources.list.d at the beginning
+  of the doc job after running *rosdep init*.
+  Note that *rosdep init* will add the 20-default.list file from the public
+  rosdistro by default.
+  To override this, add an entry to this list corresponding to the
+  20-default.list file from your forked rosdistro repository.
+

ros_buildfarm/ci_job.py

@@ -287,6 +287,7 @@ def _get_ci_job_config(
         'repository_names': repository_names,
         'package_names': package_names,
         'package_dependencies': package_dependencies,
+        'custom_rosdep_urls': build_file.custom_rosdep_urls,
 
         'skip_rosdep_keys': build_file.skip_rosdep_keys,
         'install_packages': build_file.install_packages,

ros_buildfarm/config/ci_build_file.py

@@ -152,3 +152,10 @@ def __init__(self, name, data):  # noqa: D107
         self.upload_directory = None
         if 'upload_directory' in data:
             self.upload_directory = data['upload_directory']
+
+        self.custom_rosdep_urls = []
+        if '_config' in data['targets']:
+            if 'custom_rosdep_urls' in data['targets']['_config']:
+                self.custom_rosdep_urls = \
+                    data['targets']['_config']['custom_rosdep_urls']
+                assert isinstance(self.custom_rosdep_urls, list)

ros_buildfarm/scripts/ci/create_workspace_task_generator.py

@@ -19,6 +19,7 @@
 
 from apt import Cache
 from ros_buildfarm.argument import add_argument_arch
+from ros_buildfarm.argument import add_argument_custom_rosdep_urls
 from ros_buildfarm.argument import \
     add_argument_distribution_repository_key_files
 from ros_buildfarm.argument import add_argument_distribution_repository_urls
@@ -50,6 +51,7 @@ def main(argv=sys.argv[1:]):
     add_argument_os_code_name(parser)
     add_argument_arch(parser)
 
+    add_argument_custom_rosdep_urls(parser)
     add_argument_distribution_repository_key_files(parser)
     add_argument_distribution_repository_urls(parser)
     add_argument_dockerfile_dir(parser)
@@ -105,7 +107,7 @@ def main(argv=sys.argv[1:]):
 
         'rosdistro_name': args.rosdistro_name,
 
-        'custom_rosdep_urls': [],
+        'custom_rosdep_urls': args.custom_rosdep_urls,
 
         'uid': get_user_id(),
 

ros_buildfarm/scripts/ci/run_ci_job.py

@@ -22,6 +22,7 @@
 from ros_buildfarm.argument import add_argument_build_tool
 from ros_buildfarm.argument import add_argument_build_tool_args
 from ros_buildfarm.argument import add_argument_build_tool_test_args
+from ros_buildfarm.argument import add_argument_custom_rosdep_urls
 from ros_buildfarm.argument import \
     add_argument_distribution_repository_key_files
 from ros_buildfarm.argument import add_argument_distribution_repository_urls
@@ -56,6 +57,7 @@ def main(argv=sys.argv[1:]):
     add_argument_arch(parser)
 
     add_argument_build_tool(parser, required=True)
+    add_argument_custom_rosdep_urls(parser)
     add_argument_distribution_repository_key_files(parser)
     add_argument_distribution_repository_urls(parser)
     add_argument_dockerfile_dir(parser)

ros_buildfarm/templates/ci/ci_create_tasks.Dockerfile.em

@@ -80,6 +80,7 @@ cmds = [
     ' --repository-names ' + ' '.join(repository_names) + \
     ((' --package-names ' + ' '.join(package_names)) if package_names else '') + \
     (' --package-dependencies' if package_dependencies else '') + \
+    ' --custom-rosdep-urls '  + ' '.join(custom_rosdep_urls) + \
     ' --test-branch "%s"' % (test_branch) + \
     ' --skip-rosdep-keys ' + ' '.join(skip_rosdep_keys) + \
     ' --package-selection-args ' + ' '.join(package_selection_args),

ros_buildfarm/templates/ci/create_workspace.Dockerfile.em

@@ -72,7 +72,7 @@ RUN echo "@today_str"
 ))@
 
 # needed for 'vcs custom --git --args merge' invocation
-RUN python3 -u /tmp/wrapper_scripts/apt.py update-install-clean -q --no-install-recommends sudo
+RUN python3 -u /tmp/wrapper_scripts/apt.py update-install-clean -q -y --no-install-recommends sudo wget
 RUN sudo -H -u buildfarm -- git config --global user.email "jenkins@@ros.invalid" && sudo -H -u buildfarm -- git config --global user.name "Jenkins ROS"
 
 @(TEMPLATE(