Add cvat cli to datumaro project export (#870)

cvat/apps/dataset_manager/export_templates/README.md

@@ -0,0 +1,22 @@
+# Quick start
+
+``` bash
+# optionally make a virtualenv
+python -m virtualenv .venv
+. .venv/bin/activate
+
+# install dependencies
+sed -r "s/^(.*)#.*$/\1/g" datumaro/requirements.txt | xargs -n 1 -L 1 pip install
+pip install -r cvat/utils/cli/requirements.txt
+
+# set up environment
+PYTHONPATH=':'
+export PYTHONPATH
+ln -s $PWD/datumaro/datum.py ./datum
+chmod a+x datum
+
+# use Datumaro
+./datum --help
+```
+
+Check Datumaro [QUICKSTART.md](datumaro/docs/quickstart.md) for further info.

cvat/apps/dataset_manager/export_templates/extractors/cvat_rest_api_task_images.py

@@ -19,6 +19,10 @@
     .add('server_port', int) \
     .build()
 
+DEFAULT_CONFIG = Config({
+    'server_port': 80
+}, schema=CONFIG_SCHEMA, mutable=False)
+
 class cvat_rest_api_task_images(datumaro.Extractor):
     def _image_local_path(self, item_id):
         task_id = self._config.task_id
@@ -83,7 +87,8 @@ def __init__(self, url):
 
         with open(osp.join(url, 'config.json'), 'r') as config_file:
             config = json.load(config_file)
-            config = Config(config, schema=CONFIG_SCHEMA)
+            config = Config(config,
+                fallback=DEFAULT_CONFIG, schema=CONFIG_SCHEMA)
         self._config = config
 
         with open(osp.join(url, 'images_meta.json'), 'r') as images_file:

cvat/apps/dataset_manager/task.py

@@ -5,7 +5,6 @@
 import shutil
 import sys
 import tempfile
-from urllib.parse import urlsplit
 
 from django.utils import timezone
 import django_rq
@@ -14,7 +13,8 @@
 from cvat.apps.engine.models import Task, ShapeType
 from .util import current_function_name, make_zip_archive
 
-_DATUMARO_REPO_PATH = osp.join(__file__[:__file__.rfind('cvat/')], 'datumaro')
+_CVAT_ROOT_DIR = __file__[:__file__.rfind('cvat/')]
+_DATUMARO_REPO_PATH = osp.join(_CVAT_ROOT_DIR, 'datumaro')
 sys.path.append(_DATUMARO_REPO_PATH)
 from datumaro.components.project import Project
 import datumaro.components.extractor as datumaro
@@ -227,16 +227,17 @@ def _remote_image_converter(self, save_dir, server_url=None):
         items = []
         config = {
             'server_host': 'localhost',
-            'server_port': '',
             'task_id': db_task.id,
         }
         if server_url:
-            parsed_url = urlsplit(server_url)
-            config['server_host'] = parsed_url.netloc
-            port = 80
-            if parsed_url.port:
-                port = parsed_url.port
-            config['server_port'] = int(port)
+            if ':' in server_url:
+                host, port = server_url.rsplit(':', maxsplit=1)
+            else:
+                host = server_url
+                port = None
+            config['server_host'] = host
+            if port is not None:
+                config['server_port'] = int(port)
 
         images_meta = {
             'images': items,
@@ -271,10 +272,16 @@ def _remote_export(self, save_dir, server_url=None):
             server_url=server_url)
         exported_project.save()
 
-        templates_dir = osp.join(osp.dirname(__file__),
-            'export_templates', 'extractors')
-        target_dir = osp.join(
-            exported_project.config.project_dir,
+
+        templates_dir = osp.join(osp.dirname(__file__), 'export_templates')
+        target_dir = exported_project.config.project_dir
+        os.makedirs(target_dir, exist_ok=True)
+        shutil.copyfile(
+            osp.join(templates_dir, 'README.md'),
+            osp.join(target_dir, 'README.md'))
+
+        templates_dir = osp.join(templates_dir, 'extractors')
+        target_dir = osp.join(target_dir,
             exported_project.config.env_dir,
             exported_project.env.config.extractors_dir)
         os.makedirs(target_dir, exist_ok=True)
@@ -291,6 +298,12 @@ def _remote_export(self, save_dir, server_url=None):
                         ['.pyx', '.pyo', '.pyd', '.pyc']])
             ])
 
+        # include CVAT CLI module also
+        cvat_utils_dst_dir = osp.join(save_dir, 'cvat', 'utils')
+        os.makedirs(cvat_utils_dst_dir)
+        shutil.copytree(osp.join(_CVAT_ROOT_DIR, 'utils', 'cli'),
+            osp.join(cvat_utils_dst_dir, 'cli'))
+
 
 DEFAULT_FORMAT = EXPORT_FORMAT_DATUMARO_PROJECT
 DEFAULT_CACHE_TTL = timedelta(hours=10)

cvat/apps/engine/views.py

@@ -481,8 +481,8 @@ def dataset_export(self, request, pk):
         rq_job = queue.fetch_job(rq_id)
         if rq_job:
             task_time = timezone.localtime(db_task.updated_date)
-            request_time = timezone.localtime(
-                rq_job.meta.get('request_time', datetime.min))
+            request_time = rq_job.meta.get('request_time',
+                timezone.make_aware(datetime.min))
             if request_time < task_time:
                 rq_job.cancel()
                 rq_job.delete()
@@ -508,9 +508,14 @@ def dataset_export(self, request, pk):
                 else:
                     return Response(status=status.HTTP_202_ACCEPTED)
 
+        try:
+            server_address = request.get_host()
+        except Exception:
+            server_address = None
+
         ttl = DatumaroTask.CACHE_TTL.total_seconds()
         queue.enqueue_call(func=DatumaroTask.export_project,
-            args=(pk, request.user, dst_format), job_id=rq_id,
+            args=(pk, request.user, dst_format, server_address), job_id=rq_id,
             meta={ 'request_time': timezone.localtime() },
             result_ttl=ttl, failure_ttl=ttl)
         return Response(status=status.HTTP_202_ACCEPTED)