diff --git a/cvat/apps/dataset_manager/export_templates/README.md b/cvat/apps/dataset_manager/export_templates/README.md new file mode 100644 index 00000000000..82067fa24c0 --- /dev/null +++ b/cvat/apps/dataset_manager/export_templates/README.md @@ -0,0 +1,22 @@ +# Quick start + +``` bash +# optionally make a virtualenv +python -m virtualenv .venv +. .venv/bin/activate + +# install dependencies +sed -r "s/^(.*)#.*$/\1/g" datumaro/requirements.txt | xargs -n 1 -L 1 pip install +pip install -r cvat/utils/cli/requirements.txt + +# set up environment +PYTHONPATH=':' +export PYTHONPATH +ln -s $PWD/datumaro/datum.py ./datum +chmod a+x datum + +# use Datumaro +./datum --help +``` + +Check Datumaro [QUICKSTART.md](datumaro/docs/quickstart.md) for further info. diff --git a/cvat/apps/dataset_manager/export_templates/extractors/cvat_rest_api_task_images.py b/cvat/apps/dataset_manager/export_templates/extractors/cvat_rest_api_task_images.py index 276caa62f8b..9b6065daa9d 100644 --- a/cvat/apps/dataset_manager/export_templates/extractors/cvat_rest_api_task_images.py +++ b/cvat/apps/dataset_manager/export_templates/extractors/cvat_rest_api_task_images.py @@ -19,6 +19,10 @@ .add('server_port', int) \ .build() +DEFAULT_CONFIG = Config({ + 'server_port': 80 +}, schema=CONFIG_SCHEMA, mutable=False) + class cvat_rest_api_task_images(datumaro.Extractor): def _image_local_path(self, item_id): task_id = self._config.task_id @@ -83,7 +87,8 @@ def __init__(self, url): with open(osp.join(url, 'config.json'), 'r') as config_file: config = json.load(config_file) - config = Config(config, schema=CONFIG_SCHEMA) + config = Config(config, + fallback=DEFAULT_CONFIG, schema=CONFIG_SCHEMA) self._config = config with open(osp.join(url, 'images_meta.json'), 'r') as images_file: diff --git a/cvat/apps/dataset_manager/task.py b/cvat/apps/dataset_manager/task.py index 91807ede39a..74c9a3898e9 100644 --- a/cvat/apps/dataset_manager/task.py +++ b/cvat/apps/dataset_manager/task.py @@ -5,7 +5,6 @@ import shutil import sys import tempfile -from urllib.parse import urlsplit from django.utils import timezone import django_rq @@ -14,7 +13,8 @@ from cvat.apps.engine.models import Task, ShapeType from .util import current_function_name, make_zip_archive -_DATUMARO_REPO_PATH = osp.join(__file__[:__file__.rfind('cvat/')], 'datumaro') +_CVAT_ROOT_DIR = __file__[:__file__.rfind('cvat/')] +_DATUMARO_REPO_PATH = osp.join(_CVAT_ROOT_DIR, 'datumaro') sys.path.append(_DATUMARO_REPO_PATH) from datumaro.components.project import Project import datumaro.components.extractor as datumaro @@ -227,16 +227,17 @@ def _remote_image_converter(self, save_dir, server_url=None): items = [] config = { 'server_host': 'localhost', - 'server_port': '', 'task_id': db_task.id, } if server_url: - parsed_url = urlsplit(server_url) - config['server_host'] = parsed_url.netloc - port = 80 - if parsed_url.port: - port = parsed_url.port - config['server_port'] = int(port) + if ':' in server_url: + host, port = server_url.rsplit(':', maxsplit=1) + else: + host = server_url + port = None + config['server_host'] = host + if port is not None: + config['server_port'] = int(port) images_meta = { 'images': items, @@ -271,10 +272,16 @@ def _remote_export(self, save_dir, server_url=None): server_url=server_url) exported_project.save() - templates_dir = osp.join(osp.dirname(__file__), - 'export_templates', 'extractors') - target_dir = osp.join( - exported_project.config.project_dir, + + templates_dir = osp.join(osp.dirname(__file__), 'export_templates') + target_dir = exported_project.config.project_dir + os.makedirs(target_dir, exist_ok=True) + shutil.copyfile( + osp.join(templates_dir, 'README.md'), + osp.join(target_dir, 'README.md')) + + templates_dir = osp.join(templates_dir, 'extractors') + target_dir = osp.join(target_dir, exported_project.config.env_dir, exported_project.env.config.extractors_dir) os.makedirs(target_dir, exist_ok=True) @@ -291,6 +298,12 @@ def _remote_export(self, save_dir, server_url=None): ['.pyx', '.pyo', '.pyd', '.pyc']]) ]) + # include CVAT CLI module also + cvat_utils_dst_dir = osp.join(save_dir, 'cvat', 'utils') + os.makedirs(cvat_utils_dst_dir) + shutil.copytree(osp.join(_CVAT_ROOT_DIR, 'utils', 'cli'), + osp.join(cvat_utils_dst_dir, 'cli')) + DEFAULT_FORMAT = EXPORT_FORMAT_DATUMARO_PROJECT DEFAULT_CACHE_TTL = timedelta(hours=10) diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 6a5fc34014d..642256e4c5a 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -481,8 +481,8 @@ def dataset_export(self, request, pk): rq_job = queue.fetch_job(rq_id) if rq_job: task_time = timezone.localtime(db_task.updated_date) - request_time = timezone.localtime( - rq_job.meta.get('request_time', datetime.min)) + request_time = rq_job.meta.get('request_time', + timezone.make_aware(datetime.min)) if request_time < task_time: rq_job.cancel() rq_job.delete() @@ -508,9 +508,14 @@ def dataset_export(self, request, pk): else: return Response(status=status.HTTP_202_ACCEPTED) + try: + server_address = request.get_host() + except Exception: + server_address = None + ttl = DatumaroTask.CACHE_TTL.total_seconds() queue.enqueue_call(func=DatumaroTask.export_project, - args=(pk, request.user, dst_format), job_id=rq_id, + args=(pk, request.user, dst_format, server_address), job_id=rq_id, meta={ 'request_time': timezone.localtime() }, result_ttl=ttl, failure_ttl=ttl) return Response(status=status.HTTP_202_ACCEPTED)