From d8d9b922f1e3edca890974229c374e38d59031a2 Mon Sep 17 00:00:00 2001 From: "cvat-bot[bot]" <147643061+cvat-bot[bot]@users.noreply.github.com> Date: Tue, 13 Aug 2024 17:04:07 +0000 Subject: [PATCH 01/22] Update develop after v2.16.3 --- cvat-cli/requirements/base.txt | 2 +- cvat-cli/src/cvat_cli/version.py | 2 +- cvat-sdk/gen/generate.sh | 2 +- cvat/__init__.py | 2 +- cvat/schema.yml | 2 +- docker-compose.yml | 18 +++++++++--------- helm-chart/values.yaml | 4 ++-- 7 files changed, 16 insertions(+), 16 deletions(-) diff --git a/cvat-cli/requirements/base.txt b/cvat-cli/requirements/base.txt index ed788104590..849357d912d 100644 --- a/cvat-cli/requirements/base.txt +++ b/cvat-cli/requirements/base.txt @@ -1,3 +1,3 @@ -cvat-sdk~=2.16.3 +cvat-sdk~=2.17.0 Pillow>=10.3.0 setuptools>=70.0.0 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/cvat-cli/src/cvat_cli/version.py b/cvat-cli/src/cvat_cli/version.py index 85582c8a9e6..82ac3c4665d 100644 --- a/cvat-cli/src/cvat_cli/version.py +++ b/cvat-cli/src/cvat_cli/version.py @@ -1 +1 @@ -VERSION = "2.16.3" +VERSION = "2.17.0" diff --git a/cvat-sdk/gen/generate.sh b/cvat-sdk/gen/generate.sh index c2f9a575dc2..462f1284f28 100755 --- a/cvat-sdk/gen/generate.sh +++ b/cvat-sdk/gen/generate.sh @@ -8,7 +8,7 @@ set -e GENERATOR_VERSION="v6.0.1" -VERSION="2.16.3" +VERSION="2.17.0" LIB_NAME="cvat_sdk" LAYER1_LIB_NAME="${LIB_NAME}/api_client" DST_DIR="$(cd "$(dirname -- "$0")/.." && pwd)" diff --git a/cvat/__init__.py b/cvat/__init__.py index 572a69c7f2d..dff5c95b13b 100644 --- a/cvat/__init__.py +++ b/cvat/__init__.py @@ -4,6 +4,6 @@ from cvat.utils.version import get_version -VERSION = (2, 16, 3, 'final', 0) +VERSION = (2, 17, 0, 'alpha', 0) __version__ = get_version(VERSION) diff --git a/cvat/schema.yml b/cvat/schema.yml index f768eb1616c..d4f01b9642a 100644 --- a/cvat/schema.yml +++ b/cvat/schema.yml @@ -1,7 +1,7 @@ openapi: 3.0.3 info: title: CVAT REST API - version: 2.16.3 + version: 2.17.0 description: REST API for Computer Vision Annotation Tool (CVAT) termsOfService: https://www.google.com/policies/terms/ contact: diff --git a/docker-compose.yml b/docker-compose.yml index 5031ed1d416..051bd0bfd8c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -78,7 +78,7 @@ services: cvat_server: container_name: cvat_server - image: cvat/server:${CVAT_VERSION:-v2.16.3} + image: cvat/server:${CVAT_VERSION:-dev} restart: always depends_on: <<: *backend-deps @@ -112,7 +112,7 @@ services: cvat_utils: container_name: cvat_utils - image: cvat/server:${CVAT_VERSION:-v2.16.3} + image: cvat/server:${CVAT_VERSION:-dev} restart: always depends_on: *backend-deps environment: @@ -129,7 +129,7 @@ services: cvat_worker_import: container_name: cvat_worker_import - image: cvat/server:${CVAT_VERSION:-v2.16.3} + image: cvat/server:${CVAT_VERSION:-dev} restart: always depends_on: *backend-deps environment: @@ -145,7 +145,7 @@ services: cvat_worker_export: container_name: cvat_worker_export - image: cvat/server:${CVAT_VERSION:-v2.16.3} + image: cvat/server:${CVAT_VERSION:-dev} restart: always depends_on: *backend-deps environment: @@ -161,7 +161,7 @@ services: cvat_worker_annotation: container_name: cvat_worker_annotation - image: cvat/server:${CVAT_VERSION:-v2.16.3} + image: cvat/server:${CVAT_VERSION:-dev} restart: always depends_on: *backend-deps environment: @@ -177,7 +177,7 @@ services: cvat_worker_webhooks: container_name: cvat_worker_webhooks - image: cvat/server:${CVAT_VERSION:-v2.16.3} + image: cvat/server:${CVAT_VERSION:-dev} restart: always depends_on: *backend-deps environment: @@ -193,7 +193,7 @@ services: cvat_worker_quality_reports: container_name: cvat_worker_quality_reports - image: cvat/server:${CVAT_VERSION:-v2.16.3} + image: cvat/server:${CVAT_VERSION:-dev} restart: always depends_on: *backend-deps environment: @@ -209,7 +209,7 @@ services: cvat_worker_analytics_reports: container_name: cvat_worker_analytics_reports - image: cvat/server:${CVAT_VERSION:-v2.16.3} + image: cvat/server:${CVAT_VERSION:-dev} restart: always depends_on: *backend-deps environment: @@ -225,7 +225,7 @@ services: cvat_ui: container_name: cvat_ui - image: cvat/ui:${CVAT_VERSION:-v2.16.3} + image: cvat/ui:${CVAT_VERSION:-dev} restart: always depends_on: - cvat_server diff --git a/helm-chart/values.yaml b/helm-chart/values.yaml index bd3abce5cf6..91e4493258f 100644 --- a/helm-chart/values.yaml +++ b/helm-chart/values.yaml @@ -115,7 +115,7 @@ cvat: additionalVolumeMounts: [] replicas: 1 image: cvat/server - tag: v2.16.3 + tag: dev imagePullPolicy: Always permissionFix: enabled: true @@ -139,7 +139,7 @@ cvat: frontend: replicas: 1 image: cvat/ui - tag: v2.16.3 + tag: dev imagePullPolicy: Always labels: {} # test: test From e0a60d847aff64113390a4232d8b8aa6e9bd676d Mon Sep 17 00:00:00 2001 From: Roman Donchenko Date: Thu, 15 Aug 2024 13:42:40 +0300 Subject: [PATCH 02/22] Layer tus-js-client on top of Axios (#8281) This ensures that requests coming from tus-js-client have the same defaults as the ones coming from the rest of the UI. In particular, this ensures that TUS requests include the `X-CSRFTOKEN` header. Currently, this doesn't matter much, because TUS requests are authenticated using the token. However, I'd like to get rid of token authentication in the UI, after which `X-CSRFTOKEN` will become important. --- cvat-core/src/axios-tus.ts | 87 +++++++++++++++++++++++++++++++++++ cvat-core/src/server-proxy.ts | 12 +---- cvat/apps/engine/mixins.py | 3 +- 3 files changed, 91 insertions(+), 11 deletions(-) create mode 100644 cvat-core/src/axios-tus.ts diff --git a/cvat-core/src/axios-tus.ts b/cvat-core/src/axios-tus.ts new file mode 100644 index 00000000000..d26ef7c85c7 --- /dev/null +++ b/cvat-core/src/axios-tus.ts @@ -0,0 +1,87 @@ +// Copyright (C) 2024 CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import Axios, { AxiosRequestConfig, AxiosResponse } from 'axios'; +import * as tus from 'tus-js-client'; + +class AxiosHttpResponse implements tus.HttpResponse { + readonly #axiosResponse: AxiosResponse; + + constructor(axiosResponse: AxiosResponse) { + this.#axiosResponse = axiosResponse; + } + + getStatus(): number { + return this.#axiosResponse.status; + } + getHeader(header: string): string | undefined { + return this.#axiosResponse.headers[header.toLowerCase()]; + } + getBody(): string { + return this.#axiosResponse.data; + } + getUnderlyingObject(): AxiosResponse { + return this.#axiosResponse; + } +} + +class AxiosHttpRequest implements tus.HttpRequest { + readonly #axiosConfig: AxiosRequestConfig; + readonly #abortController: AbortController; + + constructor(method: string, url: string) { + this.#abortController = new AbortController(); + this.#axiosConfig = { + method, + url, + headers: {}, + signal: this.#abortController.signal, + validateStatus: () => true, + }; + } + + getMethod(): string { + return this.#axiosConfig.method; + } + getURL(): string { + return this.#axiosConfig.url; + } + + setHeader(header: string, value: string): void { + this.#axiosConfig.headers[header.toLowerCase()] = value; + } + getHeader(header: string): string | undefined { + return this.#axiosConfig.headers[header.toLowerCase()]; + } + + setProgressHandler(handler: (bytesSent: number) => void): void { + this.#axiosConfig.onUploadProgress = (progressEvent) => { + handler(progressEvent.loaded); + }; + } + + async send(body: any): Promise { + const axiosResponse = await Axios({ ...this.#axiosConfig, data: body }); + return new AxiosHttpResponse(axiosResponse); + } + + async abort(): Promise { + this.#abortController.abort(); + } + + getUnderlyingObject(): AxiosRequestConfig { + return this.#axiosConfig; + } +} + +class AxiosHttpStack implements tus.HttpStack { + createRequest(method: string, url: string): tus.HttpRequest { + return new AxiosHttpRequest(method, url); + } + getName(): string { + return 'AxiosHttpStack'; + } +} + +export const axiosTusHttpStack = new AxiosHttpStack(); diff --git a/cvat-core/src/server-proxy.ts b/cvat-core/src/server-proxy.ts index caa945d8448..1400b1e3db3 100644 --- a/cvat-core/src/server-proxy.ts +++ b/cvat-core/src/server-proxy.ts @@ -10,6 +10,7 @@ import * as tus from 'tus-js-client'; import { ChunkQuality } from 'cvat-data'; import './axios-config'; +import { axiosTusHttpStack } from './axios-tus'; import { SerializedLabel, SerializedAnnotationFormats, ProjectsFilter, SerializedProject, SerializedTask, TasksFilter, SerializedUser, SerializedOrganization, @@ -117,7 +118,6 @@ function fetchAll(url, filter = {}): Promise { } async function chunkUpload(file: File, uploadConfig): Promise<{ uploadSentSize: number; filename: string }> { - const params = enableOrganization(); const { endpoint, chunkSize, totalSize, onUpdate, metadata, totalSentSize, } = uploadConfig; @@ -130,9 +130,7 @@ async function chunkUpload(file: File, uploadConfig): Promise<{ uploadSentSize: filetype: file.type, ...metadata, }, - headers: { - Authorization: Axios.defaults.headers.common.Authorization, - }, + httpStack: axiosTusHttpStack, chunkSize, retryDelays: [2000, 4000, 8000, 16000, 32000, 64000], onShouldRetry(err: tus.DetailedError | Error): boolean { @@ -151,12 +149,6 @@ async function chunkUpload(file: File, uploadConfig): Promise<{ uploadSentSize: onError(error) { reject(error); }, - onBeforeRequest(req) { - const xhr = req.getUnderlyingObject(); - const { org } = params; - req.setHeader('X-Organization', org); - xhr.withCredentials = true; - }, onProgress(bytesUploaded) { if (onUpdate && Number.isInteger(totalSentSize) && Number.isInteger(totalSize)) { const currentUploadedSize = totalSentSize + bytesUploaded; diff --git a/cvat/apps/engine/mixins.py b/cvat/apps/engine/mixins.py index b0ab8315ae0..b4cfb279c7d 100644 --- a/cvat/apps/engine/mixins.py +++ b/cvat/apps/engine/mixins.py @@ -14,6 +14,7 @@ from unittest import mock from textwrap import dedent from typing import Optional, Callable, Dict, Any, Mapping +from urllib.parse import urljoin import django_rq from attr.converters import to_bool @@ -315,7 +316,7 @@ def init_tus_upload(self, request): return self._tus_response( status=status.HTTP_201_CREATED, - extra_headers={'Location': '{}{}'.format(location, tus_file.file_id), + extra_headers={'Location': urljoin(location, tus_file.file_id), 'Upload-Filename': tus_file.filename}) def append_tus_chunk(self, request, file_id): From 31d8fe08950be83dfe75febb1a458f75d3158ee5 Mon Sep 17 00:00:00 2001 From: Andrey Zhavoronkov Date: Thu, 15 Aug 2024 15:41:46 +0300 Subject: [PATCH 03/22] Fix unstable e2e test (#8303) Wait until the canvas is ready to paint points ### Motivation and context ![Single object annotation mode -- Tests basic features of single shape annotation mode -- Check basic single shape annotation pipeline for rectangle -- after each hook resetAfterTestCase (failed)](https://github.com/user-attachments/assets/74e039e4-4be4-434b-8f7b-fa702e744158) ### How has this been tested? ### Checklist - [x] I submit my changes into the `develop` branch - [ ] I have created a changelog fragment - [ ] I have updated the documentation accordingly - [ ] I have added tests to cover my changes - [ ] I have linked related issues (see [GitHub docs]( https://help.github.com/en/github/managing-your-work-on-github/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword)) - [x] I have increased versions of npm packages if it is necessary ([cvat-canvas](https://github.com/cvat-ai/cvat/tree/develop/cvat-canvas#versioning), [cvat-core](https://github.com/cvat-ai/cvat/tree/develop/cvat-core#versioning), [cvat-data](https://github.com/cvat-ai/cvat/tree/develop/cvat-data#versioning) and [cvat-ui](https://github.com/cvat-ai/cvat/tree/develop/cvat-ui#versioning)) ### License - [x] I submit _my code changes_ under the same [MIT License]( https://github.com/cvat-ai/cvat/blob/develop/LICENSE) that covers the project. Feel free to contact the maintainers if that's a concern. --- tests/cypress/e2e/features/single_object_annotation.js | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/cypress/e2e/features/single_object_annotation.js b/tests/cypress/e2e/features/single_object_annotation.js index 1ebbe9f6b6b..0b544ac3ca8 100644 --- a/tests/cypress/e2e/features/single_object_annotation.js +++ b/tests/cypress/e2e/features/single_object_annotation.js @@ -74,6 +74,7 @@ context('Single object annotation mode', { scrollBehavior: false }, () => { cy.get('.cvat-single-shape-annotation-sidebar-hint').should('exist'); cy.get('.cvat-single-shape-annotation-sidebar-ux-hints').should('exist'); + cy.get('#cvat_canvas_wrapper').should('have.css', 'cursor', 'crosshair'); } function openJob(params) { From 3f9083b48f0c7ec7d39b9a74b8befbea233a5130 Mon Sep 17 00:00:00 2001 From: Roman Donchenko Date: Thu, 15 Aug 2024 15:52:18 +0300 Subject: [PATCH 04/22] Prolong user sessions while they are used (#8288) Currently, this shouldn't have any visible effect, because the UI uses token authentication alongside session cookies, and the tokens last indefinitely. However, I'd like to end this practice and rely solely on session cookies. When that's implemented, the user will get logged out as soon as the session cookie expires, or the server-side session data expires (which should happen at the same time). This will irritate users if it happens too often (or worse, in the middle of their work). Therefore, we should prolong a session as long as it is used. --- cvat/apps/iam/middleware.py | 64 +++++++++++++++++++++++++++++++++++++ cvat/settings/base.py | 1 + 2 files changed, 65 insertions(+) diff --git a/cvat/apps/iam/middleware.py b/cvat/apps/iam/middleware.py index 2139e639cc6..f2f1a4bae2e 100644 --- a/cvat/apps/iam/middleware.py +++ b/cvat/apps/iam/middleware.py @@ -2,9 +2,13 @@ # # SPDX-License-Identifier: MIT +from datetime import timedelta +from typing import Callable + from django.utils.functional import SimpleLazyObject from rest_framework.exceptions import ValidationError, NotFound from django.conf import settings +from django.http import HttpRequest, HttpResponse def get_organization(request): @@ -57,3 +61,63 @@ def __call__(self, request): request.iam_context = SimpleLazyObject(lambda: get_organization(request)) return self.get_response(request) + +class SessionRefreshMiddleware: + """ + Implements behavior similar to SESSION_SAVE_EVERY_REQUEST=True, but instead of + saving the session on every request, saves it at most once per REFRESH_INTERVAL. + This is accomplished by setting a parallel cookie that expires whenever the session + needs to be prolonged. + + This ensures that user sessions are automatically prolonged while in use, + but avoids making an extra DB query on every HTTP request. + + Must be listed after SessionMiddleware in the MIDDLEWARE list. + """ + + _REFRESH_INTERVAL = timedelta(days=1) + _COOKIE_NAME = "sessionfresh" + + def __init__(self, get_response: Callable[[HttpRequest], HttpResponse]) -> None: + self.get_response = get_response + + def __call__(self, request: HttpRequest) -> HttpResponse: + response = self.get_response(request) + + shared_cookie_args = { + "key": self._COOKIE_NAME, + "domain": getattr(settings, "SESSION_COOKIE_DOMAIN"), + "path": getattr(settings, "SESSION_COOKIE_PATH", "/"), + "samesite": getattr(settings, "SESSION_COOKIE_SAMESITE", "Lax"), + } + + if request.session.is_empty(): + if self._COOKIE_NAME in request.COOKIES: + response.delete_cookie(**shared_cookie_args) + return response + + if self._COOKIE_NAME in request.COOKIES: + # Session is still fresh. + return response + + if response.status_code >= 500: + # SessionMiddleware does not save the session for 5xx responses, + # so we should not set our cookie either. + return response + + response.set_cookie( + **shared_cookie_args, + value="1", + max_age=min( + self._REFRESH_INTERVAL, + # Refresh no later than after half of the session lifetime. + timedelta(seconds=request.session.get_expiry_age() // 2), + ), + httponly=True, + secure=getattr(settings, "SESSION_COOKIE_SECURE", False), + ) + + # Force SessionMiddleware to re-save the session. + request.session.modified = True + + return response diff --git a/cvat/settings/base.py b/cvat/settings/base.py index 1f4b3592d69..6b1158690b2 100644 --- a/cvat/settings/base.py +++ b/cvat/settings/base.py @@ -183,6 +183,7 @@ def generate_secret_key(): MIDDLEWARE = [ 'django.middleware.security.SecurityMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', + 'cvat.apps.iam.middleware.SessionRefreshMiddleware', 'corsheaders.middleware.CorsMiddleware', 'django.middleware.common.CommonMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', From ac9ff3368684bf146385e8174315b1bff736c7db Mon Sep 17 00:00:00 2001 From: Roman Donchenko Date: Thu, 15 Aug 2024 20:31:23 +0300 Subject: [PATCH 05/22] Refactor RQId / RQIdManager (#8306) * Merge these into one class. These classes clearly deal with the same concept, so it doesn't make sense to divide the logic into two classes. * Turn `build` into an instance method (`render`). That way, the validation logic can be reused between it and the `RQId` constructor. Adjust the fields so that the first three fields can be specified as positional arguments. * Make the class frozen (I don't see a compelling case to mutate it). * Change string fields into corresponding enums. This reduces the amount of hardcoded string literals everywhere. Note that I had to move the enums into the `models` module to avoid a circular import. * Rename `resource` to `target`, because that's the name of the enum and the corresponding field in the API. --- cvat/apps/engine/background_operations.py | 29 ++++--- cvat/apps/engine/backup.py | 16 +++- cvat/apps/engine/mixins.py | 13 +-- cvat/apps/engine/models.py | 23 ++++- cvat/apps/engine/permissions.py | 6 +- cvat/apps/engine/rq_job_handler.py | 100 +++++++++++++--------- cvat/apps/engine/serializers.py | 34 ++------ cvat/apps/engine/task.py | 5 +- cvat/apps/engine/views.py | 54 ++++++------ cvat/settings/base.py | 2 +- 10 files changed, 161 insertions(+), 121 deletions(-) diff --git a/cvat/apps/engine/background_operations.py b/cvat/apps/engine/background_operations.py index b23eafa88a5..de21c248dce 100644 --- a/cvat/apps/engine/background_operations.py +++ b/cvat/apps/engine/background_operations.py @@ -28,9 +28,11 @@ from cvat.apps.engine.cloud_provider import export_resource_to_cloud_storage from cvat.apps.engine.location import StorageType, get_location_configuration from cvat.apps.engine.log import ServerLogManager -from cvat.apps.engine.models import Location, Project, Task +from cvat.apps.engine.models import ( + Location, Project, Task, RequestAction, RequestTarget, RequestSubresource, +) from cvat.apps.engine.permissions import get_cloud_storage_for_import_or_export -from cvat.apps.engine.rq_job_handler import RQIdManager +from cvat.apps.engine.rq_job_handler import RQId from cvat.apps.engine.serializers import RqIdSerializer from cvat.apps.engine.utils import ( build_annotations_file_name, @@ -342,14 +344,15 @@ def export(self) -> Response: return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) queue: DjangoRQ = django_rq.get_queue(self.QUEUE_NAME) - rq_id = RQIdManager.build( - "export", - self.resource, + rq_id = RQId( + RequestAction.EXPORT, + RequestTarget(self.resource), self.db_instance.pk, - subresource="dataset" if self.export_args.save_images else "annotations", - anno_format=self.export_args.format, + subresource=RequestSubresource.DATASET + if self.export_args.save_images else RequestSubresource.ANNOTATIONS, + format=self.export_args.format, user_id=self.request.user.id, - ) + ).render() rq_job = queue.fetch_job(rq_id) @@ -580,13 +583,13 @@ def _handle_rq_job_v1( def export(self) -> Response: queue: DjangoRQ = django_rq.get_queue(self.QUEUE_NAME) - rq_id = RQIdManager.build( - "export", - self.resource, + rq_id = RQId( + RequestAction.EXPORT, + RequestTarget(self.resource), self.db_instance.pk, - subresource="backup", + subresource=RequestSubresource.BACKUP, user_id=self.request.user.id, - ) + ).render() rq_job = queue.fetch_job(rq_id) if rq_job: diff --git a/cvat/apps/engine/backup.py b/cvat/apps/engine/backup.py index 6733c035400..bd3918a037a 100644 --- a/cvat/apps/engine/backup.py +++ b/cvat/apps/engine/backup.py @@ -39,9 +39,11 @@ get_rq_job_meta, import_resource_with_clean_up_after, define_dependent_job, get_rq_lock_by_user, ) -from cvat.apps.engine.rq_job_handler import RQIdManager, RQJobMetaField +from cvat.apps.engine.rq_job_handler import RQId, RQJobMetaField from cvat.apps.engine.models import ( - StorageChoice, StorageMethodChoice, DataChoice, Project, Location) + StorageChoice, StorageMethodChoice, DataChoice, Project, Location, + RequestAction, RequestTarget, RequestSubresource, +) from cvat.apps.engine.task import JobFileMapping, _create_thread from cvat.apps.engine.cloud_provider import import_resource_from_cloud_storage from cvat.apps.engine.location import StorageType, get_location_configuration @@ -1013,7 +1015,10 @@ def import_project(request, queue_name, filename=None): if 'rq_id' in request.data: rq_id = request.data['rq_id'] else: - rq_id = RQIdManager.build('import', 'project', uuid.uuid4(), subresource='backup') + rq_id = RQId( + RequestAction.IMPORT, RequestTarget.PROJECT, uuid.uuid4(), + subresource=RequestSubresource.BACKUP, + ).render() Serializer = ProjectFileSerializer file_field_name = 'project_file' @@ -1036,7 +1041,10 @@ def import_project(request, queue_name, filename=None): ) def import_task(request, queue_name, filename=None): - rq_id = request.data.get('rq_id', RQIdManager.build('import', 'task', uuid.uuid4(), subresource='backup')) + rq_id = request.data.get('rq_id', RQId( + RequestAction.IMPORT, RequestTarget.TASK, uuid.uuid4(), + subresource=RequestSubresource.BACKUP, + ).render()) Serializer = TaskFileSerializer file_field_name = 'task_file' diff --git a/cvat/apps/engine/mixins.py b/cvat/apps/engine/mixins.py index b4cfb279c7d..675152bf663 100644 --- a/cvat/apps/engine/mixins.py +++ b/cvat/apps/engine/mixins.py @@ -34,8 +34,8 @@ from cvat.apps.engine.handlers import clear_import_cache from cvat.apps.engine.location import StorageType, get_location_configuration from cvat.apps.engine.log import ServerLogManager -from cvat.apps.engine.models import Location -from cvat.apps.engine.rq_job_handler import RQIdManager +from cvat.apps.engine.models import Location, RequestAction, RequestTarget, RequestSubresource +from cvat.apps.engine.rq_job_handler import RQId from cvat.apps.engine.serializers import DataSerializer, RqIdSerializer from cvat.apps.engine.utils import is_dataset_export @@ -276,7 +276,10 @@ def init_tus_upload(self, request): if file_exists: # check whether the rq_job is in progress or has been finished/failed object_class_name = self._object.__class__.__name__.lower() - template = RQIdManager.build('import', object_class_name, self._object.pk, subresource=import_type) + template = RQId( + RequestAction.IMPORT, RequestTarget(object_class_name), self._object.pk, + subresource=RequestSubresource(import_type) + ).render() queue = django_rq.get_queue(settings.CVAT_QUEUES.IMPORT_DATA.value) finished_job_ids = queue.finished_job_registry.get_job_ids() failed_job_ids = queue.failed_job_registry.get_job_ids() @@ -474,7 +477,7 @@ def export_dataset_v2(self, request: HttpRequest, pk: int): return dataset_export_manager.export() # FUTURE-TODO: migrate to new API - def import_annotations(self, request, db_obj, import_func, rq_func, rq_id_template): + def import_annotations(self, request, db_obj, import_func, rq_func, rq_id_factory): is_tus_request = request.headers.get('Upload-Length', None) is not None or \ request.method == 'OPTIONS' if is_tus_request: @@ -493,7 +496,7 @@ def import_annotations(self, request, db_obj, import_func, rq_func, rq_id_templa return import_func( request=request, - rq_id_template=rq_id_template, + rq_id_factory=rq_id_factory, rq_func=rq_func, db_obj=self._object, format_name=format_name, diff --git a/cvat/apps/engine/models.py b/cvat/apps/engine/models.py index fe7fcbf79ad..84ab337e931 100644 --- a/cvat/apps/engine/models.py +++ b/cvat/apps/engine/models.py @@ -19,7 +19,7 @@ from django.core.exceptions import ValidationError from django.db import IntegrityError, models, transaction from django.db.models.fields import FloatField -from django.db.models import Q +from django.db.models import Q, TextChoices from drf_spectacular.types import OpenApiTypes from drf_spectacular.utils import extend_schema_field from cvat.apps.engine.lazy_list import LazyList @@ -1196,3 +1196,24 @@ def organization_id(self): def get_asset_dir(self): return os.path.join(settings.ASSETS_ROOT, str(self.uuid)) + +class RequestStatus(TextChoices): + QUEUED = "queued" + STARTED = "started" + FAILED = "failed" + FINISHED = "finished" + +class RequestAction(TextChoices): + CREATE = "create" + IMPORT = "import" + EXPORT = "export" + +class RequestTarget(TextChoices): + PROJECT = "project" + TASK = "task" + JOB = "job" + +class RequestSubresource(TextChoices): + ANNOTATIONS = "annotations" + DATASET = "dataset" + BACKUP = "backup" diff --git a/cvat/apps/engine/permissions.py b/cvat/apps/engine/permissions.py index 5e0b2f5125b..414d6488263 100644 --- a/cvat/apps/engine/permissions.py +++ b/cvat/apps/engine/permissions.py @@ -1227,7 +1227,7 @@ def create(cls, request, view, obj: Optional[RQJob], iam_context: Dict): ('export', 'task', 'backup'): (TaskPermission, TaskPermission.Scopes.EXPORT_BACKUP), ('export', 'job', 'annotations'): (JobPermission, JobPermission.Scopes.EXPORT_ANNOTATIONS), ('export', 'job', 'dataset'): (JobPermission, JobPermission.Scopes.EXPORT_DATASET), - }[(parsed_rq_id.action, parsed_rq_id.resource, parsed_rq_id.subresource)] + }[(parsed_rq_id.action, parsed_rq_id.target, parsed_rq_id.subresource)] resource = None @@ -1236,12 +1236,12 @@ def create(cls, request, view, obj: Optional[RQJob], iam_context: Dict): 'project': Project, 'task': Task, 'job': Job, - }[parsed_rq_id.resource] + }[parsed_rq_id.target] try: resource = resource_model.objects.get(id=resource_id) except resource_model.DoesNotExist as ex: - raise NotFound(f'The {parsed_rq_id.resource!r} with specified id#{resource_id} does not exist') from ex + raise NotFound(f'The {parsed_rq_id.target!r} with specified id#{resource_id} does not exist') from ex permissions.append(permission_class.create_base_perm(request, view, scope=resource_scope, iam_context=iam_context, obj=resource)) diff --git a/cvat/apps/engine/rq_job_handler.py b/cvat/apps/engine/rq_job_handler.py index 7c7b41deef9..dfbfbbf6ae2 100644 --- a/cvat/apps/engine/rq_job_handler.py +++ b/cvat/apps/engine/rq_job_handler.py @@ -2,12 +2,16 @@ # # SPDX-License-Identifier: MIT +from __future__ import annotations + import attrs from typing import Optional, Union from uuid import UUID from rq.job import Job as RQJob +from .models import RequestAction, RequestTarget, RequestSubresource + class RQJobMetaField: # common fields FORMATTED_EXCEPTION = "formatted_exception" @@ -28,82 +32,96 @@ class RQJobMetaField: def is_rq_job_owner(rq_job: RQJob, user_id: int) -> bool: return rq_job.meta.get(RQJobMetaField.USER, {}).get('id') == user_id -@attrs.define(kw_only=True) +@attrs.frozen() class RQId: - action: str = attrs.field( - validator=attrs.validators.instance_of(str) + action: RequestAction = attrs.field( + validator=attrs.validators.instance_of(RequestAction) ) - resource: str = attrs.field( - validator=attrs.validators.instance_of(str) + target: RequestTarget = attrs.field( + validator=attrs.validators.instance_of(RequestTarget) ) identifier: Union[int, UUID] = attrs.field( validator=attrs.validators.instance_of((int, UUID)) ) - subresource: Optional[str] = attrs.field( + subresource: Optional[RequestSubresource] = attrs.field( validator=attrs.validators.optional( - attrs.validators.instance_of(str) - ) + attrs.validators.instance_of(RequestSubresource) + ), + kw_only=True, default=None, ) user_id: Optional[int] = attrs.field( - validator=attrs.validators.optional(attrs.validators.instance_of(int)) + validator=attrs.validators.optional(attrs.validators.instance_of(int)), + kw_only=True, default=None, ) format: Optional[str] = attrs.field( - validator=attrs.validators.optional(attrs.validators.instance_of(str)) + validator=attrs.validators.optional(attrs.validators.instance_of(str)), + kw_only=True, default=None, ) + _OPTIONAL_FIELD_REQUIREMENTS = { + RequestAction.CREATE: {"subresource": False, "format": False, "user_id": False}, + RequestAction.EXPORT: {"subresource": True, "user_id": True}, + RequestAction.IMPORT: {"subresource": True, "format": False, "user_id": False}, + } + + def __attrs_post_init__(self) -> None: + for field, req in self._OPTIONAL_FIELD_REQUIREMENTS[self.action].items(): + if req: + if getattr(self, field) is None: + raise ValueError(f"{field} is required for the {self.action} action") + else: + if getattr(self, field) is not None: + raise ValueError(f"{field} is not allowed for the {self.action} action") -class RQIdManager: # RQ ID templates: # import:-- # create:task- # export:---in--format-by- # export:--backup-by- - @staticmethod - def build( - action: str, - resource: str, - identifier: Union[int, UUID], - *, - subresource: Optional[str] = None, - user_id: Optional[int] = None, - anno_format: Optional[str] = None, + def render( + self, ) -> str: - if "import" == action: - return f"{action}:{resource}-{identifier}-{subresource}" - elif "export" == action: - if anno_format is None: + common_prefix = f"{self.action}:{self.target}-{self.identifier}" + + if RequestAction.IMPORT == self.action: + return f"{common_prefix}-{self.subresource}" + elif RequestAction.EXPORT == self.action: + if self.format is None: return ( - f"{action}:{resource}-{identifier}-{subresource}-by-{user_id}" + f"{common_prefix}-{self.subresource}-by-{self.user_id}" ) - format_to_be_used_in_urls = anno_format.replace(" ", "_").replace(".", "@") - return f"{action}:{resource}-{identifier}-{subresource}-in-{format_to_be_used_in_urls}-format-by-{user_id}" - elif "create" == action: - assert "task" == resource - return f"{action}:{resource}-{identifier}" + + format_to_be_used_in_urls = self.format.replace(" ", "_").replace(".", "@") + return f"{common_prefix}-{self.subresource}-in-{format_to_be_used_in_urls}-format-by-{self.user_id}" + elif RequestAction.CREATE == self.action: + return common_prefix else: - raise ValueError(f"Unsupported action {action!r} was found") + assert False, f"Unsupported action {self.action!r} was found" @staticmethod def parse(rq_id: str) -> RQId: - action: Optional[str] = None - resource: Optional[str] = None identifier: Optional[Union[UUID, int]] = None - subresource: Optional[str] = None + subresource: Optional[RequestSubresource] = None user_id: Optional[int] = None anno_format: Optional[str] = None try: action_and_resource, unparsed = rq_id.split("-", maxsplit=1) - action, resource = action_and_resource.split(":") + action_str, target_str = action_and_resource.split(":") + action = RequestAction(action_str) + target = RequestTarget(target_str) - if "create" == action: + if RequestAction.CREATE == action: identifier = unparsed - elif "import" == action: - identifier, subresource = unparsed.rsplit("-", maxsplit=1) + elif RequestAction.IMPORT == action: + identifier, subresource_str = unparsed.rsplit("-", maxsplit=1) + subresource = RequestSubresource(subresource_str) else: # action == export - identifier, subresource, unparsed = unparsed.split("-", maxsplit=2) - if "backup" == subresource: + identifier, subresource_str, unparsed = unparsed.split("-", maxsplit=2) + subresource = RequestSubresource(subresource_str) + + if RequestSubresource.BACKUP == subresource: _, user_id = unparsed.split("-") else: unparsed, _, user_id = unparsed.rsplit("-", maxsplit=2) @@ -122,7 +140,7 @@ def parse(rq_id: str) -> RQId: return RQId( action=action, - resource=resource, + target=target, identifier=identifier, subresource=subresource, user_id=user_id, diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index 766da57da47..d0177140e45 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -23,7 +23,6 @@ from rest_framework import serializers, exceptions from django.contrib.auth.models import User, Group from django.db import transaction -from django.db.models import TextChoices from django.utils import timezone from cvat.apps.dataset_manager.formats.utils import get_label_color @@ -2201,30 +2200,10 @@ class Meta(BasicUserSerializer.Meta): "username", ) -class RequestStatus(TextChoices): - QUEUED = "queued" - STARTED = "started" - FAILED = "failed" - FINISHED = "finished" - -class RequestAction(TextChoices): - CREATE = "create" - IMPORT = "import" - EXPORT = "export" - -class RequestTarget(TextChoices): - PROJECT = "project" - TASK = "task" - JOB = "job" - -class RequestSubresource(TextChoices): - ANNOTATIONS = "annotations" - DATASET = "dataset" - BACKUP = "backup" class RequestDataOperationSerializer(serializers.Serializer): type = serializers.CharField() - target = serializers.ChoiceField(choices=RequestTarget.choices) + target = serializers.ChoiceField(choices=models.RequestTarget.choices) project_id = serializers.IntegerField(required=False, allow_null=True) task_id = serializers.IntegerField(required=False, allow_null=True) job_id = serializers.IntegerField(required=False, allow_null=True) @@ -2237,10 +2216,10 @@ def to_representation(self, rq_job: RQJob) -> Dict[str, Any]: "type": ":".join( [ parsed_rq_id.action, - parsed_rq_id.subresource or parsed_rq_id.resource, + parsed_rq_id.subresource or parsed_rq_id.target, ] ), - "target": parsed_rq_id.resource, + "target": parsed_rq_id.target, "project_id": rq_job.meta[RQJobMetaField.PROJECT_ID], "task_id": rq_job.meta[RQJobMetaField.TASK_ID], "job_id": rq_job.meta[RQJobMetaField.JOB_ID], @@ -2252,7 +2231,7 @@ class RequestSerializer(serializers.Serializer): # Marking them as read_only leads to generating type as allOf with one reference to RequestStatus component. # The client generated using openapi-generator from such a schema contains wrong type like: # status (bool, date, datetime, dict, float, int, list, str, none_type): [optional] - status = serializers.ChoiceField(source="get_status", choices=RequestStatus.choices) + status = serializers.ChoiceField(source="get_status", choices=models.RequestStatus.choices) message = serializers.SerializerMethodField() id = serializers.CharField() operation = RequestDataOperationSerializer(source="*") @@ -2320,7 +2299,10 @@ def to_representation(self, rq_job: RQJob) -> Dict[str, Any]: if result_url := rq_job.meta.get(RQJobMetaField.RESULT_URL): representation["result_url"] = result_url - if rq_job.parsed_rq_id.action == RequestAction.IMPORT and rq_job.parsed_rq_id.subresource == RequestSubresource.BACKUP: + if ( + rq_job.parsed_rq_id.action == models.RequestAction.IMPORT + and rq_job.parsed_rq_id.subresource == models.RequestSubresource.BACKUP + ): representation["result_id"] = rq_job.return_value() return representation diff --git a/cvat/apps/engine/task.py b/cvat/apps/engine/task.py index c44f01e1f35..0db84cebc32 100644 --- a/cvat/apps/engine/task.py +++ b/cvat/apps/engine/task.py @@ -27,10 +27,11 @@ from cvat.apps.engine.log import ServerLogManager from cvat.apps.engine.media_extractors import (MEDIA_TYPES, ImageListReader, Mpeg4ChunkWriter, Mpeg4CompressedChunkWriter, ValidateDimension, ZipChunkWriter, ZipCompressedChunkWriter, get_mime, sort) +from cvat.apps.engine.models import RequestAction, RequestTarget from cvat.apps.engine.utils import ( av_scan_paths,get_rq_job_meta, define_dependent_job, get_rq_lock_by_user, preload_images ) -from cvat.apps.engine.rq_job_handler import RQIdManager +from cvat.apps.engine.rq_job_handler import RQId from cvat.utils.http import make_requests_session, PROXIES_FOR_UNTRUSTED_URLS from utils.dataset_manifest import ImageManifestManager, VideoManifestManager, is_manifest from utils.dataset_manifest.core import VideoManifestValidator, is_dataset_manifest @@ -49,7 +50,7 @@ def create( """Schedule a background job to create a task and return that job's identifier""" q = django_rq.get_queue(settings.CVAT_QUEUES.IMPORT_DATA.value) user_id = request.user.id - rq_id = RQIdManager.build('create', 'task', db_task.pk) + rq_id = RQId(RequestAction.CREATE, RequestTarget.TASK, db_task.pk).render() with get_rq_lock_by_user(q, user_id): q.enqueue_call( diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index e43930caaa8..92d9db78316 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -66,7 +66,8 @@ ClientFile, Job, JobType, Label, SegmentType, Task, Project, Issue, Data, Comment, StorageMethodChoice, StorageChoice, CloudProviderChoice, Location, CloudStorage as CloudStorageModel, - Asset, AnnotationGuide) + Asset, AnnotationGuide, RequestStatus, RequestAction, RequestTarget, RequestSubresource +) from cvat.apps.engine.serializers import ( AboutSerializer, AnnotationFileSerializer, BasicUserSerializer, DataMetaReadSerializer, DataMetaWriteSerializer, DataSerializer, @@ -80,7 +81,7 @@ IssueWriteSerializer, CommentReadSerializer, CommentWriteSerializer, CloudStorageWriteSerializer, CloudStorageReadSerializer, DatasetFileSerializer, ProjectFileSerializer, TaskFileSerializer, RqIdSerializer, CloudStorageContentSerializer, - RequestSerializer, RequestStatus, RequestAction, RequestSubresource, + RequestSerializer, ) from cvat.apps.engine.permissions import get_cloud_storage_for_import_or_export @@ -90,7 +91,7 @@ parse_exception_message, get_rq_job_meta, import_resource_with_clean_up_after, sendfile, define_dependent_job, get_rq_lock_by_user, ) -from cvat.apps.engine.rq_job_handler import RQIdManager, is_rq_job_owner, RQJobMetaField +from cvat.apps.engine.rq_job_handler import RQId, is_rq_job_owner, RQJobMetaField from cvat.apps.engine import backup from cvat.apps.engine.mixins import ( PartialUpdateModelMixin, UploadMixin, DatasetMixin, BackupMixin, CsrfWorkaroundMixin @@ -279,8 +280,8 @@ class ProjectViewSet(viewsets.GenericViewSet, mixins.ListModelMixin, ordering = "-id" lookup_fields = {'owner': 'owner__username', 'assignee': 'assignee__username'} iam_organization_field = 'organization' - IMPORT_RQ_ID_TEMPLATE = RQIdManager.build( - 'import', 'project', {}, subresource='dataset' + IMPORT_RQ_ID_FACTORY = functools.partial(RQId, + RequestAction.IMPORT, RequestTarget.PROJECT, subresource=RequestSubresource.DATASET ) def get_serializer_class(self): @@ -401,7 +402,7 @@ def dataset(self, request, pk): db_obj=self._object, import_func=_import_project_dataset, rq_func=dm.project.import_dataset_as_project, - rq_id_template=self.IMPORT_RQ_ID_TEMPLATE + rq_id_factory=self.IMPORT_RQ_ID_FACTORY, ) else: action = request.query_params.get("action", "").lower() @@ -467,7 +468,7 @@ def upload_finished(self, request): return _import_project_dataset( request=request, filename=uploaded_file, - rq_id_template=self.IMPORT_RQ_ID_TEMPLATE, + rq_id_factory=self.IMPORT_RQ_ID_FACTORY, rq_func=dm.project.import_dataset_as_project, db_obj=self._object, format_name=format_name, @@ -857,8 +858,8 @@ class TaskViewSet(viewsets.GenericViewSet, mixins.ListModelMixin, ordering_fields = list(filter_fields) ordering = "-id" iam_organization_field = 'organization' - IMPORT_RQ_ID_TEMPLATE = RQIdManager.build( - 'import', 'task', {}, subresource='annotations' + IMPORT_RQ_ID_FACTORY = functools.partial(RQId, + RequestAction.IMPORT, RequestTarget.TASK, subresource=RequestSubresource.ANNOTATIONS, ) def get_serializer_class(self): @@ -1085,7 +1086,7 @@ def _handle_upload_annotations(request): return _import_annotations( request=request, filename=annotation_file, - rq_id_template=self.IMPORT_RQ_ID_TEMPLATE, + rq_id_factory=self.IMPORT_RQ_ID_FACTORY, rq_func=dm.task.import_task_annotations, db_obj=self._object, format_name=format_name, @@ -1456,7 +1457,7 @@ def annotations(self, request, pk): db_obj=self._object, import_func=_import_annotations, rq_func=dm.task.import_task_annotations, - rq_id_template=self.IMPORT_RQ_ID_TEMPLATE + rq_id_factory=self.IMPORT_RQ_ID_FACTORY, ) elif request.method == 'PUT': format_name = request.query_params.get('format', '') @@ -1468,7 +1469,7 @@ def annotations(self, request, pk): ) return _import_annotations( request=request, - rq_id_template=self.IMPORT_RQ_ID_TEMPLATE, + rq_id_factory=self.IMPORT_RQ_ID_FACTORY, rq_func=dm.task.import_task_annotations, db_obj=self._object, format_name=format_name, @@ -1514,10 +1515,10 @@ def append_annotations_chunk(self, request, pk, file_id): ) @action(detail=True, methods=['GET'], serializer_class=RqStatusSerializer) def status(self, request, pk): - self.get_object() # force call of check_object_permissions() + task = self.get_object() # force call of check_object_permissions() response = self._get_rq_response( queue=settings.CVAT_QUEUES.IMPORT_DATA.value, - job_id=RQIdManager.build('create', 'task', pk) + job_id=RQId(RequestAction.CREATE, RequestTarget.TASK, task.id).render() ) serializer = RqStatusSerializer(data=response) @@ -1727,8 +1728,8 @@ class JobViewSet(viewsets.GenericViewSet, mixins.ListModelMixin, mixins.CreateMo 'project_name': 'segment__task__project__name', 'assignee': 'assignee__username' } - IMPORT_RQ_ID_TEMPLATE = RQIdManager.build( - 'import', 'job', {}, subresource='annotations' + IMPORT_RQ_ID_FACTORY = functools.partial(RQId, + RequestAction.IMPORT, RequestTarget.JOB, subresource=RequestSubresource.ANNOTATIONS ) def get_queryset(self): @@ -1775,7 +1776,7 @@ def upload_finished(self, request): return _import_annotations( request=request, filename=annotation_file, - rq_id_template=self.IMPORT_RQ_ID_TEMPLATE, + rq_id_factory=self.IMPORT_RQ_ID_FACTORY, rq_func=dm.task.import_job_annotations, db_obj=self._object, format_name=format_name, @@ -1926,7 +1927,7 @@ def annotations(self, request, pk): db_obj=self._object, import_func=_import_annotations, rq_func=dm.task.import_job_annotations, - rq_id_template=self.IMPORT_RQ_ID_TEMPLATE + rq_id_factory=self.IMPORT_RQ_ID_FACTORY, ) elif request.method == 'PUT': @@ -1938,7 +1939,7 @@ def annotations(self, request, pk): ) return _import_annotations( request=request, - rq_id_template=self.IMPORT_RQ_ID_TEMPLATE, + rq_id_factory=self.IMPORT_RQ_ID_FACTORY, rq_func=dm.task.import_job_annotations, db_obj=self._object, format_name=format_name, @@ -3025,7 +3026,7 @@ def rq_exception_handler(rq_job, exc_type, exc_value, tb): return True -def _import_annotations(request, rq_id_template, rq_func, db_obj, format_name, +def _import_annotations(request, rq_id_factory, rq_func, db_obj, format_name, filename=None, location_conf=None, conv_mask_to_poly=True): format_desc = {f.DISPLAY_NAME: f @@ -3039,7 +3040,7 @@ def _import_annotations(request, rq_id_template, rq_func, db_obj, format_name, rq_id = request.query_params.get('rq_id') rq_id_should_be_checked = bool(rq_id) if not rq_id: - rq_id = rq_id_template.format(db_obj.pk) + rq_id = rq_id_factory(db_obj.pk).render() queue = django_rq.get_queue(settings.CVAT_QUEUES.IMPORT_DATA.value) rq_job = queue.fetch_job(rq_id) @@ -3141,7 +3142,10 @@ def _import_annotations(request, rq_id_template, rq_func, db_obj, format_name, return Response(status=status.HTTP_202_ACCEPTED) -def _import_project_dataset(request, rq_id_template, rq_func, db_obj, format_name, filename=None, conv_mask_to_poly=True, location_conf=None): +def _import_project_dataset( + request, rq_id_factory, rq_func, db_obj, format_name, + filename=None, conv_mask_to_poly=True, location_conf=None +): format_desc = {f.DISPLAY_NAME: f for f in dm.views.get_import_formats()}.get(format_name) if format_desc is None: @@ -3150,7 +3154,7 @@ def _import_project_dataset(request, rq_id_template, rq_func, db_obj, format_nam elif not format_desc.ENABLED: return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) - rq_id = rq_id_template.format(db_obj.pk) + rq_id = rq_id_factory(db_obj.pk).render() queue = django_rq.get_queue(settings.CVAT_QUEUES.IMPORT_DATA.value) rq_job = queue.fetch_job(rq_id) @@ -3319,7 +3323,7 @@ def _get_rq_jobs_from_queue(self, queue: DjangoRQ, user_id: int) -> List[RQJob]: for job in queue.job_class.fetch_many(job_ids, queue.connection): if job and is_rq_job_owner(job, user_id): try: - parsed_rq_id = RQIdManager.parse(job.id) + parsed_rq_id = RQId.parse(job.id) except Exception: # nosec B112 continue job.parsed_rq_id = parsed_rq_id @@ -3356,7 +3360,7 @@ def _get_rq_job_by_id(self, rq_id: str) -> Optional[RQJob]: Optional[RQJob]: The retrieved RQJob, or None if not found. """ try: - parsed_rq_id = RQIdManager.parse(rq_id) + parsed_rq_id = RQId.parse(rq_id) except Exception: return None diff --git a/cvat/settings/base.py b/cvat/settings/base.py index 6b1158690b2..106048d8601 100644 --- a/cvat/settings/base.py +++ b/cvat/settings/base.py @@ -637,7 +637,7 @@ class CVAT_QUEUES(Enum): 'SortingMethod': 'cvat.apps.engine.models.SortingMethod', 'WebhookType': 'cvat.apps.webhooks.models.WebhookTypeChoice', 'WebhookContentType': 'cvat.apps.webhooks.models.WebhookContentTypeChoice', - 'RequestStatus': 'cvat.apps.engine.serializers.RequestStatus', + 'RequestStatus': 'cvat.apps.engine.models.RequestStatus', }, # Coercion of {pk} to {id} is controlled by SCHEMA_COERCE_PATH_PK. Additionally, From a69e1228ac8ab120ca9f1274348e83ddcb89d4be Mon Sep 17 00:00:00 2001 From: Andrey Zhavoronkov Date: Fri, 16 Aug 2024 14:06:44 +0300 Subject: [PATCH 06/22] Update dependencies (#8308) Updated: backend python packages golang image frontend nginx base image --- Dockerfile | 4 ++-- Dockerfile.ui | 3 ++- cvat/requirements/base.txt | 18 +++++++++--------- cvat/requirements/development.txt | 4 ++-- cvat/requirements/production.txt | 2 +- 5 files changed, 16 insertions(+), 15 deletions(-) diff --git a/Dockerfile b/Dockerfile index b094a134aa2..b4f2b8ef254 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ ARG PIP_VERSION=24.0 ARG BASE_IMAGE=ubuntu:22.04 -FROM ${BASE_IMAGE} as build-image-base +FROM ${BASE_IMAGE} AS build-image-base RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get --no-install-recommends install -yq \ @@ -84,7 +84,7 @@ RUN --mount=type=cache,target=/root/.cache/pip/http-v2 \ -r /tmp/cvat/requirements/${CVAT_CONFIGURATION}.txt \ -w /tmp/wheelhouse -FROM golang:1.22.4 AS build-smokescreen +FROM golang:1.23.0 AS build-smokescreen RUN git clone --filter=blob:none --no-checkout https://github.com/stripe/smokescreen.git RUN cd smokescreen && git checkout eb1ac09 && go build -o /tmp/smokescreen diff --git a/Dockerfile.ui b/Dockerfile.ui index 9de10429846..170ee1a7663 100644 --- a/Dockerfile.ui +++ b/Dockerfile.ui @@ -34,7 +34,8 @@ DISABLE_SOURCE_MAPS="${DISABLE_SOURCE_MAPS}" \ UI_APP_CONFIG="${UI_APP_CONFIG}" \ SOURCE_MAPS_TOKEN="${SOURCE_MAPS_TOKEN}" yarn run build:cvat-ui -FROM nginx:1.25.4-alpine3.18 +FROM nginx:1.26.1-alpine3.19-slim + # Replace default.conf configuration to remove unnecessary rules COPY cvat-ui/react_nginx.conf /etc/nginx/conf.d/default.conf COPY --from=cvat-ui /tmp/cvat-ui/dist /usr/share/nginx/html/ diff --git a/cvat/requirements/base.txt b/cvat/requirements/base.txt index 470506abae8..4e4130de921 100644 --- a/cvat/requirements/base.txt +++ b/cvat/requirements/base.txt @@ -35,7 +35,7 @@ certifi==2024.7.4 # clickhouse-connect # msrest # requests -cffi==1.16.0 +cffi==1.17.0 # via cryptography charset-normalizer==3.3.2 # via requests @@ -69,7 +69,7 @@ dj-pagination==2.5.0 # via -r cvat/requirements/base.in dj-rest-auth[with-social]==5.0.2 # via -r cvat/requirements/base.in -django==4.2.14 +django==4.2.15 # via # -r cvat/requirements/base.in # dj-rest-auth @@ -127,7 +127,7 @@ google-api-core==2.19.1 # via # google-cloud-core # google-cloud-storage -google-auth==2.32.0 +google-auth==2.33.0 # via # google-api-core # google-cloud-core @@ -138,7 +138,7 @@ google-cloud-storage==1.42.0 # via -r cvat/requirements/base.in google-crc32c==1.5.0 # via google-resumable-media -google-resumable-media==2.7.1 +google-resumable-media==2.7.2 # via google-cloud-storage googleapis-common-protos==1.63.2 # via google-api-core @@ -148,7 +148,7 @@ idna==3.7 # via requests importlib-metadata==8.2.0 # via clickhouse-connect -importlib-resources==6.4.0 +importlib-resources==6.4.2 # via limits inflection==0.5.1 # via drf-spectacular @@ -170,7 +170,7 @@ kiwisolver==1.4.5 # via matplotlib limits==3.13.0 # via python-logstash-async -lxml==5.2.2 +lxml==5.3.0 # via # -r cvat/requirements/base.in # datumaro @@ -196,7 +196,7 @@ oauthlib==3.2.2 # via requests-oauthlib orderedmultidict==1.0.1 # via furl -orjson==3.10.6 +orjson==3.10.7 # via datumaro packaging==24.1 # via @@ -269,7 +269,7 @@ pytz==2024.1 # pandas pyunpack==0.2.1 # via -r cvat/requirements/base.in -pyyaml==6.0.1 +pyyaml==6.0.2 # via # datumaro # drf-spectacular @@ -355,7 +355,7 @@ xmlsec==1.3.14 # via # -r cvat/requirements/base.in # python3-saml -zipp==3.19.2 +zipp==3.20.0 # via importlib-metadata zstandard==0.23.0 # via clickhouse-connect diff --git a/cvat/requirements/development.txt b/cvat/requirements/development.txt index 6417758d86c..63f262430eb 100644 --- a/cvat/requirements/development.txt +++ b/cvat/requirements/development.txt @@ -57,11 +57,11 @@ tomli==2.0.1 # autopep8 # black # pylint -tomlkit==0.13.0 +tomlkit==0.13.2 # via pylint tornado==6.4.1 # via snakeviz # The following packages are considered to be unsafe in a requirements file: -setuptools==72.1.0 +setuptools==72.2.0 # via astroid diff --git a/cvat/requirements/production.txt b/cvat/requirements/production.txt index 0c76e5e57e1..0659df392c9 100644 --- a/cvat/requirements/production.txt +++ b/cvat/requirements/production.txt @@ -27,7 +27,7 @@ uvicorn[standard]==0.22.0 # via -r cvat/requirements/production.in uvloop==0.19.0 # via uvicorn -watchfiles==0.22.0 +watchfiles==0.23.0 # via uvicorn websockets==12.0 # via uvicorn From 0c5545bf08002940751b9fe1babf84126788233c Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Mon, 19 Aug 2024 10:37:48 +0300 Subject: [PATCH 07/22] Clear some client-side events collected by analytics (#8304) --- .../20240814_151947_boris_update_events.md | 3 ++ .../20240814_153835_boris_update_events.md | 4 ++ cvat-canvas/README.md | 4 +- cvat-canvas/package.json | 2 +- cvat-canvas/src/typescript/canvasView.ts | 10 ++++- cvat-core/package.json | 2 +- cvat-core/src/enums.ts | 7 +--- cvat-core/src/logger.ts | 29 +++++-------- cvat-ui/package.json | 2 +- cvat-ui/src/actions/annotation-actions.ts | 36 ++-------------- cvat-ui/src/actions/boundaries-actions.ts | 5 +-- cvat-ui/src/actions/import-actions.ts | 2 - .../annotation-page/annotation-page.tsx | 10 ++++- .../attribute-annotation-sidebar.tsx | 18 +------- .../canvas/views/canvas2d/canvas-wrapper.tsx | 42 +++++++++---------- .../objects-side-bar/object-buttons.tsx | 7 +--- .../objects-side-bar/object-item-details.tsx | 23 +++------- .../objects-side-bar/object-item.tsx | 8 +--- cvat-ui/src/utils/event-recorder.ts | 7 ++++ cvat/apps/events/serializers.py | 11 +++-- .../docs/administration/advanced/analytics.md | 12 +----- 21 files changed, 89 insertions(+), 155 deletions(-) create mode 100644 changelog.d/20240814_151947_boris_update_events.md create mode 100644 changelog.d/20240814_153835_boris_update_events.md diff --git a/changelog.d/20240814_151947_boris_update_events.md b/changelog.d/20240814_151947_boris_update_events.md new file mode 100644 index 00000000000..d65e561e065 --- /dev/null +++ b/changelog.d/20240814_151947_boris_update_events.md @@ -0,0 +1,3 @@ +### Removed + +- Client event `restore:job` () diff --git a/changelog.d/20240814_153835_boris_update_events.md b/changelog.d/20240814_153835_boris_update_events.md new file mode 100644 index 00000000000..605d93f187e --- /dev/null +++ b/changelog.d/20240814_153835_boris_update_events.md @@ -0,0 +1,4 @@ +### Deprecated + +- Client events `upload:annotations`, `lock:object`, `change:attribute`, `change:label` + () diff --git a/cvat-canvas/README.md b/cvat-canvas/README.md index 188b4e9f1d0..6c5115d9433 100644 --- a/cvat-canvas/README.md +++ b/cvat-canvas/README.md @@ -72,9 +72,9 @@ Standard JS events are used. - canvas.reshape - canvas.fit - canvas.regionselected => {points: number[]} - - canvas.dragshape => {id: number} + - canvas.dragshape => {duration: number, state: ObjectState} - canvas.roiselected => {points: number[]} - - canvas.resizeshape => {id: number} + - canvas.resizeshape => {duration: number, state: ObjectState} - canvas.contextmenu => { mouseEvent: MouseEvent, objectState: ObjectState, pointID: number } - canvas.message => { messages: { type: 'text' | 'list'; content: string | string[]; className?: string; icon?: 'info' | 'loading' }[] | null, topic: string } - canvas.error => { exception: Error, domain?: string } diff --git a/cvat-canvas/package.json b/cvat-canvas/package.json index 5dc500eb6ee..2b24ff47e34 100644 --- a/cvat-canvas/package.json +++ b/cvat-canvas/package.json @@ -1,6 +1,6 @@ { "name": "cvat-canvas", - "version": "2.20.8", + "version": "2.20.9", "type": "module", "description": "Part of Computer Vision Annotation Tool which presents its canvas library", "main": "src/canvas.ts", diff --git a/cvat-canvas/src/typescript/canvasView.ts b/cvat-canvas/src/typescript/canvasView.ts index 3e066457144..480a5d3aea5 100644 --- a/cvat-canvas/src/typescript/canvasView.ts +++ b/cvat-canvas/src/typescript/canvasView.ts @@ -1074,6 +1074,7 @@ export class CanvasViewImpl implements CanvasView, Listener { } if (state) { + let start = Date.now(); let aborted = false; let skeletonSVGTemplate: SVG.G = null; shape.addClass('cvat_canvas_shape_draggable'); @@ -1084,6 +1085,7 @@ export class CanvasViewImpl implements CanvasView, Listener { draggableInstance.on('dragstart', (): void => { onDragStart(); this.draggableShape = shape; + start = Date.now(); }).on('dragmove', (e: CustomEvent): void => { onDragMove(); if (state.shapeType === 'skeleton' && e.target) { @@ -1159,7 +1161,8 @@ export class CanvasViewImpl implements CanvasView, Listener { bubbles: false, cancelable: true, detail: { - id: state.clientID, + state, + duration: Date.now() - start, }, }), ); @@ -1243,6 +1246,7 @@ export class CanvasViewImpl implements CanvasView, Listener { if (state) { let resized = false; let aborted = false; + let start = Date.now(); (resizableInstance as any) .resize({ @@ -1252,6 +1256,7 @@ export class CanvasViewImpl implements CanvasView, Listener { .on('resizestart', (): void => { onResizeStart(); resized = false; + start = Date.now(); this.resizableShape = shape; }) .on('resizing', (e: CustomEvent): void => { @@ -1344,7 +1349,8 @@ export class CanvasViewImpl implements CanvasView, Listener { bubbles: false, cancelable: true, detail: { - id: state.clientID, + state, + duration: Date.now() - start, }, }), ); diff --git a/cvat-core/package.json b/cvat-core/package.json index a93411284bc..d9b3a8f9779 100644 --- a/cvat-core/package.json +++ b/cvat-core/package.json @@ -1,6 +1,6 @@ { "name": "cvat-core", - "version": "15.1.1", + "version": "15.1.2", "type": "module", "description": "Part of Computer Vision Tool which presents an interface for client-side integration", "main": "src/api.ts", diff --git a/cvat-core/src/enums.ts b/cvat-core/src/enums.ts index 5543d4712af..1b291662d21 100644 --- a/cvat-core/src/enums.ts +++ b/cvat-core/src/enums.ts @@ -94,11 +94,9 @@ export enum EventScope { loadTool = 'load:cvat', loadJob = 'load:job', + loadWorkspace = 'load:workspace', saveJob = 'save:job', - restoreJob = 'restore:job', - uploadAnnotations = 'upload:annotations', exception = 'send:exception', - sendTaskInfo = 'send:task_info', drawObject = 'draw:object', pasteObject = 'paste:object', @@ -107,14 +105,11 @@ export enum EventScope { dragObject = 'drag:object', resizeObject = 'resize:object', deleteObject = 'delete:object', - lockObject = 'lock:object', mergeObjects = 'merge:objects', splitObjects = 'split:objects', groupObjects = 'group:objects', sliceObject = 'slice:object', joinObjects = 'join:objects', - changeAttribute = 'change:attribute', - changeLabel = 'change:label', changeFrame = 'change:frame', zoomImage = 'zoom:image', diff --git a/cvat-core/src/logger.ts b/cvat-core/src/logger.ts index 876423720fc..0950fa87f9d 100644 --- a/cvat-core/src/logger.ts +++ b/cvat-core/src/logger.ts @@ -16,9 +16,13 @@ function sleep(ms): Promise { } function defaultUpdate(previousEvent: Event, currentPayload: JSONEventPayload): JSONEventPayload { + const count = Number.isInteger(previousEvent.payload.count) ? previousEvent.payload.count as number : 1; + return { ...previousEvent.payload, ...currentPayload, + count: count + 1, + duration: Date.now() - previousEvent.timestamp.getTime(), }; } @@ -29,19 +33,14 @@ interface IgnoreRule { update: (previousEvent: Event, currentPayload: JSONEventPayload) => JSONEventPayload; } -type IgnoredRules = ( - EventScope.zoomImage | EventScope.changeAttribute | - EventScope.changeFrame | EventScope.exception -); - class Logger { public clientID: string; public collection: Array; public lastSentEvent: Event | null; - public ignoreRules: Record; + public ignoreRules: Record; public isActiveChecker: () => boolean; public saving: boolean; - public compressedScopes: Array; + public compressedScopes: Array; constructor() { this.clientID = Date.now().toString().substr(-6); @@ -54,6 +53,8 @@ class Logger { [EventScope.zoomImage]: { lastEvent: null, ignore: (previousEvent: Event): boolean => { + // previous event from the same scope is the latest push event in the collection + // it means, no more events were pushed between the previous and this one const [lastCollectionEvent] = this.collection.slice(-1); return previousEvent === lastCollectionEvent; }, @@ -78,16 +79,6 @@ class Logger { }; }, }, - [EventScope.changeAttribute]: { - lastEvent: null, - ignore(previousEvent: Event, currentPayload: JSONEventPayload): boolean { - return ( - currentPayload.object_id === previousEvent.payload.object_id && - currentPayload.id === previousEvent.payload.id - ); - }, - update: defaultUpdate, - }, [EventScope.changeFrame]: { lastEvent: null, ignore(previousEvent: Event, currentPayload: JSONEventPayload): boolean { @@ -168,7 +159,7 @@ Object.defineProperties(Logger.prototype.log, { } if (scope in this.ignoreRules) { - const ignoreRule = this.ignoreRules[scope as IgnoredRules]; + const ignoreRule = this.ignoreRules[scope]; const { lastEvent } = ignoreRule; if (lastEvent && ignoreRule.ignore(lastEvent, payload)) { lastEvent.payload = ignoreRule.update(lastEvent, payload); @@ -189,7 +180,7 @@ Object.defineProperties(Logger.prototype.log, { this.collection.push(event); if (scope in this.ignoreRules) { - this.ignoreRules[scope as IgnoredRules].lastEvent = event; + this.ignoreRules[scope].lastEvent = event; } }; diff --git a/cvat-ui/package.json b/cvat-ui/package.json index ffbc6f08b7f..89baa2e1dfa 100644 --- a/cvat-ui/package.json +++ b/cvat-ui/package.json @@ -1,6 +1,6 @@ { "name": "cvat-ui", - "version": "1.64.5", + "version": "1.64.6", "description": "CVAT single-page application", "main": "src/index.tsx", "scripts": { diff --git a/cvat-ui/src/actions/annotation-actions.ts b/cvat-ui/src/actions/annotation-actions.ts index 5f49728f11f..6a121a60521 100644 --- a/cvat-ui/src/actions/annotation-actions.ts +++ b/cvat-ui/src/actions/annotation-actions.ts @@ -88,31 +88,6 @@ export function computeZRange(states: any[]): number[] { return [minZ, maxZ]; } -export async function jobInfoGenerator(job: any): Promise> { - const { total } = await job.annotations.statistics(); - return { - 'frame count': job.stopFrame - job.startFrame + 1, - 'track count': - total.rectangle.shape + - total.rectangle.track + - total.polygon.shape + - total.polygon.track + - total.polyline.shape + - total.polyline.track + - total.points.shape + - total.points.track + - total.cuboid.shape + - total.cuboid.track, - 'object count': total.total, - 'box count': total.rectangle.shape + total.rectangle.track, - 'polygon count': total.polygon.shape + total.polygon.track, - 'polyline count': total.polyline.shape + total.polyline.track, - 'points count': total.points.shape + total.points.track, - 'cuboids count': total.cuboid.shape + total.cuboid.track, - 'tag count': total.tag, - }; -} - export enum AnnotationActionTypes { GET_JOB = 'GET_JOB', GET_JOB_SUCCESS = 'GET_JOB_SUCCESS', @@ -835,7 +810,7 @@ export function rotateCurrentFrame(rotation: Rotation): AnyAction { const frameAngle = (frameAngles[frameNumber - startFrame] + (rotation === Rotation.CLOCKWISE90 ? 90 : 270)) % 360; - job.logger.log(EventScope.rotateImage, { angle: frameAngle }); + job.logger.log(EventScope.rotateImage); return { type: AnnotationActionTypes.ROTATE_FRAME, @@ -914,7 +889,7 @@ export function getJobAsync({ throw new Error('Requested resource id is not valid'); } - const loadJobEvent = await logger.log(EventScope.loadJob, {}, true); + const start = Date.now(); getCore().config.globalObjectsCounter = 0; const [job] = await cvat.jobs.get({ jobID }); @@ -962,12 +937,7 @@ export function getJobAsync({ } } - loadJobEvent.close({ - ...await jobInfoGenerator(job), - jobID: job.id, - taskID: job.taskId, - projectID: job.projectId, - }); + await job.logger.log(EventScope.loadJob, { duration: Date.now() - start }); const openTime = Date.now(); dispatch({ diff --git a/cvat-ui/src/actions/boundaries-actions.ts b/cvat-ui/src/actions/boundaries-actions.ts index 097be2ec491..222105f3679 100644 --- a/cvat-ui/src/actions/boundaries-actions.ts +++ b/cvat-ui/src/actions/boundaries-actions.ts @@ -1,5 +1,5 @@ // Copyright (C) 2020-2022 Intel Corporation -// Copyright (C) 2022-2023 CVAT.ai Corporation +// Copyright (C) 2022-2024 CVAT.ai Corporation // // SPDX-License-Identifier: MIT @@ -7,7 +7,6 @@ import { ActionUnion, createAction, ThunkAction, ThunkDispatch, } from 'utils/redux'; import { getCore } from 'cvat-core-wrapper'; -import { EventScope } from 'cvat-logger'; import { fetchAnnotationsAsync } from './annotation-actions'; const cvat = getCore(); @@ -46,8 +45,6 @@ export function resetAfterErrorAsync(): ThunkAction { const frameData = await job.frames.get(frameNumber); const colors = [...cvat.enums.colors]; - await job.logger.log(EventScope.restoreJob); - dispatch(boundariesActions.resetAfterError({ job, states: [], diff --git a/cvat-ui/src/actions/import-actions.ts b/cvat-ui/src/actions/import-actions.ts index 0fef13c22f0..e47db0b4781 100644 --- a/cvat-ui/src/actions/import-actions.ts +++ b/cvat-ui/src/actions/import-actions.ts @@ -8,7 +8,6 @@ import { CombinedState } from 'reducers'; import { getCore, Storage, Job, Task, Project, ProjectOrTaskOrJob, } from 'cvat-core-wrapper'; -import { EventScope } from 'cvat-logger'; import { getProjectsAsync } from './projects-actions'; import { AnnotationActionTypes, fetchAnnotationsAsync } from './annotation-actions'; import { @@ -136,7 +135,6 @@ export const importDatasetAsync = ( if (shouldListenForProgress(rqID, state.requests)) { await listen(rqID, dispatch); - await (instance as Job).logger.log(EventScope.uploadAnnotations); await (instance as Job).annotations.clear({ reload: true }); await (instance as Job).actions.clear(); diff --git a/cvat-ui/src/components/annotation-page/annotation-page.tsx b/cvat-ui/src/components/annotation-page/annotation-page.tsx index 0b954cb282e..37ba4211671 100644 --- a/cvat-ui/src/components/annotation-page/annotation-page.tsx +++ b/cvat-ui/src/components/annotation-page/annotation-page.tsx @@ -25,6 +25,7 @@ import { Workspace } from 'reducers'; import { usePrevious } from 'utils/hooks'; import EventRecorder from 'utils/event-recorder'; import { readLatestFrame } from 'utils/remember-latest-frame'; +import { EventScope } from 'cvat-core/src/enums'; interface Props { job: Job | null | undefined; @@ -40,7 +41,8 @@ interface Props { export default function AnnotationPageComponent(props: Props): JSX.Element { const { - job, fetching, annotationsInitialized, workspace, frameNumber, getJob, closeJob, saveLogs, changeFrame, + job, fetching, annotationsInitialized, workspace, frameNumber, + getJob, closeJob, saveLogs, changeFrame, } = props; const prevJob = usePrevious(job); const prevFetching = usePrevious(fetching); @@ -126,6 +128,12 @@ export default function AnnotationPageComponent(props: Props): JSX.Element { } }, [job, fetching, prevJob, prevFetching]); + useEffect(() => { + if (job) { + job.logger.log(EventScope.loadWorkspace, { obj_name: workspace }); + } + }, [job, workspace]); + if (job === null || !annotationsInitialized) { return ; } diff --git a/cvat-ui/src/components/annotation-page/attribute-annotation-workspace/attribute-annotation-sidebar/attribute-annotation-sidebar.tsx b/cvat-ui/src/components/annotation-page/attribute-annotation-workspace/attribute-annotation-sidebar/attribute-annotation-sidebar.tsx index 18de23974f8..7ea7678ccf8 100644 --- a/cvat-ui/src/components/annotation-page/attribute-annotation-workspace/attribute-annotation-sidebar/attribute-annotation-sidebar.tsx +++ b/cvat-ui/src/components/annotation-page/attribute-annotation-workspace/attribute-annotation-sidebar/attribute-annotation-sidebar.tsx @@ -1,5 +1,5 @@ // Copyright (C) 2020-2022 Intel Corporation -// Copyright (C) 2022-2023 CVAT.ai Corporation +// Copyright (C) 2022-2024 CVAT.ai Corporation // // SPDX-License-Identifier: MIT @@ -11,7 +11,6 @@ import Text from 'antd/lib/typography/Text'; import { filterApplicableLabels } from 'utils/filter-applicable-labels'; import { Label } from 'cvat-core-wrapper'; -import { EventScope } from 'cvat-logger'; import { activateObject as activateObjectAction, changeFrameAsync, @@ -36,7 +35,6 @@ interface StateToProps { activatedAttributeID: number | null; states: any[]; labels: any[]; - jobInstance: any; keyMap: KeyMap; normalizedKeyMap: Record; canvasIsReady: boolean; @@ -121,14 +119,13 @@ function mapStateToProps(state: CombinedState): StateToProps { states, zLayer: { cur }, }, - job: { instance: jobInstance, labels }, + job: { labels }, canvas: { ready: canvasIsReady }, }, shortcuts: { keyMap, normalizedKeyMap }, } = state; return { - jobInstance, labels, activatedStateID, activatedAttributeID, @@ -160,7 +157,6 @@ function AttributeAnnotationSidebar(props: StateToProps & DispatchToProps): JSX. states, activatedStateID, activatedAttributeID, - jobInstance, updateAnnotations, changeFrame, activateObject, @@ -362,11 +358,6 @@ function AttributeAnnotationSidebar(props: StateToProps & DispatchToProps): JSX. currentLabel={activeObjectState.label.id} labels={applicableLabels} changeLabel={(value: Label): void => { - jobInstance.logger.log(EventScope.changeLabel, { - object_id: activeObjectState.clientID, - from: activeObjectState.label.id, - to: value.id, - }); activeObjectState.label = value; updateAnnotations([activeObjectState]); }} @@ -393,11 +384,6 @@ function AttributeAnnotationSidebar(props: StateToProps & DispatchToProps): JSX. currentValue={activeObjectState.attributes[activeAttribute.id]} onChange={(value: string) => { const { attributes } = activeObjectState; - jobInstance.logger.log(EventScope.changeAttribute, { - id: activeAttribute.id, - object_id: activeObjectState.clientID, - value, - }); attributes[activeAttribute.id] = value; activeObjectState.attributes = attributes; updateAnnotations([activeObjectState]); diff --git a/cvat-ui/src/components/annotation-page/canvas/views/canvas2d/canvas-wrapper.tsx b/cvat-ui/src/components/annotation-page/canvas/views/canvas2d/canvas-wrapper.tsx index c173fba1017..f80ccbff335 100644 --- a/cvat-ui/src/components/annotation-page/canvas/views/canvas2d/canvas-wrapper.tsx +++ b/cvat-ui/src/components/annotation-page/canvas/views/canvas2d/canvas-wrapper.tsx @@ -603,8 +603,8 @@ class CanvasWrapperComponent extends React.PureComponent { canvasInstance.html().removeEventListener('canvas.zoom', this.onCanvasZoomChanged); canvasInstance.html().removeEventListener('canvas.fit', this.onCanvasImageFitted); - canvasInstance.html().removeEventListener('canvas.dragshape', this.onCanvasShapeDragged); - canvasInstance.html().removeEventListener('canvas.resizeshape', this.onCanvasShapeResized); + canvasInstance.html().removeEventListener('canvas.dragshape', this.onCanvasShapeDragged as EventListener); + canvasInstance.html().removeEventListener('canvas.resizeshape', this.onCanvasShapeResized as EventListener); canvasInstance.html().removeEventListener('canvas.clicked', this.onCanvasShapeClicked); canvasInstance.html().removeEventListener('canvas.drawn', this.onCanvasShapeDrawn); canvasInstance.html().removeEventListener('canvas.merged', this.onCanvasObjectsMerged); @@ -663,20 +663,10 @@ class CanvasWrapperComponent extends React.PureComponent { }); } - const payload = { - object_type: state.objectType, - label: state.label.name, - frame: state.frame, - rotation: state.rotation, - occluded: state.occluded, - outside: state.outside, - shape_type: state.shapeType, - }; - if (isDrawnFromScratch) { - jobInstance.logger.log(EventScope.drawObject, { count: 1, duration, ...payload }); + jobInstance.logger.log(EventScope.drawObject, { count: 1, duration }); } else { - jobInstance.logger.log(EventScope.pasteObject, { count: 1, duration, ...payload }); + jobInstance.logger.log(EventScope.pasteObject, { count: 1, duration }); } const objectState = new cvat.classes.ObjectState(state); @@ -762,16 +752,23 @@ class CanvasWrapperComponent extends React.PureComponent { } }; - private onCanvasShapeDragged = (e: any): void => { + private onCanvasShapeDragged = (e: CustomEvent<{ duration: number; state: ObjectState }>): void => { const { jobInstance } = this.props; - const { id } = e.detail; - jobInstance.logger.log(EventScope.dragObject, { id }); + const { detail: { duration, state: { serverID } } } = e; + jobInstance.logger.log( + EventScope.dragObject, + { duration, ...(serverID ? { obj_id: serverID } : {}) }, + ); }; - private onCanvasShapeResized = (e: any): void => { + private onCanvasShapeResized = (e: CustomEvent<{ duration: number; state: ObjectState }>): void => { const { jobInstance } = this.props; - const { id } = e.detail; - jobInstance.logger.log(EventScope.resizeObject, { id }); + const { detail: { duration, state: { serverID } } } = e; + + jobInstance.logger.log( + EventScope.resizeObject, + { duration, ...(serverID ? { obj_id: serverID } : {}) }, + ); }; private onCanvasImageFitted = (): void => { @@ -862,7 +859,6 @@ class CanvasWrapperComponent extends React.PureComponent { jobInstance.logger.log(EventScope.sliceObject, { count: 1, duration, - clientID: state.clientID, }); onSliceAnnotations(state, results); }; @@ -1064,8 +1060,8 @@ class CanvasWrapperComponent extends React.PureComponent { canvasInstance.html().addEventListener('canvas.zoom', this.onCanvasZoomChanged); canvasInstance.html().addEventListener('canvas.fit', this.onCanvasImageFitted); - canvasInstance.html().addEventListener('canvas.dragshape', this.onCanvasShapeDragged); - canvasInstance.html().addEventListener('canvas.resizeshape', this.onCanvasShapeResized); + canvasInstance.html().addEventListener('canvas.dragshape', this.onCanvasShapeDragged as EventListener); + canvasInstance.html().addEventListener('canvas.resizeshape', this.onCanvasShapeResized as EventListener); canvasInstance.html().addEventListener('canvas.clicked', this.onCanvasShapeClicked); canvasInstance.html().addEventListener('canvas.drawn', this.onCanvasShapeDrawn); canvasInstance.html().addEventListener('canvas.merged', this.onCanvasObjectsMerged); diff --git a/cvat-ui/src/containers/annotation-page/standard-workspace/objects-side-bar/object-buttons.tsx b/cvat-ui/src/containers/annotation-page/standard-workspace/objects-side-bar/object-buttons.tsx index 3c426656e07..7656be38487 100644 --- a/cvat-ui/src/containers/annotation-page/standard-workspace/objects-side-bar/object-buttons.tsx +++ b/cvat-ui/src/containers/annotation-page/standard-workspace/objects-side-bar/object-buttons.tsx @@ -7,7 +7,6 @@ import React from 'react'; import { connect } from 'react-redux'; import { ObjectState, Job } from 'cvat-core-wrapper'; -import { EventScope } from 'cvat-logger'; import isAbleToChangeFrame from 'utils/is-able-to-change-frame'; import { ThunkDispatch } from 'utils/redux'; import { updateAnnotationsAsync, changeFrameAsync } from 'actions/annotation-actions'; @@ -114,18 +113,16 @@ class ItemButtonsWrapper extends React.PureComponent { - const { objectState, jobInstance, readonly } = this.props; + const { objectState, readonly } = this.props; if (!readonly) { - jobInstance.logger.log(EventScope.lockObject, { locked: true }); objectState.lock = true; this.commit(); } }; private unlock = (): void => { - const { objectState, jobInstance, readonly } = this.props; + const { objectState, readonly } = this.props; if (!readonly) { - jobInstance.logger.log(EventScope.lockObject, { locked: false }); objectState.lock = false; this.commit(); } diff --git a/cvat-ui/src/containers/annotation-page/standard-workspace/objects-side-bar/object-item-details.tsx b/cvat-ui/src/containers/annotation-page/standard-workspace/objects-side-bar/object-item-details.tsx index a9c05460e39..8399130b502 100644 --- a/cvat-ui/src/containers/annotation-page/standard-workspace/objects-side-bar/object-item-details.tsx +++ b/cvat-ui/src/containers/annotation-page/standard-workspace/objects-side-bar/object-item-details.tsx @@ -1,15 +1,14 @@ -// Copyright (C) 2021-2022 CVAT.ai Corporation +// Copyright (C) 2021-2024 CVAT.ai Corporation // // SPDX-License-Identifier: MIT -import React, { Dispatch } from 'react'; +import React from 'react'; import { ObjectState } from 'cvat-core-wrapper'; import { CombinedState } from 'reducers'; import ObjectItemDetails from 'components/annotation-page/standard-workspace/objects-side-bar/object-item-details'; -import { AnyAction } from 'redux'; import { updateAnnotationsAsync, collapseObjectItems } from 'actions/annotation-actions'; -import { EventScope } from 'cvat-logger'; import { connect } from 'react-redux'; +import { ThunkDispatch } from 'utils/redux'; interface OwnProps { readonly: boolean; @@ -20,7 +19,6 @@ interface OwnProps { interface StateToProps { collapsed: boolean; state: ObjectState | null; - jobInstance: any; } interface DispatchToProps { @@ -48,9 +46,6 @@ function mapStateToProps(state: CombinedState, own: OwnProps): StateToProps { collapsedAll, collapsed: statesCollapsed, }, - job: { - instance: jobInstance, - }, }, } = state; @@ -59,11 +54,10 @@ function mapStateToProps(state: CombinedState, own: OwnProps): StateToProps { return { collapsed, state: objectState, - jobInstance, }; } -function mapDispatchToProps(dispatch: Dispatch): DispatchToProps { +function mapDispatchToProps(dispatch: ThunkDispatch): DispatchToProps { return { updateState(state: ObjectState): void { dispatch(updateAnnotationsAsync([state])); @@ -77,15 +71,8 @@ function mapDispatchToProps(dispatch: Dispatch): DispatchToProps { type Props = StateToProps & DispatchToProps & OwnProps; class ObjectItemDetailsContainer extends React.PureComponent { private changeAttribute = (id: number, value: string): void => { - const { - state, readonly, jobInstance, updateState, - } = this.props; + const { state, readonly, updateState } = this.props; if (!readonly && state) { - jobInstance.logger.log(EventScope.changeAttribute, { - id, - value, - object_id: state.clientID, - }); const attr: Record = {}; attr[id] = value; state.attributes = attr; diff --git a/cvat-ui/src/containers/annotation-page/standard-workspace/objects-side-bar/object-item.tsx b/cvat-ui/src/containers/annotation-page/standard-workspace/objects-side-bar/object-item.tsx index d11fb177067..9cbb75bd75f 100644 --- a/cvat-ui/src/containers/annotation-page/standard-workspace/objects-side-bar/object-item.tsx +++ b/cvat-ui/src/containers/annotation-page/standard-workspace/objects-side-bar/object-item.tsx @@ -28,7 +28,6 @@ import { Label, ObjectState, Attribute, Job, } from 'cvat-core-wrapper'; import { Canvas, CanvasMode } from 'cvat-canvas-wrapper'; -import { EventScope } from 'cvat-logger'; import { Canvas3d } from 'cvat-canvas3d-wrapper'; import { filterApplicableLabels } from 'utils/filter-applicable-labels'; @@ -321,13 +320,8 @@ class ObjectItemContainer extends React.PureComponent { }; private changeLabel = (label: any): void => { - const { jobInstance, objectState, readonly } = this.props; + const { objectState, readonly } = this.props; if (!readonly) { - jobInstance.logger.log(EventScope.changeLabel, { - object_id: objectState.clientID, - from: objectState.label.id, - to: label.id, - }); objectState.label = label; this.commit(); } diff --git a/cvat-ui/src/utils/event-recorder.ts b/cvat-ui/src/utils/event-recorder.ts index a61e6943ff7..9308b5a02a7 100644 --- a/cvat-ui/src/utils/event-recorder.ts +++ b/cvat-ui/src/utils/event-recorder.ts @@ -5,6 +5,7 @@ import { ServerError, getCore } from 'cvat-core-wrapper'; import { EventScope } from 'cvat-logger'; import config from 'config'; +import _ from 'lodash'; import { platformInfo } from 'utils/platform-checker'; const core = getCore(); @@ -28,6 +29,12 @@ class EventRecorder { core.logger.log(EventScope.loadTool, { location: window.location.pathname, platform: platformInfo(), + screenWidth: window.screen.width, + screenHeight: window.screen.height, + language: window.navigator.language, + hardwareConcurrency: _.get(window.navigator, 'hardwareConcurrency', null), + deviceMemory: _.get(window.navigator, 'deviceMemory', null), + jsHeapSizeLimit: _.get(window.performance, 'memory', { jsHeapSizeLimit: null }).jsHeapSizeLimit, }); } diff --git a/cvat/apps/events/serializers.py b/cvat/apps/events/serializers.py index f9734c2613a..05d89d41ef1 100644 --- a/cvat/apps/events/serializers.py +++ b/cvat/apps/events/serializers.py @@ -29,12 +29,15 @@ class EventSerializer(serializers.Serializer): class ClientEventsSerializer(serializers.Serializer): ALLOWED_SCOPES = frozenset(( - 'load:cvat', 'load:job', 'save:job', 'restore:job', - 'upload:annotations', 'send:exception', 'send:task_info', + 'load:cvat', 'load:job', 'save:job','load:workspace', + 'upload:annotations', # TODO: remove in next releases + 'lock:object', # TODO: remove in next releases + 'change:attribute', # TODO: remove in next releases + 'change:label', # TODO: remove in next releases + 'send:exception', 'join:objects', 'change:frame', 'draw:object', 'paste:object', 'copy:object', 'propagate:object', - 'drag:object', 'resize:object', 'delete:object', 'lock:object', + 'drag:object', 'resize:object', 'delete:object', 'merge:objects', 'split:objects', 'group:objects', 'slice:object', - 'join:objects', 'change:attribute', 'change:label', 'change:frame', 'zoom:image', 'fit:image', 'rotate:image', 'action:undo', 'action:redo', 'debug:info', 'run:annotations_action', 'click:element' )) diff --git a/site/content/en/docs/administration/advanced/analytics.md b/site/content/en/docs/administration/advanced/analytics.md index 1d7f014583b..c69d18db2b5 100644 --- a/site/content/en/docs/administration/advanced/analytics.md +++ b/site/content/en/docs/administration/advanced/analytics.md @@ -133,21 +133,13 @@ Client events: - `load:cvat` -- `load:job`, `save:job`, `restore:job` - -- `upload:annotations` +- `load:job`, `save:job` - `send:exception` -- `send:task_info` - -- `draw:object`, `paste:object`, `copy:object`, `propagate:object`, `drag:object`, `resize:object`, `delete:object`, `lock:object`, `merge:objects`, `split:objects`, `group:objects`, `slice:object`, +- `draw:object`, `paste:object`, `copy:object`, `propagate:object`, `drag:object`, `resize:object`, `delete:object`, `merge:objects`, `split:objects`, `group:objects`, `slice:object`, `join:objects` -- `change:attribute` - -- `change:label` - - `change:frame` - `zoom:image`, `fit:image`, `rotate:image` From afbb1433af0fb2496498416a91612f7aa7bc4981 Mon Sep 17 00:00:00 2001 From: Kirill Lakhov Date: Mon, 19 Aug 2024 12:02:50 +0300 Subject: [PATCH 08/22] Fixed "Back" button redirection (#8277) ### Motivation and context Resolved #8257 The problem is in hashing system we use to save the opened tab. It clutters the history and we cant really go back using it. There are two ways to improve that. We eighter save the actual link to go back somewhere in our application or pass it as a state when moving to analytics page `history.push(/analytics, { from: somewhere})`. From my perspective the first way is more elegant TODO: - [x] Analytics page - [x] Check Guide page ### How has this been tested? ### Checklist - [x] I submit my changes into the `develop` branch - [x] I have created a changelog fragment - ~~[ ] I have updated the documentation accordingly~~ - ~~[ ] I have added tests to cover my changes~~ - [x] I have linked related issues (see [GitHub docs]( https://help.github.com/en/github/managing-your-work-on-github/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword)) - [x] I have increased versions of npm packages if it is necessary ([cvat-canvas](https://github.com/cvat-ai/cvat/tree/develop/cvat-canvas#versioning), [cvat-core](https://github.com/cvat-ai/cvat/tree/develop/cvat-core#versioning), [cvat-data](https://github.com/cvat-ai/cvat/tree/develop/cvat-data#versioning) and [cvat-ui](https://github.com/cvat-ai/cvat/tree/develop/cvat-ui#versioning)) ### License - [x] I submit _my code changes_ under the same [MIT License]( https://github.com/cvat-ai/cvat/blob/develop/LICENSE) that covers the project. Feel free to contact the maintainers if that's a concern. ## Summary by CodeRabbit - **New Features** - Enhanced navigation functionality with a configurable back link for the AnalyticsPage component. - Introduced a dynamic back navigation experience across analytics routes. - Improved the GoBackButton component to accept custom back links. - **Bug Fixes** - Fixed navigation flow issues by ensuring the previous pathname is accurately stored and used. - **Documentation** - Updated documentation to reflect changes in component props and improved navigation logic. --- .../20240813_115132_klakhov_fix_go_back.md | 4 ++ cvat-ui/src/actions/navigation-actions.ts | 17 ++++++++ cvat-ui/src/components/cvat-app.tsx | 6 ++- cvat-ui/src/config.tsx | 5 +++ cvat-ui/src/index.tsx | 3 ++ cvat-ui/src/reducers/index.ts | 5 +++ cvat-ui/src/reducers/navigation-reducer.ts | 43 +++++++++++++++++++ cvat-ui/src/reducers/root-reducer.ts | 2 + cvat-ui/src/utils/hooks.ts | 11 ++--- 9 files changed, 88 insertions(+), 8 deletions(-) create mode 100644 changelog.d/20240813_115132_klakhov_fix_go_back.md create mode 100644 cvat-ui/src/actions/navigation-actions.ts create mode 100644 cvat-ui/src/reducers/navigation-reducer.ts diff --git a/changelog.d/20240813_115132_klakhov_fix_go_back.md b/changelog.d/20240813_115132_klakhov_fix_go_back.md new file mode 100644 index 00000000000..6eb6cb9b945 --- /dev/null +++ b/changelog.d/20240813_115132_klakhov_fix_go_back.md @@ -0,0 +1,4 @@ +### Fixed + +- Go back button behavior on analytics page + () diff --git a/cvat-ui/src/actions/navigation-actions.ts b/cvat-ui/src/actions/navigation-actions.ts new file mode 100644 index 00000000000..d0135652cf5 --- /dev/null +++ b/cvat-ui/src/actions/navigation-actions.ts @@ -0,0 +1,17 @@ +// Copyright (C) 2024 CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import { ActionUnion, createAction } from 'utils/redux'; + +export enum NavigationActionTypes { + CHANGE_LOCATION = 'CHANGE_LOCATION', +} + +export const navigationActions = { + changeLocation: (from: string, to: string) => createAction( + NavigationActionTypes.CHANGE_LOCATION, { from, to }, + ), +}; + +export type NavigationActions = ActionUnion; diff --git a/cvat-ui/src/components/cvat-app.tsx b/cvat-ui/src/components/cvat-app.tsx index ab827c7be66..c9ebea4d12e 100644 --- a/cvat-ui/src/components/cvat-app.tsx +++ b/cvat-ui/src/components/cvat-app.tsx @@ -97,6 +97,7 @@ interface CVATAppProps { initInvitations: () => void; initRequests: () => void; loadServerAPISchema: () => void; + onChangeLocation: (from: string, to: string) => void; userInitialized: boolean; userFetching: boolean; organizationFetching: boolean; @@ -141,7 +142,7 @@ class CVATApplication extends React.PureComponent { customWaViewHit(newLocation.pathname, newLocation.search, newLocation.hash); const { location: prevLocation } = this.props; + + onChangeLocation(prevLocation.pathname, newLocation.pathname); + const shouldResetNotifications = RESET_NOTIFICATIONS_PATHS.from.some( (pathname) => prevLocation.pathname === pathname, ); diff --git a/cvat-ui/src/config.tsx b/cvat-ui/src/config.tsx index c79e72f309b..9719a0bc292 100644 --- a/cvat-ui/src/config.tsx +++ b/cvat-ui/src/config.tsx @@ -134,6 +134,10 @@ const LOCAL_STORAGE_LAST_FRAME_MEMORY_LIMIT = 20; const REQUEST_SUCCESS_NOTIFICATION_DURATION = 5; // seconds +const BLACKLISTED_GO_BACK_PATHS = [ + /\/auth.+/, +]; + export default { UNDEFINED_ATTRIBUTE_VALUE, NO_BREAK_SPACE, @@ -174,4 +178,5 @@ export default { LOCAL_STORAGE_SEEN_GUIDES_MEMORY_LIMIT, LOCAL_STORAGE_LAST_FRAME_MEMORY_LIMIT, REQUEST_SUCCESS_NOTIFICATION_DURATION, + BLACKLISTED_GO_BACK_PATHS, }; diff --git a/cvat-ui/src/index.tsx b/cvat-ui/src/index.tsx index fbd719816df..a838eef9cba 100644 --- a/cvat-ui/src/index.tsx +++ b/cvat-ui/src/index.tsx @@ -25,6 +25,7 @@ import { resetErrors, resetMessages } from 'actions/notification-actions'; import { getInvitationsAsync } from 'actions/invitations-actions'; import { getRequestsAsync } from 'actions/requests-async-actions'; import { getServerAPISchemaAsync } from 'actions/server-actions'; +import { navigationActions } from 'actions/navigation-actions'; import { CombinedState, NotificationsState, PluginsState } from './reducers'; createCVATStore(createRootReducer); @@ -73,6 +74,7 @@ interface DispatchToProps { initInvitations: () => void; initRequests: () => void; loadServerAPISchema: () => void; + onChangeLocation: (from: string, to: string) => void; } function mapStateToProps(state: CombinedState): StateToProps { @@ -124,6 +126,7 @@ function mapDispatchToProps(dispatch: any): DispatchToProps { initInvitations: (): void => dispatch(getInvitationsAsync({ page: 1 }, true)), initRequests: (): void => dispatch(getRequestsAsync({ page: 1 })), loadServerAPISchema: (): void => dispatch(getServerAPISchemaAsync()), + onChangeLocation: (from: string, to: string): void => dispatch(navigationActions.changeLocation(from, to)), }; } diff --git a/cvat-ui/src/reducers/index.ts b/cvat-ui/src/reducers/index.ts index 821e3a4f233..0a8041c526f 100644 --- a/cvat-ui/src/reducers/index.ts +++ b/cvat-ui/src/reducers/index.ts @@ -966,6 +966,10 @@ export interface RequestsState { query: RequestsQuery; } +export interface NavigationState { + prevLocation: string | null; +} + export interface CombinedState { auth: AuthState; projects: ProjectsState; @@ -989,6 +993,7 @@ export interface CombinedState { webhooks: WebhooksState; requests: RequestsState; serverAPI: ServerAPIState; + navigation: NavigationState; } export interface Indexable { diff --git a/cvat-ui/src/reducers/navigation-reducer.ts b/cvat-ui/src/reducers/navigation-reducer.ts new file mode 100644 index 00000000000..b7f82d23fc9 --- /dev/null +++ b/cvat-ui/src/reducers/navigation-reducer.ts @@ -0,0 +1,43 @@ +// Copyright (C) 2024 CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import { NavigationActions, NavigationActionTypes } from 'actions/navigation-actions'; +import config from 'config'; +import { NavigationState } from '.'; + +const defaultState: NavigationState = { + prevLocation: null, +}; + +export default function ( + state: NavigationState = defaultState, + action:NavigationActions, +): NavigationState { + switch (action.type) { + case NavigationActionTypes.CHANGE_LOCATION: { + const { BLACKLISTED_GO_BACK_PATHS } = config; + const { from, to } = action.payload; + + if (from === to) { + return state; + } + + for (const path of BLACKLISTED_GO_BACK_PATHS) { + if (path.test(from)) { + return { + ...state, + prevLocation: null, + }; + } + } + + return { + ...state, + prevLocation: from, + }; + } + default: + return state; + } +} diff --git a/cvat-ui/src/reducers/root-reducer.ts b/cvat-ui/src/reducers/root-reducer.ts index 91c300c9d14..13429f80d59 100644 --- a/cvat-ui/src/reducers/root-reducer.ts +++ b/cvat-ui/src/reducers/root-reducer.ts @@ -26,6 +26,7 @@ import webhooksReducer from './webhooks-reducer'; import invitationsReducer from './invitations-reducer'; import requestsReducer from './requests-reducer'; import serverAPIReducer from './server-api-reducer'; +import navigationReducer from './navigation-reducer'; export default function createRootReducer(): Reducer { return combineReducers({ @@ -51,5 +52,6 @@ export default function createRootReducer(): Reducer { invitations: invitationsReducer, requests: requestsReducer, serverAPI: serverAPIReducer, + navigation: navigationReducer, }); } diff --git a/cvat-ui/src/utils/hooks.ts b/cvat-ui/src/utils/hooks.ts index 1d3988f3172..c51f681f39f 100644 --- a/cvat-ui/src/utils/hooks.ts +++ b/cvat-ui/src/utils/hooks.ts @@ -1,5 +1,5 @@ // Copyright (C) 2021-2022 Intel Corporation -// Copyright (C) 2023 CVAT.ai Corporation +// Copyright (C) 2023-2024 CVAT.ai Corporation // // SPDX-License-Identifier: MIT @@ -64,13 +64,10 @@ export function usePlugins( export function useGoBack(): () => void { const history = useHistory(); + const prevLocation = useSelector((state: CombinedState) => state.navigation.prevLocation) ?? '/tasks'; const goBack = useCallback(() => { - if (history.action !== 'POP') { - history.goBack(); - } else { - history.push('/'); - } - }, []); + history.push(prevLocation); + }, [prevLocation]); return goBack; } From 884ffa962e8fb6d60e90ae4f9871bcdd3481a26c Mon Sep 17 00:00:00 2001 From: Kirill Lakhov Date: Tue, 20 Aug 2024 12:53:05 +0300 Subject: [PATCH 09/22] Added tests for `Requests` page (#8287) ### Motivation and context Tests for #8095 And for the problem with sumultaneous job exports crush ### How has this been tested? ### Checklist - [x] I submit my changes into the `develop` branch - ~~[ ] I have created a changelog fragment ~~ - ~~[ ] I have updated the documentation accordingly~~ - [x] I have added tests to cover my changes - ~~[ ] I have linked related issues (see [GitHub docs]( https://help.github.com/en/github/managing-your-work-on-github/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword))~ - ~~[ ] I have increased versions of npm packages if it is necessary ([cvat-canvas](https://github.com/cvat-ai/cvat/tree/develop/cvat-canvas#versioning), [cvat-core](https://github.com/cvat-ai/cvat/tree/develop/cvat-core#versioning), [cvat-data](https://github.com/cvat-ai/cvat/tree/develop/cvat-data#versioning) and [cvat-ui](https://github.com/cvat-ai/cvat/tree/develop/cvat-ui#versioning))~~ ### License - [x] I submit _my code changes_ under the same [MIT License]( https://github.com/cvat-ai/cvat/blob/develop/LICENSE) that covers the project. Feel free to contact the maintainers if that's a concern. ## Summary by CodeRabbit - **New Features** - Introduced an extensive suite of automated end-to-end tests for the Requests page, enhancing verification of task creation, requests handling, and export processes. - Updated the `downloadExport` command to accept a configuration object, allowing users to control notification verification during tests. - **Bug Fixes** - Improved error handling for task creation with invalid configurations, ensuring proper feedback is provided to users. - **Tests** - Expanded testing coverage to ensure the robustness and reliability of the Requests page functionalities. --- .../components/requests-page/request-card.tsx | 12 +- .../requests-page/request-status.tsx | 31 +- ...t_storage_for_import_export_annotations.js | 12 +- ...k_storage_for_import_export_annotations.js | 12 +- ...m_storage_for_import_export_annotations.js | 14 +- tests/cypress/e2e/features/requests_page.js | 356 ++++++++++++++++++ ...load_annotations_different_file_formats.js | 10 +- tests/cypress/support/commands.js | 48 ++- 8 files changed, 440 insertions(+), 55 deletions(-) create mode 100644 tests/cypress/e2e/features/requests_page.js diff --git a/cvat-ui/src/components/requests-page/request-card.tsx b/cvat-ui/src/components/requests-page/request-card.tsx index 69b09e89d07..980af114563 100644 --- a/cvat-ui/src/components/requests-page/request-card.tsx +++ b/cvat-ui/src/components/requests-page/request-card.tsx @@ -196,11 +196,13 @@ function RequestCard(props: Props): JSX.Element { {' '} - - {linkToEntity ? - ({name}) : - {name}} - + {name && ( + + {linkToEntity ? + ({name}) : + {name}} + + )} {timestamps} diff --git a/cvat-ui/src/components/requests-page/request-status.tsx b/cvat-ui/src/components/requests-page/request-status.tsx index 5a8ba8b9d2a..55f927c6f72 100644 --- a/cvat-ui/src/components/requests-page/request-status.tsx +++ b/cvat-ui/src/components/requests-page/request-status.tsx @@ -4,6 +4,7 @@ import React from 'react'; import Text from 'antd/lib/typography/Text'; +import { BaseType } from 'antd/es/typography/Base'; import LoadingOutlined from '@ant-design/icons/lib/icons/LoadingOutlined'; import { RQStatus } from 'cvat-core-wrapper'; @@ -34,19 +35,27 @@ function StatusMessage(props: Props): JSX.Element { let { status, message } = props; message = message || ''; status = status || RQStatus.FINISHED; - let textType: 'success' | 'danger' | 'warning' | undefined; - - if ([RQStatus.FAILED, RQStatus.UNKNOWN].includes(status)) { - textType = 'danger'; - } else if ([RQStatus.QUEUED].includes(status)) { - textType = 'warning'; - } else if ([RQStatus.FINISHED].includes(status)) { - textType = 'success'; - } + + const [textType, classHelper] = ((_status: RQStatus) => { + if (_status === RQStatus.FINISHED) { + return ['success', 'success']; + } + + if (_status === RQStatus.QUEUED) { + return ['warning', 'queued']; + } + + if (_status === RQStatus.STARTED) { + return [undefined, 'started']; + } + + return ['danger', 'failed']; + })(status); + return ( {((): JSX.Element => { diff --git a/tests/cypress/e2e/actions_tasks3/case_113_use_default_project_storage_for_import_export_annotations.js b/tests/cypress/e2e/actions_tasks3/case_113_use_default_project_storage_for_import_export_annotations.js index 15ec3e1c3d1..7d16e85da68 100644 --- a/tests/cypress/e2e/actions_tasks3/case_113_use_default_project_storage_for_import_export_annotations.js +++ b/tests/cypress/e2e/actions_tasks3/case_113_use_default_project_storage_for_import_export_annotations.js @@ -165,12 +165,12 @@ context('Tests for source and target storage.', () => { cy.interactMenu('Upload annotations'); cy.intercept('GET', '/api/jobs/**/annotations?**').as('uploadAnnotationsGet'); - cy.uploadAnnotations( - format.split(' ')[0], - 'job_annotations.zip', - '.cvat-modal-content-load-job-annotation', - project.advancedConfiguration.sourceStorage, - ); + cy.uploadAnnotations({ + format: format.split(' ')[0], + filePath: 'job_annotations.zip', + confirmModalClassName: '.cvat-modal-content-load-job-annotation', + sourceStorage: project.advancedConfiguration.sourceStorage, + }); cy.get('.cvat-notification-notice-upload-annotations-fail').should('not.exist'); cy.get('#cvat_canvas_shape_1').should('exist'); diff --git a/tests/cypress/e2e/actions_tasks3/case_114_use_default_task_storage_for_import_export_annotations.js b/tests/cypress/e2e/actions_tasks3/case_114_use_default_task_storage_for_import_export_annotations.js index 8ebded744f2..3462ffb9b3e 100644 --- a/tests/cypress/e2e/actions_tasks3/case_114_use_default_task_storage_for_import_export_annotations.js +++ b/tests/cypress/e2e/actions_tasks3/case_114_use_default_task_storage_for_import_export_annotations.js @@ -174,12 +174,12 @@ context('Tests for source and target storage.', () => { // upload annotations from default local storage cy.interactMenu('Upload annotations'); cy.intercept('GET', '/api/jobs/**/annotations?**').as('uploadAnnotationsGet'); - cy.uploadAnnotations( - format.split(' ')[0], - annotationsArchiveName, - '.cvat-modal-content-load-job-annotation', - task.advancedConfiguration.sourceStorage, - ); + cy.uploadAnnotations({ + format: format.split(' ')[0], + filePath: 'job_annotations.zip', + confirmModalClassName: '.cvat-modal-content-load-job-annotation', + sourceStorage: task.advancedConfiguration.sourceStorage, + }); cy.get('.cvat-notification-notice-upload-annotations-fail').should('not.exist'); cy.get('#cvat_canvas_shape_1').should('exist'); diff --git a/tests/cypress/e2e/actions_tasks3/case_115_use_custom_storage_for_import_export_annotations.js b/tests/cypress/e2e/actions_tasks3/case_115_use_custom_storage_for_import_export_annotations.js index ac549171b63..ff1b4960aef 100644 --- a/tests/cypress/e2e/actions_tasks3/case_115_use_custom_storage_for_import_export_annotations.js +++ b/tests/cypress/e2e/actions_tasks3/case_115_use_custom_storage_for_import_export_annotations.js @@ -150,16 +150,16 @@ context('Import and export annotations: specify source and target storage in mod it('Import job annotations from custom minio "public" bucket', () => { cy.interactMenu('Upload annotations'); cy.intercept('GET', '/api/jobs/**/annotations?**').as('uploadAnnotationsGet'); - cy.uploadAnnotations( - format.split(' ')[0], - 'job_annotations.zip', - '.cvat-modal-content-load-job-annotation', - { + cy.uploadAnnotations({ + format: format.split(' ')[0], + filePath: 'job_annotations.zip', + confirmModalClassName: '.cvat-modal-content-load-job-annotation', + sourceStorage: { location: 'Cloud storage', cloudStorageId: createdCloudStorageId, }, - false, - ); + useDefaultLocation: false, + }); cy.get('.cvat-notification-notice-upload-annotations-fail').should('not.exist'); cy.get('#cvat_canvas_shape_1').should('exist'); cy.get('#cvat-objects-sidebar-state-item-1').should('exist'); diff --git a/tests/cypress/e2e/features/requests_page.js b/tests/cypress/e2e/features/requests_page.js new file mode 100644 index 00000000000..8ce9cbd901e --- /dev/null +++ b/tests/cypress/e2e/features/requests_page.js @@ -0,0 +1,356 @@ +// Copyright (C) 2024 CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +/// + +context('Requests page', () => { + const projectName = 'Project for testing requests page'; + const mainLabelName = 'requests_page_label'; + + const cloudStorageData = { + displayName: 'Demo bucket', + resource: 'public', + manifest: 'manifest.jsonl', + endpointUrl: Cypress.config('minioUrl'), + }; + const rectanglePayload = { + frame: 0, + objectType: 'shape', + shapeType: 'rectangle', + points: [250, 64, 491, 228], + occluded: false, + labelName: mainLabelName, + }; + const attrName = 'requests_attr'; + const imagesCount = 3; + const imageFileName = `image_${mainLabelName}`; + const width = 800; + const height = 800; + const posX = 10; + const posY = 10; + const color = 'gray'; + const archiveName = `${imageFileName}.zip`; + const archivePath = `cypress/fixtures/${archiveName}`; + const brokenArchiveName = `${imageFileName}_empty.zip`; + const brokenArchivePath = `cypress/fixtures/${brokenArchiveName}`; + const badAnnotationsName = `${imageFileName}_incorrect.xml`; + const badAnnotationsPath = `cypress/fixtures/${badAnnotationsName}`; + const imagesFolder = `cypress/fixtures/${imageFileName}`; + const directoryToArchive = imagesFolder; + const emptyDirectoryToArchive = `${imagesFolder}_empty`; + const annotationsArchiveNameLocal = 'requests_annotations_archive_local'; + const annotationsArchiveNameCloud = 'requests_annotations_archive_cloud'; + const backupArchiveName = 'requests_backup'; + const exportFormat = 'CVAT for images'; + let exportFileName; + + const taskName = 'Annotation task for testing requests page'; + const brokenTaskName = 'Broken Annotation task for testing requests page'; + + const data = { + projectID: null, + taskID: null, + cloudStorageID: null, + }; + + function checkRequestStatus( + selector, + innerCheck, + { + checkResourceLink, resourceLink, + } = { checkResourceLink: true, resourceLink: `/tasks/${data.taskID}` }, + ) { + cy.contains('.cvat-header-button', 'Requests').click(); + cy.get(selector ? `.cvat-requests-card:contains("${selector}")` : '.cvat-requests-card') + .first() + .within(() => { + innerCheck(); + if (checkResourceLink) { + cy.get('.cvat-requests-name').click(); + } + }); + + if (checkResourceLink) { + cy.get('.cvat-spinner').should('not.exist'); + + if (resourceLink) { + cy.url().should('include', resourceLink); + } else { + cy.url().should('include', '/requests'); + } + } + } + + function openTask() { + cy.contains('.cvat-header-button', 'Tasks').should('be.visible').click(); + cy.url().should('include', '/tasks'); + cy.openTask(taskName); + } + + before(() => { + cy.visit('/auth/login'); + cy.login(); + + cy.imageGenerator(imagesFolder, imageFileName, width, height, color, posX, posY, mainLabelName, imagesCount); + cy.createZipArchive(directoryToArchive, badAnnotationsPath); + cy.createZipArchive(directoryToArchive, archivePath); + cy.createZipArchive(emptyDirectoryToArchive, brokenArchivePath); + + data.cloudStorageID = cy.attachS3Bucket(cloudStorageData); + + const defaultAttrValue = 'Requests attr'; + const multiAttrParams = false; + const advancedConfigurationParams = false; + cy.goToProjectsList(); + cy.createProjects( + projectName, + mainLabelName, + attrName, + defaultAttrValue, + multiAttrParams, + advancedConfigurationParams, + ); + cy.openProject(projectName); + cy.url().then((url) => { + data.projectID = Number(url.split('/').slice(-1)[0].split('?')[0]); + + const forProject = true; + const attachToProject = true; + let expectedResult = 'success'; + cy.goToTaskList(); + cy.createAnnotationTask( + taskName, + mainLabelName, + attrName, + defaultAttrValue, + archiveName, + multiAttrParams, + advancedConfigurationParams, + forProject, + attachToProject, + projectName, + expectedResult, + ); + + expectedResult = 'fail'; + cy.goToTaskList(); + cy.createAnnotationTask( + brokenTaskName, + mainLabelName, + attrName, + defaultAttrValue, + brokenArchiveName, + multiAttrParams, + advancedConfigurationParams, + forProject, + attachToProject, + projectName, + expectedResult, + ); + + cy.goToTaskList(); + cy.openTask(taskName); + cy.url().then((taskUrl) => { + data.taskID = Number(taskUrl.split('/').slice(-1)[0].split('?')[0]); + cy.getJobIDFromIdx(0).then((jobID) => { + cy.headlessCreateObjects([rectanglePayload], jobID); + }); + }); + }); + }); + + after(() => { + cy.contains('.cvat-header-button', 'Cloud Storages').should('be.visible').click(); + cy.url().should('include', '/cloudstorages'); + cy.deleteCloudStorage(cloudStorageData.displayName); + + cy.headlessDeleteProject(data.projectID); + }); + + describe('Requests page. Create a task.', () => { + beforeEach(() => { + cy.contains('.cvat-header-button', 'Requests').should('be.visible').click(); + cy.url().should('include', '/requests'); + }); + + it('Creating a task creates a request. Correct task can be opened.', () => { + checkRequestStatus(`Task #${data.taskID}`, () => { + cy.contains('Create Task').should('exist'); + cy.get('.cvat-request-item-progress-success').should('exist'); + }); + }); + + it('Creating a task creates a request. Incorrect task can not be opened.', () => { + checkRequestStatus('', () => { + cy.contains('Create Task').should('exist'); + cy.get('.cvat-request-item-progress-failed').should('exist'); + }, { resourceLink: '' }); + }); + }); + + describe('Requests page. Export a task.', () => { + beforeEach(openTask); + + it('Export creates a request. Task can be opened from request. Export can be downloaded after page reload.', () => { + cy.exportTask({ + type: 'annotations', + format: exportFormat, + archiveCustomName: annotationsArchiveNameLocal, + }); + + checkRequestStatus(`Task #${data.taskID}`, () => { + cy.contains('Export Annotations').should('exist'); + cy.get('.cvat-request-item-progress-success').should('exist'); + cy.contains('Expires').should('exist'); + cy.contains(exportFormat).should('exist'); + }); + + cy.visit('/requests'); + cy.downloadExport({ expectNotification: false }).then((file) => { + cy.verifyDownload(file); + exportFileName = file; + }); + }); + + it('Export on cloud storage creates a request. Expire field does not exist.', () => { + cy.exportTask({ + type: 'annotations', + format: exportFormat, + archiveCustomName: annotationsArchiveNameCloud, + targetStorage: { + location: 'Cloud storage', + cloudStorageId: data.cloudStorageID, + }, + useDefaultLocation: false, + }); + cy.waitForFileUploadToCloudStorage(); + + checkRequestStatus(`Task #${data.taskID}`, () => { + cy.contains('Export Annotations').should('exist'); + cy.get('.cvat-request-item-progress-success').should('exist'); + cy.contains('Expires').should('not.exist'); + cy.contains(exportFormat).should('exist'); + }); + }); + }); + + describe('Requests page. Import a task.', () => { + beforeEach(() => { + openTask(); + cy.clickInTaskMenu('Upload annotations', true); + }); + + it('Import creates a request. Task can be opened from request.', () => { + cy.uploadAnnotations({ + format: exportFormat.split(' ')[0], + filePath: exportFileName, + confirmModalClassName: '.cvat-modal-content-load-task-annotation', + waitAnnotationsGet: false, + }); + + checkRequestStatus(`Task #${data.taskID}`, () => { + cy.contains('Import Annotations').should('exist'); + cy.get('.cvat-request-item-progress-success').should('exist'); + }); + }); + + it('Import creates a request. Task can be opened from incorrect request.', () => { + cy.uploadAnnotations({ + format: exportFormat.split(' ')[0], + filePath: badAnnotationsName, + confirmModalClassName: '.cvat-modal-content-load-task-annotation', + waitAnnotationsGet: false, + expectedResult: 'fail', + }); + + checkRequestStatus(`Task #${data.taskID}`, () => { + cy.contains('Import Annotations').should('exist'); + cy.get('.cvat-request-item-progress-failed').should('exist'); + }); + }); + }); + + describe('Requests page. Project backup.', () => { + beforeEach(() => { + cy.contains('.cvat-header-button', 'Projects').should('be.visible').click(); + cy.url().should('include', '/projects'); + }); + + it('Export backup creates a request. Project can be opened.', () => { + cy.backupProject( + projectName, + backupArchiveName, + ); + + checkRequestStatus(`Project #${data.projectID}`, () => { + cy.contains('Export Backup').should('exist'); + cy.get('.cvat-request-item-progress-success').should('exist'); + cy.contains('Expires').should('exist'); + }, `/projects/${data.projectID}`); + cy.downloadExport().then((file) => { + cy.verifyDownload(file); + }); + }); + + it('Import backup creates a request. Project can not be opened.', () => { + cy.restoreProject( + `${backupArchiveName}.zip`, + ); + + checkRequestStatus('', () => { + cy.contains('Import Backup').should('exist'); + cy.get('.cvat-request-item-progress-success').should('exist'); + cy.get('.cvat-requests-name').should('not.exist'); + }, { checkResourceLink: false }); + }); + }); + + describe('Regression tests', () => { + beforeEach(openTask); + + it('Export job in different formats from task page simultaneously.', () => { + cy.intercept('GET', '/api/requests/**', (req) => { + req.on('response', (res) => { + res.setDelay(5000); + }); + }); + + cy.getJobIDFromIdx(0).then((jobID) => { + const closeExportNotification = () => { + cy.contains('Export is finished').should('be.visible'); + cy.closeNotification('.ant-notification-notice-info'); + }; + + const exportParams = { + type: 'dataset', + format: exportFormat, + archiveCustomName: 'job_annotations_cvat', + jobOnTaskPage: jobID, + }; + + cy.exportJob(exportParams); + const newExportParams = { + ...exportParams, + format: 'COCO', + archiveCustomName: 'job_annotations_coco', + }; + cy.exportJob(newExportParams); + + closeExportNotification(); + closeExportNotification(); + + cy.contains('.cvat-header-button', 'Requests').should('be.visible').click(); + cy.url().should('include', '/requests'); + + cy.get(`.cvat-requests-card:contains("Job #${jobID}")`) + .should('have.length', 2) + .each((card) => { + cy.wrap(card).within(() => { + cy.get('.cvat-request-item-progress-success').should('exist'); + }); + }); + }); + }); + }); +}); diff --git a/tests/cypress/e2e/issues_prs2/issue_5274_upload_annotations_different_file_formats.js b/tests/cypress/e2e/issues_prs2/issue_5274_upload_annotations_different_file_formats.js index 1ac3d259dca..bcf4d627bfe 100644 --- a/tests/cypress/e2e/issues_prs2/issue_5274_upload_annotations_different_file_formats.js +++ b/tests/cypress/e2e/issues_prs2/issue_5274_upload_annotations_different_file_formats.js @@ -53,11 +53,11 @@ context('Upload annotations in different file formats', () => { cy.intercept('GET', '/api/jobs/**/annotations?**').as('uploadAnnotationsGet'); for (const archive of archives) { cy.interactMenu('Upload annotations'); - cy.uploadAnnotations( - archive.format.split(' ')[0], - `${archive.archiveCustomName}/${archive.annotationsPath}`, - '.cvat-modal-content-load-job-annotation', - ); + cy.uploadAnnotations({ + format: archive.format.split(' ')[0], + filePath: `${archive.archiveCustomName}/${archive.annotationsPath}`, + confirmModalClassName: '.cvat-modal-content-load-job-annotation', + }); cy.get('.cvat-notification-notice-upload-annotations-fail').should('not.exist'); cy.get('#cvat_canvas_shape_1').should('exist'); diff --git a/tests/cypress/support/commands.js b/tests/cypress/support/commands.js index 2803812a634..83628abfb66 100644 --- a/tests/cypress/support/commands.js +++ b/tests/cypress/support/commands.js @@ -208,7 +208,7 @@ Cypress.Commands.add( cy.get('.cvat-project-search-field').first().within(() => { cy.get('[type="search"]').should('have.value', projectName); }); - cy.get('.cvat-project-subset-field').type(projectSubsetFieldValue); + cy.get('.cvat-project-subset-field').type(`${projectSubsetFieldValue}{Enter}`); cy.get('.cvat-constructor-viewer-new-item').should('not.exist'); } cy.get('input[type="file"]').attachFile(image, { subjectType: 'drag-n-drop' }); @@ -219,6 +219,8 @@ Cypress.Commands.add( cy.get('.cvat-submit-continue-task-button').click(); if (expectedResult === 'success') { cy.get('.cvat-notification-create-task-success').should('exist').find('[data-icon="close"]').click(); + } else if (expectedResult === 'fail') { + cy.get('.cvat-notification-notice-create-task-failed').should('exist').find('[data-icon="close"]').click(); } if (!forProject) { cy.goToTaskList(); @@ -899,12 +901,11 @@ Cypress.Commands.add('confirmUpdate', (modalWindowClassName) => { }); Cypress.Commands.add( - 'uploadAnnotations', ( - format, - filePath, - confirmModalClassName, - sourceStorage = null, - useDefaultLocation = true, + 'uploadAnnotations', ({ + format, filePath, confirmModalClassName, + sourceStorage = null, useDefaultLocation = true, waitAnnotationsGet = true, + expectedResult = 'success', + }, ) => { cy.get('.cvat-modal-import-dataset').find('.cvat-modal-import-select').click(); cy.contains('.cvat-modal-import-dataset-option-item', format).click(); @@ -937,9 +938,16 @@ Cypress.Commands.add( cy.confirmUpdate(confirmModalClassName); cy.get('.cvat-notification-notice-import-annotation-start').should('be.visible'); cy.closeNotification('.cvat-notification-notice-import-annotation-start'); - cy.wait('@uploadAnnotationsGet').its('response.statusCode').should('equal', 200); - cy.contains('Annotations have been loaded').should('be.visible'); - cy.closeNotification('.ant-notification-notice-info'); + if (waitAnnotationsGet) { + cy.wait('@uploadAnnotationsGet').its('response.statusCode').should('equal', 200); + } + if (expectedResult === 'success') { + cy.contains('Annotations have been loaded').should('be.visible'); + cy.closeNotification('.ant-notification-notice-info'); + } else if (expectedResult === 'fail') { + cy.contains('Could not upload annotation').should('be.visible'); + cy.closeNotification('.ant-notification-notice-error'); + } }, ); @@ -1258,9 +1266,17 @@ Cypress.Commands.add('exportTask', ({ Cypress.Commands.add('exportJob', ({ type, format, archiveCustomName, - targetStorage = null, useDefaultLocation = true, + targetStorage = null, useDefaultLocation = true, jobOnTaskPage = null, }) => { - cy.interactMenu('Export job dataset'); + if (!jobOnTaskPage) { + cy.interactMenu('Export job dataset'); + } else { + cy.get('.cvat-job-item').contains('a', `Job #${jobOnTaskPage}`) + .parents('.cvat-job-item') + .find('.cvat-job-item-more-button') + .click(); + cy.contains('Export annotations').click(); + } cy.get('.cvat-modal-export-job').should('be.visible').find('.cvat-modal-export-select').click(); cy.get('.ant-select-dropdown') .not('.ant-select-dropdown-hidden') @@ -1288,13 +1304,15 @@ Cypress.Commands.add('exportJob', ({ cy.get('.cvat-cloud-storage-select-provider').click(); } } - cy.contains('button', 'OK').click(); + cy.get('.cvat-modal-export-job').contains('button', 'OK').click(); cy.get('.cvat-notification-notice-export-job-start').should('be.visible'); cy.closeNotification('.cvat-notification-notice-export-job-start'); }); -Cypress.Commands.add('downloadExport', () => { - cy.verifyNotification(); +Cypress.Commands.add('downloadExport', ({ expectNotification = true } = {}) => { + if (expectNotification) { + cy.verifyNotification(); + } cy.get('.cvat-header-requests-button').click(); cy.get('.cvat-spinner').should('not.exist'); cy.get('.cvat-requests-list').should('be.visible'); From 3cd4c391e115c5094503645e2ea9fd564f728f68 Mon Sep 17 00:00:00 2001 From: smit <90560950+smit9924@users.noreply.github.com> Date: Tue, 20 Aug 2024 15:48:12 +0530 Subject: [PATCH 10/22] docs: Add VS Code WSL extension install step for dev env setup (#8314) - This commit adds a step to the CVAT development setup guide for users working with WSL (Windows Subsystem for Linux). The added instruction guides users to install the VS Code extension for WSL, ensuring that Visual Studio Code opens correctly inside the WSL environment. - This change addresses an issue where users might encounter a 'DEBUG STOPPED' error if the extension is not installed, improving the overall setup experience. - Related to issue #8313. ### Motivation and context ### How has this been tested? ### Checklist - [x] I submit my changes into the `develop` branch - [ ] I have created a changelog fragment - [x] I have updated the documentation accordingly - [ ] I have added tests to cover my changes - [x] I have linked related issues (see [GitHub docs]( https://help.github.com/en/github/managing-your-work-on-github/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword)) - [ ] I have increased versions of npm packages if it is necessary ([cvat-canvas](https://github.com/cvat-ai/cvat/tree/develop/cvat-canvas#versioning), [cvat-core](https://github.com/cvat-ai/cvat/tree/develop/cvat-core#versioning), [cvat-data](https://github.com/cvat-ai/cvat/tree/develop/cvat-data#versioning) and [cvat-ui](https://github.com/cvat-ai/cvat/tree/develop/cvat-ui#versioning)) ### License - [x] I submit _my code changes_ under the same [MIT License]( https://github.com/cvat-ai/cvat/blob/develop/LICENSE) that covers the project. Feel free to contact the maintainers if that's a concern. ## Summary by CodeRabbit - **Documentation** - Added instructions for installing a Visual Studio Code extension for WSL to improve the development environment setup. --------- Co-authored-by: Andrey Zhavoronkov --- site/content/en/docs/contributing/development-environment.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/site/content/en/docs/contributing/development-environment.md b/site/content/en/docs/contributing/development-environment.md index bf39343bec8..cf2b1c01d71 100644 --- a/site/content/en/docs/contributing/development-environment.md +++ b/site/content/en/docs/contributing/development-environment.md @@ -234,7 +234,8 @@ You develop CVAT under WSL (Windows subsystem for Linux) following next steps. ```powershell wsl -d Ubuntu-18.04 ``` - +- Install the VS Code extension for WSL, which helps you to open VS Code correctly inside WSL. + You can find the extension [here](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-wsl). - Run all commands from this installation guide in WSL Ubuntu shell. - You might have to manually start the redis server in wsl before you can start the configuration inside Visual Studio Code. You can do this with `sudo service redis-server start`. Alternatively you can also From 3fdb03264a9c1107301218b2ba7eb8dc48f660f0 Mon Sep 17 00:00:00 2001 From: Maria Khrustaleva Date: Wed, 21 Aug 2024 10:13:33 +0200 Subject: [PATCH 11/22] Add REST API tests for /requests API && test both versions of the export API (#8216) This PR fixes the following issues: - [export API v1] do not reinitialize dataset export process when downloading a result file if a resource (project|task|job) has been updated since the first initialized export request - [export API v1] return `rq_id` for all requests with 202 status code (not only for initialization requests) - [requests API] Fixed filtering by format && added resource to allowed filters REST API tests updates: - Added tests to check requests filtration using simple filters - Added tests to check specific requests retrieving - Updated all tests that export project|task|job datasets|annotations|backups: - to test both API versions (including API mixing) - to use only appropriate resources by checking the default export location - Added fixtures to filter projects/tasks assets - Updated default target|source buckets to `import/export` bucket to exclude the same bucket usage as a data source in several tests (when bucket content is used as task data) and as a bucket for results ## Summary by CodeRabbit - **New Features** - Enhanced job handling for exports, improving error management and job state tracking. - Introduced a new `resource` field in the request handling system to improve data categorization. - Added new filtering capabilities for API queries, allowing users to filter by the `resource` field. - **Bug Fixes** - Improved status checks and handling for job requests. - Introduced exception handling for forbidden access during project backup attempts. - **Tests** - Refactored test suites to improve coverage and ensure compatibility across versions with new methods and exception handling. - New tests added to validate request handling functionality. --------- Co-authored-by: Maxim Zhiltsov --- .../20240820_134050_maria_rest_api_tests.md | 9 + ...background_operations.py => background.py} | 392 +++++++++------- cvat/apps/engine/filters.py | 10 +- cvat/apps/engine/mixins.py | 2 +- cvat/apps/engine/utils.py | 4 + cvat/apps/engine/views.py | 12 +- cvat/schema.yml | 13 +- dev/format_python_code.sh | 1 + tests/python/rest_api/test_cloud_storages.py | 2 +- tests/python/rest_api/test_invitations.py | 2 +- tests/python/rest_api/test_issues.py | 4 +- tests/python/rest_api/test_jobs.py | 158 +++++-- tests/python/rest_api/test_labels.py | 2 +- tests/python/rest_api/test_memberships.py | 2 +- tests/python/rest_api/test_organizations.py | 2 +- tests/python/rest_api/test_projects.py | 432 ++++++++++++------ tests/python/rest_api/test_quality_control.py | 6 +- tests/python/rest_api/test_requests.py | 337 ++++++++++++++ tests/python/rest_api/test_tasks.py | 137 ++++-- tests/python/rest_api/test_users.py | 2 +- tests/python/rest_api/test_webhooks.py | 2 +- tests/python/rest_api/utils.py | 396 +++++++++++++++- tests/python/shared/assets/cvat_db/data.json | 8 +- tests/python/shared/assets/jobs.json | 4 +- tests/python/shared/assets/projects.json | 4 +- tests/python/shared/assets/tasks.json | 4 +- tests/python/shared/fixtures/data.py | 47 ++ 27 files changed, 1568 insertions(+), 426 deletions(-) create mode 100644 changelog.d/20240820_134050_maria_rest_api_tests.md rename cvat/apps/engine/{background_operations.py => background.py} (69%) create mode 100644 tests/python/rest_api/test_requests.py diff --git a/changelog.d/20240820_134050_maria_rest_api_tests.md b/changelog.d/20240820_134050_maria_rest_api_tests.md new file mode 100644 index 00000000000..42b9bccf4fb --- /dev/null +++ b/changelog.d/20240820_134050_maria_rest_api_tests.md @@ -0,0 +1,9 @@ +### Fixed + +- Prevent export process from restarting when downloading a result file, + that resulted in downloading a file with new request ID + () +- Race condition occurred while handling parallel export requests + () +- Requests filtering using format and target filters + () diff --git a/cvat/apps/engine/background_operations.py b/cvat/apps/engine/background.py similarity index 69% rename from cvat/apps/engine/background_operations.py rename to cvat/apps/engine/background.py index de21c248dce..441d4702014 100644 --- a/cvat/apps/engine/background_operations.py +++ b/cvat/apps/engine/background.py @@ -4,8 +4,6 @@ import os import os.path as osp -import cvat.apps.dataset_manager as dm - from abc import ABC, abstractmethod from dataclasses import dataclass from datetime import datetime @@ -14,25 +12,32 @@ import django_rq from attrs.converters import to_bool from django.conf import settings -from django.http.request import HttpRequest +from django.http.response import HttpResponseBadRequest from django.utils import timezone from django_rq.queues import DjangoRQ from rest_framework import serializers, status +from rest_framework.request import Request from rest_framework.response import Response from rest_framework.reverse import reverse from rq.job import Job as RQJob from rq.job import JobStatus as RQJobStatus +import cvat.apps.dataset_manager as dm from cvat.apps.engine import models from cvat.apps.engine.backup import ProjectExporter, TaskExporter, create_backup from cvat.apps.engine.cloud_provider import export_resource_to_cloud_storage from cvat.apps.engine.location import StorageType, get_location_configuration from cvat.apps.engine.log import ServerLogManager from cvat.apps.engine.models import ( - Location, Project, Task, RequestAction, RequestTarget, RequestSubresource, + Location, + Project, + RequestAction, + RequestSubresource, + RequestTarget, + Task, ) from cvat.apps.engine.permissions import get_cloud_storage_for_import_or_export -from cvat.apps.engine.rq_job_handler import RQId +from cvat.apps.engine.rq_job_handler import RQId, RQJobMetaField from cvat.apps.engine.serializers import RqIdSerializer from cvat.apps.engine.utils import ( build_annotations_file_name, @@ -40,6 +45,7 @@ define_dependent_job, get_rq_job_meta, get_rq_lock_by_user, + get_rq_lock_for_job, sendfile, ) from cvat.apps.events.handlers import handle_dataset_export @@ -67,9 +73,7 @@ def __init__( self.db_instance = db_instance self.resource = db_instance.__class__.__name__.lower() if self.resource not in self.SUPPORTED_RESOURCES: - raise ValueError( - "Unexpected type of db_instance: {}".format(type(db_instance)) - ) + raise ValueError("Unexpected type of db_instance: {}".format(type(db_instance))) self.export_callback = export_callback @@ -82,24 +86,26 @@ def setup_background_job(self, queue: DjangoRQ, rq_id: str) -> None: pass @abstractmethod - def _handle_rq_job_v1(self, rq_job: RQJob, queue: DjangoRQ) -> Optional[Response]: + def _handle_rq_job_v1(self, rq_job: Optional[RQJob], queue: DjangoRQ) -> Optional[Response]: pass - def _handle_rq_job_v2(self, rq_job: RQJob, *args, **kwargs) -> Optional[Response]: - rq_job_status = rq_job.get_status(refresh=False) - if rq_job_status in { - RQJobStatus.FINISHED, - RQJobStatus.FAILED, - RQJobStatus.CANCELED, - RQJobStatus.STOPPED, - }: - rq_job.delete() + def _handle_rq_job_v2(self, rq_job: Optional[RQJob], *args, **kwargs) -> Optional[Response]: + if not rq_job: return None - return Response( - data=f"Export process is already {'started' if rq_job_status == RQJobStatus.STARTED else 'queued'}", - status=status.HTTP_409_CONFLICT, - ) + rq_job_status = rq_job.get_status(refresh=False) + + if rq_job_status in {RQJobStatus.STARTED, RQJobStatus.QUEUED}: + return Response( + data="Export request is being processed", + status=status.HTTP_409_CONFLICT, + ) + + if rq_job_status in (RQJobStatus.SCHEDULED, RQJobStatus.DEFERRED): + rq_job.cancel(enqueue_dependents=settings.ONE_RUNNING_JOB_IN_QUEUE_PER_USER) + + rq_job.delete() + return None def handle_rq_job(self, *args, **kwargs) -> Optional[Response]: if self.version == 1: @@ -115,9 +121,7 @@ def get_v1_endpoint_view_name(self) -> str: def make_result_url(self) -> str: view_name = self.get_v1_endpoint_view_name() - result_url = reverse( - view_name, args=[self.db_instance.pk], request=self.request - ) + result_url = reverse(view_name, args=[self.db_instance.pk], request=self.request) query_dict = self.request.query_params.copy() query_dict["action"] = "download" result_url += "?" + query_dict.urlencode() @@ -139,6 +143,14 @@ def get_instance_update_time(self) -> datetime: def get_timestamp(self, time_: datetime) -> str: return datetime.strftime(time_, "%Y_%m_%d_%H_%M_%S") + +def cancel_and_delete(rq_job: RQJob) -> None: + # In the case the server is configured with ONE_RUNNING_JOB_IN_QUEUE_PER_USER + # we have to enqueue dependent jobs after canceling one. + rq_job.cancel(enqueue_dependents=settings.ONE_RUNNING_JOB_IN_QUEUE_PER_USER) + rq_job.delete() + + class DatasetExportManager(_ResourceExportManager): SUPPORTED_RESOURCES = {"project", "task", "job"} @@ -156,7 +168,7 @@ def location(self) -> Location: def __init__( self, db_instance: Union[models.Project, models.Task, models.Job], - request: HttpRequest, + request: Request, export_callback: Callable, save_images: Optional[bool] = None, *, @@ -167,6 +179,7 @@ def __init__( format_name = request.query_params.get("format", "") filename = request.query_params.get("filename", "") + # can be passed directly when it is initialized based on API request, not query param save_images = ( save_images @@ -199,51 +212,14 @@ def __init__( def _handle_rq_job_v1( self, - rq_job: RQJob, + rq_job: Optional[RQJob], queue: DjangoRQ, ) -> Optional[Response]: - action = self.request.query_params.get("action") - if action not in {None, "download"}: - raise serializers.ValidationError( - "Unexpected action specified for the request" - ) - - request_time = rq_job.meta.get("request", {}).get("timestamp") - instance_update_time = self.get_instance_update_time() - if request_time is None or request_time < instance_update_time: - # The result is outdated, need to restart the export. - # Cancel the current job. - # The new attempt will be made after the last existing job. - # In the case the server is configured with ONE_RUNNING_JOB_IN_QUEUE_PER_USER - # we have to enqueue dependent jobs after canceling one. - rq_job.cancel(enqueue_dependents=settings.ONE_RUNNING_JOB_IN_QUEUE_PER_USER) - rq_job.delete() - return None - - instance_timestamp = self.get_timestamp(instance_update_time) - REQUEST_TIMEOUT = 60 - - if action == "download": - if self.export_args.location != Location.LOCAL: - return Response( - 'Action "download" is only supported for a local export location', - status=status.HTTP_400_BAD_REQUEST, - ) - - if not rq_job.is_finished: - return Response( - "Export has not finished", status=status.HTTP_400_BAD_REQUEST - ) - - file_path = rq_job.return_value() - - if not file_path: - return Response( - "A result for exporting job was not found for finished RQ job", - status=status.HTTP_500_INTERNAL_SERVER_ERROR, - ) + def is_result_outdated() -> bool: + return rq_job.meta[RQJobMetaField.REQUEST]["timestamp"] < instance_update_time + def handle_local_download() -> Response: with dm.util.get_export_cache_lock(file_path, ttl=REQUEST_TIMEOUT): if not osp.exists(file_path): return Response( @@ -272,11 +248,59 @@ def _handle_rq_job_v1( attachment_filename=filename, ) - if rq_job.is_finished: + action = self.request.query_params.get("action") + + if action not in {None, "download"}: + raise serializers.ValidationError( + f"Unexpected action {action!r} specified for the request" + ) + + msg_no_such_job_when_downloading = ( + "Unknown export request id. " + "Please request export first by sending a request without the action=download parameter." + ) + if not rq_job: + return ( + None + if action != "download" + else HttpResponseBadRequest(msg_no_such_job_when_downloading) + ) + + # define status once to avoid refreshing it on each check + # FUTURE-TODO: get_status will raise InvalidJobOperation exception instead of returning None in one of the next releases + rq_job_status = rq_job.get_status(refresh=False) + + # handle cases where the status is None for some reason + if not rq_job_status: + rq_job.delete() + return ( + None + if action != "download" + else HttpResponseBadRequest(msg_no_such_job_when_downloading) + ) + + if action == "download": + if self.export_args.location != Location.LOCAL: + return HttpResponseBadRequest( + 'Action "download" is only supported for a local dataset location' + ) + if rq_job_status not in { + RQJobStatus.FINISHED, + RQJobStatus.FAILED, + RQJobStatus.CANCELED, + RQJobStatus.STOPPED, + }: + return HttpResponseBadRequest("Dataset export has not been finished yet") + + instance_update_time = self.get_instance_update_time() + instance_timestamp = self.get_timestamp(instance_update_time) + + REQUEST_TIMEOUT = 60 + + if rq_job_status == RQJobStatus.FINISHED: if self.export_args.location == Location.CLOUD_STORAGE: rq_job.delete() return Response(status=status.HTTP_200_OK) - elif self.export_args.location == Location.LOCAL: file_path = rq_job.return_value() @@ -286,32 +310,29 @@ def _handle_rq_job_v1( status=status.HTTP_500_INTERNAL_SERVER_ERROR, ) - with dm.util.get_export_cache_lock(file_path, ttl=REQUEST_TIMEOUT): - if osp.exists(file_path): - # Update last update time to prolong the export lifetime - # as the last access time is not available on every filesystem - os.utime(file_path, None) - - return Response(status=status.HTTP_201_CREATED) - else: - # Cancel and reenqueue the job. - # The new attempt will be made after the last existing job. - # In the case the server is configured with ONE_RUNNING_JOB_IN_QUEUE_PER_USER - # we have to enqueue dependent jobs after canceling one. - rq_job.cancel( - enqueue_dependents=settings.ONE_RUNNING_JOB_IN_QUEUE_PER_USER - ) - rq_job.delete() + if action == "download": + return handle_local_download() + else: + with dm.util.get_export_cache_lock(file_path, ttl=REQUEST_TIMEOUT): + if osp.exists(file_path) and not is_result_outdated(): + # Update last update time to prolong the export lifetime + # as the last access time is not available on every filesystem + os.utime(file_path, None) + + return Response(status=status.HTTP_201_CREATED) + + cancel_and_delete(rq_job) + return None else: raise NotImplementedError( f"Export to {self.export_args.location} location is not implemented yet" ) - elif rq_job.is_failed: - exc_info = rq_job.meta.get("formatted_exception", str(rq_job.exc_info)) + elif rq_job_status == RQJobStatus.FAILED: + exc_info = rq_job.meta.get(RQJobMetaField.FORMATTED_EXCEPTION, str(rq_job.exc_info)) rq_job.delete() return Response(exc_info, status=status.HTTP_500_INTERNAL_SERVER_ERROR) elif ( - rq_job.is_deferred + rq_job_status == RQJobStatus.DEFERRED and rq_job.id not in queue.deferred_job_registry.get_job_ids() ): # Sometimes jobs can depend on outdated jobs in the deferred jobs registry. @@ -322,24 +343,32 @@ def _handle_rq_job_v1( # Such dependencies are never removed or finished, # as there is no TTL for deferred jobs, # so the current job can be blocked indefinitely. + cancel_and_delete(rq_job) + return None - # Cancel the current job and then reenqueue it, considering the current situation. - # The new attempt will be made after the last existing job. - # In the case the server is configured with ONE_RUNNING_JOB_IN_QUEUE_PER_USER - # we have to enqueue dependent jobs after canceling one. - rq_job.cancel(enqueue_dependents=settings.ONE_RUNNING_JOB_IN_QUEUE_PER_USER) + elif rq_job_status in {RQJobStatus.CANCELED, RQJobStatus.STOPPED}: rq_job.delete() - else: - return Response(status=status.HTTP_202_ACCEPTED) + return ( + None + if action != "download" + else Response( + "Export was cancelled, please request it one more time", + status=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + ) + + if is_result_outdated(): + cancel_and_delete(rq_job) + return None + + return Response(RqIdSerializer({"rq_id": rq_job.id}).data, status=status.HTTP_202_ACCEPTED) def export(self) -> Response: format_desc = {f.DISPLAY_NAME: f for f in dm.views.get_export_formats()}.get( self.export_args.format ) if format_desc is None: - raise serializers.ValidationError( - "Unknown format specified for the request" - ) + raise serializers.ValidationError("Unknown format specified for the request") elif not format_desc.ENABLED: return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) @@ -348,20 +377,21 @@ def export(self) -> Response: RequestAction.EXPORT, RequestTarget(self.resource), self.db_instance.pk, - subresource=RequestSubresource.DATASET - if self.export_args.save_images else RequestSubresource.ANNOTATIONS, + subresource=( + RequestSubresource.DATASET + if self.export_args.save_images + else RequestSubresource.ANNOTATIONS + ), format=self.export_args.format, user_id=self.request.user.id, ).render() - rq_job = queue.fetch_job(rq_id) - - if rq_job: - response = self.handle_rq_job(rq_job, queue) - if response: + # ensure that there is no race condition when processing parallel requests + with get_rq_lock_for_job(queue, rq_id): + rq_job = queue.fetch_job(rq_id) + if response := self.handle_rq_job(rq_job, queue): return response - - self.setup_background_job(queue, rq_id) + self.setup_background_job(queue, rq_id) handle_dataset_export( self.db_instance, @@ -478,7 +508,7 @@ def location(self) -> Location: def __init__( self, db_instance: Union[models.Project, models.Task], - request: HttpRequest, + request: Request, *, version: int = 2, ) -> None: @@ -486,6 +516,7 @@ def __init__( self.request = request filename = request.query_params.get("filename", "") + location_config = get_location_configuration( db_instance=self.db_instance, query_params=self.request.query_params, @@ -495,72 +526,97 @@ def __init__( def _handle_rq_job_v1( self, - rq_job: RQJob, + rq_job: Optional[RQJob], queue: DjangoRQ, ) -> Optional[Response]: last_instance_update_time = timezone.localtime(self.db_instance.updated_date) timestamp = self.get_timestamp(last_instance_update_time) action = self.request.query_params.get("action") - if action not in (None, "download"): raise serializers.ValidationError( - "Unexpected action specified for the request" + f"Unexpected action {action!r} specified for the request" ) - if action == "download": - if self.export_args.location != Location.LOCAL: - return Response( - 'Action "download" is only supported for a local backup location', - status=status.HTTP_400_BAD_REQUEST, - ) + msg_no_such_job_when_downloading = ( + "Unknown export request id. " + "Please request export first by sending a request without the action=download parameter." + ) + if not rq_job: + return ( + None + if action != "download" + else HttpResponseBadRequest(msg_no_such_job_when_downloading) + ) - if not rq_job.is_finished: - return Response( - "Backup has not finished", status=status.HTTP_400_BAD_REQUEST - ) + # define status once to avoid refreshing it on each check + # FUTURE-TODO: get_status will raise InvalidJobOperation exception instead of None in one of the next releases + rq_job_status = rq_job.get_status(refresh=False) - file_path = rq_job.return_value() + # handle cases where the status is None for some reason + if not rq_job_status: + rq_job.delete() + return ( + None + if action != "download" + else HttpResponseBadRequest(msg_no_such_job_when_downloading) + ) - if not file_path: - return Response( - "A result for exporting job was not found for finished RQ job", - status=status.HTTP_500_INTERNAL_SERVER_ERROR, + if action == "download": + if self.export_args.location != Location.LOCAL: + return HttpResponseBadRequest( + 'Action "download" is only supported for a local backup location' ) + if rq_job_status not in { + RQJobStatus.FINISHED, + RQJobStatus.FAILED, + RQJobStatus.CANCELED, + RQJobStatus.STOPPED, + }: + return HttpResponseBadRequest("Backup export has not been finished yet") + + if rq_job_status == RQJobStatus.FINISHED: + if self.export_args.location == Location.CLOUD_STORAGE: + rq_job.delete() + return Response(status=status.HTTP_200_OK) + elif self.export_args.location == Location.LOCAL: + file_path = rq_job.return_value() - elif not os.path.exists(file_path): - return Response( - "The result file does not exist in export cache", - status=status.HTTP_500_INTERNAL_SERVER_ERROR, - ) + if not file_path: + return Response( + "Export is completed, but has no results", + status=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) - filename = self.export_args.filename or build_backup_file_name( - class_name=self.resource, - identifier=self.db_instance.name, - timestamp=timestamp, - extension=os.path.splitext(file_path)[1], - ) + elif not os.path.exists(file_path): + return Response( + "The export result is not found", + status=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + if action == "download": + filename = self.export_args.filename or build_backup_file_name( + class_name=self.resource, + identifier=self.db_instance.name, + timestamp=timestamp, + extension=os.path.splitext(file_path)[1], + ) - rq_job.delete() - return sendfile( - self.request, file_path, attachment=True, attachment_filename=filename - ) + rq_job.delete() + return sendfile( + self.request, file_path, attachment=True, attachment_filename=filename + ) - if rq_job.is_finished: - if self.export_args.location == Location.LOCAL: return Response(status=status.HTTP_201_CREATED) - - elif self.export_args.location == Location.CLOUD_STORAGE: - rq_job.delete() - return Response(status=status.HTTP_200_OK) else: - raise NotImplementedError() - elif rq_job.is_failed: - exc_info = rq_job.meta.get("formatted_exception", str(rq_job.exc_info)) + raise NotImplementedError( + f"Export to {self.export_args.location} location is not implemented yet" + ) + elif rq_job_status == RQJobStatus.FAILED: + exc_info = rq_job.meta.get(RQJobMetaField.FORMATTED_EXCEPTION, str(rq_job.exc_info)) rq_job.delete() return Response(exc_info, status=status.HTTP_500_INTERNAL_SERVER_ERROR) elif ( - rq_job.is_deferred + rq_job_status == RQJobStatus.DEFERRED and rq_job.id not in queue.deferred_job_registry.get_job_ids() ): # Sometimes jobs can depend on outdated jobs in the deferred jobs registry. @@ -571,15 +627,21 @@ def _handle_rq_job_v1( # Such dependencies are never removed or finished, # as there is no TTL for deferred jobs, # so the current job can be blocked indefinitely. + cancel_and_delete(rq_job) + return None - # Cancel the current job and then reenqueue it, considering the current situation. - # The new attempt will be made after the last existing job. - # In the case the server is configured with ONE_RUNNING_JOB_IN_QUEUE_PER_USER - # we have to enqueue dependent jobs after canceling one. - rq_job.cancel(enqueue_dependents=settings.ONE_RUNNING_JOB_IN_QUEUE_PER_USER) + elif rq_job_status in {RQJobStatus.CANCELED, RQJobStatus.STOPPED}: rq_job.delete() - else: - return Response(status=status.HTTP_202_ACCEPTED) + return ( + None + if action != "download" + else Response( + "Export was cancelled, please request it one more time", + status=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + ) + + return Response(RqIdSerializer({"rq_id": rq_job.id}).data, status=status.HTTP_202_ACCEPTED) def export(self) -> Response: queue: DjangoRQ = django_rq.get_queue(self.QUEUE_NAME) @@ -590,14 +652,14 @@ def export(self) -> Response: subresource=RequestSubresource.BACKUP, user_id=self.request.user.id, ).render() - rq_job = queue.fetch_job(rq_id) - if rq_job: - response = self.handle_rq_job(rq_job, queue) - if response: + # ensure that there is no race condition when processing parallel requests + with get_rq_lock_for_job(queue, rq_id): + rq_job = queue.fetch_job(rq_id) + if response := self.handle_rq_job(rq_job, queue): return response + self.setup_background_job(queue, rq_id) - self.setup_background_job(queue, rq_id) serializer = RqIdSerializer(data={"rq_id": rq_id}) serializer.is_valid(raise_exception=True) @@ -642,9 +704,7 @@ def setup_background_job( is_default=self.export_args.location_config["is_default"], ) - last_instance_update_time = timezone.localtime( - self.db_instance.updated_date - ) + last_instance_update_time = timezone.localtime(self.db_instance.updated_date) timestamp = self.get_timestamp(last_instance_update_time) filename_pattern = build_backup_file_name( diff --git a/cvat/apps/engine/filters.py b/cvat/apps/engine/filters.py index 5ae98bf3ad8..663b6554e16 100644 --- a/cvat/apps/engine/filters.py +++ b/cvat/apps/engine/filters.py @@ -16,7 +16,7 @@ from django.db.models.query import QuerySet from django.utils.translation import gettext_lazy as _ from django.utils.encoding import force_str -from django.http import HttpRequest +from rest_framework.request import Request from rest_framework import filters from rest_framework.compat import coreapi, coreschema from rest_framework.exceptions import ValidationError @@ -412,11 +412,11 @@ def get_schema_operation_parameters(self, view): parameters.append(parameter) return parameters - def filter_queryset(self, request: HttpRequest, queryset: Iterable, view): + def filter_queryset(self, request: Request, queryset: Iterable, view): filtered_queryset = queryset query_params = request.query_params - filters_to_use = set(query_params) + filters_to_use = set(query_params.keys()) simple_filters = getattr(view, self.filter_fields_attr, None) lookup_fields = self.get_lookup_fields(view) @@ -465,7 +465,7 @@ def get_ordering(self, request, queryset, view) -> Tuple[List[str], bool]: return result, reverse - def filter_queryset(self, request: HttpRequest, queryset: Iterable, view) -> Iterable: + def filter_queryset(self, request: Request, queryset: Iterable, view) -> Iterable: ordering, reverse = self.get_ordering(request, queryset, view) if ordering: @@ -520,7 +520,7 @@ def _apply_filter(self, rules, lookup_fields, obj): else: raise ValidationError(f'filter: {op} operation with {args} arguments is not implemented') - def filter_queryset(self, request: HttpRequest, queryset: Iterable, view) -> Iterable: + def filter_queryset(self, request: Request, queryset: Iterable, view) -> Iterable: filtered_queryset = queryset json_rules = request.query_params.get(self.filter_param) if json_rules: diff --git a/cvat/apps/engine/mixins.py b/cvat/apps/engine/mixins.py index 675152bf663..3e48bf85327 100644 --- a/cvat/apps/engine/mixins.py +++ b/cvat/apps/engine/mixins.py @@ -29,7 +29,7 @@ from rest_framework.response import Response from rest_framework.views import APIView -from cvat.apps.engine.background_operations import (BackupExportManager, +from cvat.apps.engine.background import (BackupExportManager, DatasetExportManager) from cvat.apps.engine.handlers import clear_import_cache from cvat.apps.engine.location import StorageType, get_location_configuration diff --git a/cvat/apps/engine/utils.py b/cvat/apps/engine/utils.py index 2fe1b3e8c33..01748778339 100644 --- a/cvat/apps/engine/utils.py +++ b/cvat/apps/engine/utils.py @@ -203,6 +203,10 @@ def get_rq_lock_by_user(queue: DjangoRQ, user_id: int) -> Union[Lock, nullcontex return queue.connection.lock(f'{queue.name}-lock-{user_id}', timeout=30) return nullcontext() +def get_rq_lock_for_job(queue: DjangoRQ, rq_id: str) -> Lock: + # lock timeout corresponds to the nginx request timeout (proxy_read_timeout) + return queue.connection.lock(f'lock-for-job-{rq_id}'.lower(), timeout=60) + def get_rq_job_meta( request: HttpRequest, db_obj: Any, diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 92d9db78316..05a50857b28 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -491,7 +491,7 @@ def upload_finished(self, request): return Response(data='Unknown upload was finished', status=status.HTTP_400_BAD_REQUEST) - @extend_schema(summary='Get project annotations or export them as a dataset', + @extend_schema(summary='Export project annotations as a dataset', description=textwrap.dedent("""\ Deprecation warning: @@ -535,11 +535,7 @@ def upload_finished(self, request): def annotations(self, request, pk): # FUTURE-TODO: mark exporting dataset using this endpoint as deprecated when new API for result file downloading will be implemented self._object = self.get_object() # force call of check_object_permissions() - return self.export_dataset_v1( - request=request, - save_images=False, - get_data=dm.task.get_job_data, - ) + return self.export_dataset_v1(request=request, save_images=False) @extend_schema(summary='Back up a project', description=textwrap.dedent("""\ @@ -3275,6 +3271,7 @@ class RequestViewSet(viewsets.GenericViewSet): 'job_id', # derivatives fields (from parsed rq_id) 'action', + 'target', 'subresource', 'format', ] @@ -3284,7 +3281,9 @@ class RequestViewSet(viewsets.GenericViewSet): lookup_fields = { 'created_date': 'created_at', 'action': 'parsed_rq_id.action', + 'target': 'parsed_rq_id.target', 'subresource': 'parsed_rq_id.subresource', + 'format': 'parsed_rq_id.format', 'status': 'get_status', 'project_id': 'meta.project_id', 'task_id': 'meta.task_id', @@ -3300,6 +3299,7 @@ class RequestViewSet(viewsets.GenericViewSet): 'task_id': SchemaField('integer'), 'job_id': SchemaField('integer'), 'action': SchemaField('string', RequestAction.choices), + 'target': SchemaField('string', RequestTarget.choices), 'subresource': SchemaField('string', RequestSubresource.choices), 'format': SchemaField('string'), 'org': SchemaField('string'), diff --git a/cvat/schema.yml b/cvat/schema.yml index d4f01b9642a..4c6a14a51b6 100644 --- a/cvat/schema.yml +++ b/cvat/schema.yml @@ -3552,7 +3552,7 @@ paths: - POST /api/projects//dataset/export?save_images=False to initiate exporting process - GET /api/requests/ to check export status, where rq_id is request id returned on initializing request' - summary: Get project annotations or export them as a dataset + summary: Export project annotations as a dataset parameters: - in: query name: action @@ -4541,7 +4541,7 @@ paths: Details about the syntax used can be found at the link: https://jsonlogic.com/ - Available filter_fields: ['status', 'project_id', 'task_id', 'job_id', 'action', 'subresource', 'format']. + Available filter_fields: ['status', 'project_id', 'task_id', 'job_id', 'action', 'target', 'subresource', 'format']. schema: type: string - name: format @@ -4602,6 +4602,15 @@ paths: - annotations - dataset - backup + - name: target + in: query + description: A simple equality filter for the target field + schema: + type: string + enum: + - project + - task + - job - name: task_id in: query description: A simple equality filter for the task_id field diff --git a/dev/format_python_code.sh b/dev/format_python_code.sh index 9483c5fbcb2..5b455a296f4 100755 --- a/dev/format_python_code.sh +++ b/dev/format_python_code.sh @@ -24,6 +24,7 @@ for paths in \ "cvat/apps/quality_control" \ "cvat/apps/analytics_report" \ "cvat/apps/engine/lazy_list.py" \ + "cvat/apps/engine/background.py" \ ; do ${BLACK} -- ${paths} ${ISORT} -- ${paths} diff --git a/tests/python/rest_api/test_cloud_storages.py b/tests/python/rest_api/test_cloud_storages.py index cd89d9ebba7..1d00e0c680c 100644 --- a/tests/python/rest_api/test_cloud_storages.py +++ b/tests/python/rest_api/test_cloud_storages.py @@ -149,7 +149,7 @@ def _get_endpoint(self, api_client: ApiClient) -> Endpoint: ("provider_type", "name", "resource", "credentials_type", "owner"), ) def test_can_use_simple_filter_for_object_list(self, field): - return super().test_can_use_simple_filter_for_object_list(field) + return super()._test_can_use_simple_filter_for_object_list(field) @pytest.mark.usefixtures("restore_db_per_function") diff --git a/tests/python/rest_api/test_invitations.py b/tests/python/rest_api/test_invitations.py index 3626df9c5d0..b43d093f926 100644 --- a/tests/python/rest_api/test_invitations.py +++ b/tests/python/rest_api/test_invitations.py @@ -121,7 +121,7 @@ def _get_endpoint(self, api_client: ApiClient) -> Endpoint: ("owner",), ) def test_can_use_simple_filter_for_object_list(self, field): - return super().test_can_use_simple_filter_for_object_list(field) + return super()._test_can_use_simple_filter_for_object_list(field) @pytest.mark.usefixtures("restore_db_per_class") diff --git a/tests/python/rest_api/test_issues.py b/tests/python/rest_api/test_issues.py index 74f09c1a8d5..c6c043f2e44 100644 --- a/tests/python/rest_api/test_issues.py +++ b/tests/python/rest_api/test_issues.py @@ -355,7 +355,7 @@ def _get_endpoint(self, api_client: ApiClient) -> Endpoint: ("owner", "assignee", "job_id", "resolved", "frame_id"), ) def test_can_use_simple_filter_for_object_list(self, field): - return super().test_can_use_simple_filter_for_object_list(field) + return super()._test_can_use_simple_filter_for_object_list(field) class TestCommentsListFilters(CollectionSimpleFilterTestBase): @@ -393,7 +393,7 @@ def _get_field_samples(self, field: str) -> Tuple[Any, List[Dict[str, Any]]]: ("owner", "issue_id", "job_id", "frame_id"), ) def test_can_use_simple_filter_for_object_list(self, field): - return super().test_can_use_simple_filter_for_object_list(field) + return super()._test_can_use_simple_filter_for_object_list(field) @pytest.mark.usefixtures("restore_db_per_class") diff --git a/tests/python/rest_api/test_jobs.py b/tests/python/rest_api/test_jobs.py index cd344506336..4fbea276e0a 100644 --- a/tests/python/rest_api/test_jobs.py +++ b/tests/python/rest_api/test_jobs.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: MIT +import io import json import os import xml.etree.ElementTree as ET @@ -10,7 +11,8 @@ from copy import deepcopy from http import HTTPStatus from io import BytesIO -from typing import Any, Dict, List, Optional +from itertools import product +from typing import Any, Dict, List, Optional, Tuple, Union import numpy as np import pytest @@ -23,7 +25,12 @@ from shared.utils.config import make_api_client from shared.utils.helpers import generate_image_files -from .utils import CollectionSimpleFilterTestBase, compare_annotations, create_task, export_dataset +from .utils import ( + CollectionSimpleFilterTestBase, + compare_annotations, + create_task, + export_job_dataset, +) def get_job_staff(job, tasks, projects): @@ -900,7 +907,7 @@ def _get_endpoint(self, api_client: ApiClient) -> Endpoint: ), ) def test_can_use_simple_filter_for_object_list(self, field): - return super().test_can_use_simple_filter_for_object_list(field) + return super()._test_can_use_simple_filter_for_object_list(field) @pytest.mark.usefixtures("restore_db_per_class") @@ -1319,49 +1326,97 @@ def _check_cvat_for_video_job_annotations(content, values_to_be_checked): assert len(list(document.iter("track"))) == values_to_be_checked["tracks_length"] +@pytest.mark.usefixtures("restore_redis_inmem_per_function") @pytest.mark.usefixtures("restore_db_per_class") class TestJobDataset: - def _export_dataset(self, username, jid, **kwargs): - with make_api_client(username) as api_client: - return export_dataset(api_client.jobs_api.retrieve_dataset_endpoint, id=jid, **kwargs) - - def _export_annotations(self, username, jid, **kwargs): - with make_api_client(username) as api_client: - return export_dataset( - api_client.jobs_api.retrieve_annotations_endpoint, id=jid, **kwargs - ) - def test_can_export_dataset(self, admin_user: str, jobs_with_shapes: List): - job = jobs_with_shapes[0] - response = self._export_dataset(admin_user, job["id"]) - assert response.data + @pytest.fixture(autouse=True) + def setup(self, tasks): + self.tasks = tasks + + @staticmethod + def _test_export_dataset( + username: str, + jid: int, + *, + api_version: Union[int, Tuple[int]], + local_download: bool = True, + **kwargs, + ) -> Optional[bytes]: + dataset = export_job_dataset(username, api_version, save_images=True, id=jid, **kwargs) + if local_download: + assert zipfile.is_zipfile(io.BytesIO(dataset)) + else: + assert dataset is None + + return dataset + + @staticmethod + def _test_export_annotations( + username: str, jid: int, *, api_version: int, local_download: bool = True, **kwargs + ) -> Optional[bytes]: + dataset = export_job_dataset(username, api_version, save_images=False, id=jid, **kwargs) + if local_download: + assert zipfile.is_zipfile(io.BytesIO(dataset)) + else: + assert dataset is None + + return dataset - def test_non_admin_can_export_dataset(self, users, tasks, jobs_with_shapes): - job_id, username = next( + @pytest.mark.parametrize("api_version", product((1, 2), repeat=2)) + @pytest.mark.parametrize( + "local_download", (True, pytest.param(False, marks=pytest.mark.with_external_services)) + ) + def test_can_export_dataset_locally_and_to_cloud_with_both_api_versions( + self, + admin_user: str, + jobs_with_shapes: List, + filter_tasks, + api_version: Tuple[int], + local_download: bool, + ): + filter_ = "target_storage__location" + if local_download: + filter_ = "exclude_" + filter_ + + task_ids = [t["id"] for t in filter_tasks(**{filter_: "cloud_storage"})] + + job = next(j for j in jobs_with_shapes if j["task_id"] in task_ids) + self._test_export_dataset( + admin_user, + job["id"], + api_version=api_version, + local_download=local_download, + ) + + @pytest.mark.parametrize("api_version", (1, 2)) + def test_non_admin_can_export_dataset(self, users, jobs_with_shapes, api_version: int): + job, username = next( ( - (job["id"], tasks[job["task_id"]]["owner"]["username"]) + (job, self.tasks[job["task_id"]]["owner"]["username"]) for job in jobs_with_shapes - if "admin" not in users[tasks[job["task_id"]]["owner"]["id"]]["groups"] - and tasks[job["task_id"]]["target_storage"] is None - and tasks[job["task_id"]]["organization"] is None + if "admin" not in users[self.tasks[job["task_id"]]["owner"]["id"]]["groups"] + and self.tasks[job["task_id"]]["target_storage"] is None + and self.tasks[job["task_id"]]["organization"] is None ) ) - response = self._export_dataset(username, job_id) - assert response.data + self._test_export_dataset(username, job["id"], api_version=api_version) - def test_non_admin_can_export_annotations(self, users, tasks, jobs_with_shapes): - job_id, username = next( + @pytest.mark.parametrize("api_version", (1, 2)) + def test_non_admin_can_export_annotations(self, users, jobs_with_shapes, api_version: int): + job, username = next( ( - (job["id"], tasks[job["task_id"]]["owner"]["username"]) + (job, self.tasks[job["task_id"]]["owner"]["username"]) for job in jobs_with_shapes - if "admin" not in users[tasks[job["task_id"]]["owner"]["id"]]["groups"] - and tasks[job["task_id"]]["target_storage"] is None - and tasks[job["task_id"]]["organization"] is None + if "admin" not in users[self.tasks[job["task_id"]]["owner"]["id"]]["groups"] + and self.tasks[job["task_id"]]["target_storage"] is None + and self.tasks[job["task_id"]]["organization"] is None ) ) - response = self._export_annotations(username, job_id) - assert response.data + self._test_export_annotations(username, job["id"], api_version=api_version) + + @pytest.mark.parametrize("api_version", (1, 2)) @pytest.mark.parametrize("username, jid", [("admin1", 14)]) @pytest.mark.parametrize( "anno_format, anno_file_name, check_func", @@ -1377,15 +1432,15 @@ def test_exported_job_dataset_structure( anno_format, anno_file_name, check_func, - tasks, jobs, annotations, + api_version: int, ): job_data = jobs[jid] annotations_before = annotations["job"][str(jid)] values_to_be_checked = { - "task_size": tasks[job_data["task_id"]]["size"], + "task_size": self.tasks[job_data["task_id"]]["size"], # NOTE: data step is not stored in assets, default = 1 "job_size": job_data["stop_frame"] - job_data["start_frame"] + 1, "start_frame": job_data["start_frame"], @@ -1395,15 +1450,21 @@ def test_exported_job_dataset_structure( "mode": job_data["mode"], } - response = self._export_dataset(username, jid, format=anno_format) - assert response.data - with zipfile.ZipFile(BytesIO(response.data)) as zip_file: + dataset = self._test_export_dataset( + username, + jid, + api_version=api_version, + format=anno_format, + ) + + with zipfile.ZipFile(BytesIO(dataset)) as zip_file: assert ( len(zip_file.namelist()) == values_to_be_checked["job_size"] + 1 ) # images + annotation file content = zip_file.read(anno_file_name) check_func(content, values_to_be_checked) + @pytest.mark.parametrize("api_version", (1, 2)) @pytest.mark.parametrize("username", ["admin1"]) @pytest.mark.parametrize("jid", [25, 26]) @pytest.mark.parametrize( @@ -1419,13 +1480,21 @@ def test_exported_job_dataset_structure( ], ) def test_export_job_among_several_jobs_in_task( - self, username, jid, anno_format, anno_file_name, check_func, tasks, jobs, annotations + self, + username, + jid, + anno_format, + anno_file_name, + check_func, + jobs, + annotations, + api_version: int, ): job_data = jobs[jid] annotations_before = annotations["job"][str(jid)] values_to_be_checked = { - "task_size": tasks[job_data["task_id"]]["size"], + "task_size": self.tasks[job_data["task_id"]]["size"], # NOTE: data step is not stored in assets, default = 1 "job_size": job_data["stop_frame"] - job_data["start_frame"] + 1, "start_frame": job_data["start_frame"], @@ -1435,9 +1504,14 @@ def test_export_job_among_several_jobs_in_task( "mode": job_data["mode"], } - response = self._export_dataset(username, jid, format=anno_format) - assert response.data - with zipfile.ZipFile(BytesIO(response.data)) as zip_file: + dataset = self._test_export_dataset( + username, + jid, + api_version=api_version, + format=anno_format, + ) + + with zipfile.ZipFile(BytesIO(dataset)) as zip_file: assert ( len(zip_file.namelist()) == values_to_be_checked["job_size"] + 1 ) # images + annotation file diff --git a/tests/python/rest_api/test_labels.py b/tests/python/rest_api/test_labels.py index dfcae7a8635..f55b7209303 100644 --- a/tests/python/rest_api/test_labels.py +++ b/tests/python/rest_api/test_labels.py @@ -252,7 +252,7 @@ def _get_field_samples(self, field: str) -> Tuple[Any, List[Dict[str, Any]]]: ("name", "job_id", "task_id", "project_id", "type", "color"), ) def test_can_use_simple_filter_for_object_list(self, field): - return super().test_can_use_simple_filter_for_object_list(field) + return super()._test_can_use_simple_filter_for_object_list(field) @pytest.mark.parametrize( "key1, key2", itertools.combinations(["job_id", "task_id", "project_id"], 2) diff --git a/tests/python/rest_api/test_memberships.py b/tests/python/rest_api/test_memberships.py index 30790e2a79e..98f145cbd84 100644 --- a/tests/python/rest_api/test_memberships.py +++ b/tests/python/rest_api/test_memberships.py @@ -73,7 +73,7 @@ def _get_endpoint(self, api_client: ApiClient) -> Endpoint: ("role", "user"), ) def test_can_use_simple_filter_for_object_list(self, field): - return super().test_can_use_simple_filter_for_object_list(field) + return super()._test_can_use_simple_filter_for_object_list(field) @pytest.mark.usefixtures("restore_db_per_function") diff --git a/tests/python/rest_api/test_organizations.py b/tests/python/rest_api/test_organizations.py index 460379e2cab..915e0ac1324 100644 --- a/tests/python/rest_api/test_organizations.py +++ b/tests/python/rest_api/test_organizations.py @@ -121,7 +121,7 @@ def _get_endpoint(self, api_client: ApiClient) -> Endpoint: ("name", "owner", "slug"), ) def test_can_use_simple_filter_for_object_list(self, field): - return super().test_can_use_simple_filter_for_object_list(field) + return super()._test_can_use_simple_filter_for_object_list(field) @pytest.mark.usefixtures("restore_db_per_function") diff --git a/tests/python/rest_api/test_projects.py b/tests/python/rest_api/test_projects.py index c81d70c0b73..31f4bf8023a 100644 --- a/tests/python/rest_api/test_projects.py +++ b/tests/python/rest_api/test_projects.py @@ -14,11 +14,12 @@ from itertools import product from operator import itemgetter from time import sleep -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Tuple, Union import pytest from cvat_sdk.api_client import ApiClient, Configuration, models from cvat_sdk.api_client.api_client import Endpoint +from cvat_sdk.api_client.exceptions import ForbiddenException from cvat_sdk.core.helpers import get_paginated_collection from deepdiff import DeepDiff from PIL import Image @@ -32,7 +33,7 @@ post_method, ) -from .utils import CollectionSimpleFilterTestBase, export_dataset +from .utils import CollectionSimpleFilterTestBase, export_project_backup, export_project_dataset @pytest.mark.usefixtures("restore_db_per_class") @@ -182,138 +183,182 @@ def _get_endpoint(self, api_client: ApiClient) -> Endpoint: ), ) def test_can_use_simple_filter_for_object_list(self, field): - return super().test_can_use_simple_filter_for_object_list(field) - - -class TestGetProjectBackup: - def _test_can_get_project_backup(self, username, pid, **kwargs): - for _ in range(30): - response = get_method(username, f"projects/{pid}/backup", **kwargs) - response.raise_for_status() - if response.status_code == HTTPStatus.CREATED: - break - sleep(1) - response = get_method(username, f"projects/{pid}/backup", action="download", **kwargs) - assert response.status_code == HTTPStatus.OK + return super()._test_can_use_simple_filter_for_object_list(field) - def _test_cannot_get_project_backup(self, username, pid, **kwargs): - response = get_method(username, f"projects/{pid}/backup", **kwargs) - assert response.status_code == HTTPStatus.FORBIDDEN - def test_admin_can_get_project_backup(self, projects): - project = list(projects)[0] - self._test_can_get_project_backup("admin1", project["id"]) +class TestGetPostProjectBackup: + + @pytest.fixture(autouse=True) + def setup(self, projects): + self.projects = projects + + def _test_can_get_project_backup( + self, + username: str, + pid: int, + *, + api_version: int, + local_download: bool = True, + **kwargs, + ) -> Optional[bytes]: + backup = export_project_backup(username, id=pid, api_version=api_version, **kwargs) + if local_download: + assert zipfile.is_zipfile(io.BytesIO(backup)) + else: + assert backup is None + return backup + + def _test_cannot_get_project_backup( + self, + username: str, + pid: int, + api_version: int, + **kwargs, + ): + with pytest.raises(ForbiddenException): + export_project_backup(username, api_version, id=pid, expect_forbidden=True, **kwargs) + + @pytest.mark.parametrize("api_version", (1, 2)) + def test_admin_can_get_project_backup(self, api_version: int): + project = list(self.projects)[0] + self._test_can_get_project_backup("admin1", project["id"], api_version=api_version) # User that not in [project:owner, project:assignee] cannot get project backup. - def test_user_cannot_get_project_backup(self, find_users, projects, is_project_staff): + @pytest.mark.parametrize("api_version", (1, 2)) + def test_user_cannot_get_project_backup(self, find_users, is_project_staff, api_version: int): users = find_users(exclude_privilege="admin") user, project = next( (user, project) - for user, project in product(users, projects) + for user, project in product(users, self.projects) if not is_project_staff(user["id"], project["id"]) ) - self._test_cannot_get_project_backup(user["username"], project["id"]) + self._test_cannot_get_project_backup( + user["username"], project["id"], api_version=api_version + ) # Org worker that not in [project:owner, project:assignee] cannot get project backup. + @pytest.mark.parametrize("api_version", (1, 2)) def test_org_worker_cannot_get_project_backup( - self, find_users, projects, is_project_staff, is_org_member + self, find_users, is_project_staff, is_org_member, api_version: int ): users = find_users(role="worker", exclude_privilege="admin") user, project = next( (user, project) - for user, project in product(users, projects) + for user, project in product(users, self.projects) if not is_project_staff(user["id"], project["id"]) and project["organization"] and is_org_member(user["id"], project["organization"]) ) - self._test_cannot_get_project_backup(user["username"], project["id"]) + self._test_cannot_get_project_backup( + user["username"], project["id"], api_version=api_version + ) # Org worker that in [project:owner, project:assignee] can get project backup. + @pytest.mark.parametrize("api_version", (1, 2)) def test_org_worker_can_get_project_backup( - self, find_users, projects, is_project_staff, is_org_member + self, + find_users, + is_project_staff, + is_org_member, + api_version: int, ): users = find_users(role="worker", exclude_privilege="admin") user, project = next( (user, project) - for user, project in product(users, projects) + for user, project in product(users, self.projects) if is_project_staff(user["id"], project["id"]) and project["organization"] and is_org_member(user["id"], project["organization"]) ) - self._test_can_get_project_backup(user["username"], project["id"]) + self._test_can_get_project_backup(user["username"], project["id"], api_version=api_version) # Org supervisor that in [project:owner, project:assignee] can get project backup. + @pytest.mark.parametrize("api_version", (1, 2)) def test_org_supervisor_can_get_project_backup( - self, find_users, projects, is_project_staff, is_org_member + self, find_users, is_project_staff, is_org_member, api_version: int ): users = find_users(role="supervisor", exclude_privilege="admin") user, project = next( (user, project) - for user, project in product(users, projects) + for user, project in product(users, self.projects) if is_project_staff(user["id"], project["id"]) and project["organization"] and is_org_member(user["id"], project["organization"]) ) - self._test_can_get_project_backup(user["username"], project["id"]) + self._test_can_get_project_backup(user["username"], project["id"], api_version=api_version) # Org supervisor that not in [project:owner, project:assignee] cannot get project backup. + @pytest.mark.parametrize("api_version", (1, 2)) def test_org_supervisor_cannot_get_project_backup( - self, find_users, projects, is_project_staff, is_org_member + self, + find_users, + is_project_staff, + is_org_member, + api_version: int, ): users = find_users(exclude_privilege="admin") user, project = next( (user, project) - for user, project in product(users, projects) + for user, project in product(users, self.projects) if not is_project_staff(user["id"], project["id"]) and project["organization"] and is_org_member(user["id"], project["organization"], role="supervisor") ) - self._test_cannot_get_project_backup(user["username"], project["id"]) + self._test_cannot_get_project_backup( + user["username"], project["id"], api_version=api_version + ) # Org maintainer that not in [project:owner, project:assignee] can get project backup. + @pytest.mark.parametrize("api_version", (1, 2)) def test_org_maintainer_can_get_project_backup( - self, find_users, projects, is_project_staff, is_org_member + self, + find_users, + is_project_staff, + is_org_member, + api_version: int, ): users = find_users(role="maintainer", exclude_privilege="admin") user, project = next( (user, project) - for user, project in product(users, projects) + for user, project in product(users, self.projects) if not is_project_staff(user["id"], project["id"]) and project["organization"] and is_org_member(user["id"], project["organization"]) ) - self._test_can_get_project_backup(user["username"], project["id"]) + self._test_can_get_project_backup(user["username"], project["id"], api_version=api_version) # Org owner that not in [project:owner, project:assignee] can get project backup. + @pytest.mark.parametrize("api_version", (1, 2)) def test_org_owner_can_get_project_backup( - self, find_users, projects, is_project_staff, is_org_member + self, find_users, is_project_staff, is_org_member, api_version: int ): users = find_users(role="owner", exclude_privilege="admin") user, project = next( (user, project) - for user, project in product(users, projects) + for user, project in product(users, self.projects) if not is_project_staff(user["id"], project["id"]) and project["organization"] and is_org_member(user["id"], project["organization"]) ) - self._test_can_get_project_backup(user["username"], project["id"]) + self._test_can_get_project_backup(user["username"], project["id"], api_version=api_version) - def test_can_get_backup_project_when_some_tasks_have_no_data(self, projects): - project = next((p for p in projects if 0 < p["tasks"]["count"])) + @pytest.mark.parametrize("api_version", (1, 2)) + def test_can_get_backup_project_when_some_tasks_have_no_data(self, api_version: int): + project = next((p for p in self.projects if 0 < p["tasks"]["count"])) # add empty task to project response = post_method( @@ -321,10 +366,11 @@ def test_can_get_backup_project_when_some_tasks_have_no_data(self, projects): ) assert response.status_code == HTTPStatus.CREATED - self._test_can_get_project_backup("admin1", project["id"]) + self._test_can_get_project_backup("admin1", project["id"], api_version=api_version) - def test_can_get_backup_project_when_all_tasks_have_no_data(self, projects): - project = next((p for p in projects if 0 == p["tasks"]["count"])) + @pytest.mark.parametrize("api_version", (1, 2)) + def test_can_get_backup_project_when_all_tasks_have_no_data(self, api_version: int): + project = next((p for p in self.projects if 0 == p["tasks"]["count"])) # add empty tasks to empty project response = post_method( @@ -337,11 +383,32 @@ def test_can_get_backup_project_when_all_tasks_have_no_data(self, projects): ) assert response.status_code == HTTPStatus.CREATED - self._test_can_get_project_backup("admin1", project["id"]) + self._test_can_get_project_backup("admin1", project["id"], api_version=api_version) + + @pytest.mark.parametrize("api_version", (1, 2)) + def test_can_get_backup_for_empty_project(self, api_version: int): + empty_project = next((p for p in self.projects if 0 == p["tasks"]["count"])) + self._test_can_get_project_backup("admin1", empty_project["id"], api_version=api_version) - def test_can_get_backup_for_empty_project(self, projects): - empty_project = next((p for p in projects if 0 == p["tasks"]["count"])) - self._test_can_get_project_backup("admin1", empty_project["id"]) + @pytest.mark.parametrize("api_version", (1, 2)) + def test_admin_can_get_project_backup_and_create_project_by_backup( + self, admin_user: str, api_version: int + ): + project_id = 5 + backup = self._test_can_get_project_backup(admin_user, project_id, api_version=api_version) + + tmp_file = io.BytesIO(backup) + tmp_file.name = "dataset.zip" + + import_data = { + "project_file": tmp_file, + } + + with make_api_client(admin_user) as api_client: + (_, response) = api_client.projects_api.create_backup( + backup_write_request=deepcopy(import_data), _content_type="multipart/form-data" + ) + assert response.status == HTTPStatus.ACCEPTED @pytest.mark.usefixtures("restore_db_per_function") @@ -533,18 +600,41 @@ def _check_cvat_for_video_project_annotations_meta(content, values_to_be_checked @pytest.mark.usefixtures("restore_db_per_function") +@pytest.mark.usefixtures("restore_redis_inmem_per_function") class TestImportExportDatasetProject: - def _test_export_project(self, username: str, pid: int, **kwargs): - with make_api_client(username) as api_client: - return export_dataset( - api_client.projects_api.retrieve_dataset_endpoint, id=pid, **kwargs - ) - def _export_annotations(self, username: str, pid: int, **kwargs): - with make_api_client(username) as api_client: - return export_dataset( - api_client.projects_api.retrieve_annotations_endpoint, id=pid, **kwargs - ) + @pytest.fixture(autouse=True) + def setup(self, projects): + self.projects = projects + + @staticmethod + def _test_export_dataset( + username: str, + pid: int, + *, + api_version: Union[int, Tuple[int]], + local_download: bool = True, + **kwargs, + ) -> Optional[bytes]: + dataset = export_project_dataset(username, api_version, save_images=True, id=pid, **kwargs) + if local_download: + assert zipfile.is_zipfile(io.BytesIO(dataset)) + else: + assert dataset is None + + return dataset + + @staticmethod + def _test_export_annotations( + username: str, pid: int, *, api_version: int, local_download: bool = True, **kwargs + ) -> Optional[bytes]: + dataset = export_project_dataset(username, api_version, save_images=False, id=pid, **kwargs) + if local_download: + assert zipfile.is_zipfile(io.BytesIO(dataset)) + else: + assert dataset is None + + return dataset def _test_import_project(self, username, project_id, format_name, data): with make_api_client(username) as api_client: @@ -558,13 +648,19 @@ def _test_import_project(self, username, project_id, format_name, data): rq_id = json.loads(response.data).get("rq_id") assert rq_id, "The rq_id was not found in the response" - while True: - # TODO: It's better be refactored to a separate endpoint to get request status - (_, response) = api_client.projects_api.retrieve_dataset( - project_id, action="import_status", rq_id=rq_id - ) - if response.status == HTTPStatus.CREATED: + for _ in range(50): + (background_request, response) = api_client.requests_api.retrieve(rq_id) + assert response.status == HTTPStatus.OK + if ( + background_request.status.value + == models.RequestStatus.allowed_values[("value",)]["FINISHED"] + ): break + sleep(0.1) + else: + assert ( + False + ), f"Import process was not finished, last status: {background_request.status.value}" def _test_get_annotations_from_task(self, username, task_id): with make_api_client(username) as api_client: @@ -574,12 +670,16 @@ def _test_get_annotations_from_task(self, username, task_id): response_data = json.loads(response.data) return response_data - def test_can_import_dataset_in_org(self, admin_user): + def test_can_import_dataset_in_org(self, admin_user: str): project_id = 4 - response = self._test_export_project(admin_user, project_id) + dataset = self._test_export_dataset( + admin_user, + project_id, + api_version=2, + ) - tmp_file = io.BytesIO(response.data) + tmp_file = io.BytesIO(dataset) tmp_file.name = "dataset.zip" import_data = { @@ -606,15 +706,23 @@ def test_can_export_and_import_dataset_with_skeletons( for element in annotations["task"][task_id]["shapes"] if element["type"] == "skeleton" ] - project_id = next( - task["project_id"] - for task in tasks - if task["id"] in tasks_with_skeletons and task["project_id"] is not None - ) + for task in tasks: + if task["id"] in tasks_with_skeletons and task["project_id"] is not None: + project_id = task["project_id"] + project = next(p for p in self.projects if p["id"] == project_id) + if (project["target_storage"] or {}).get("location") == "local": + break + else: + assert False, "Can't find suitable project" - response = self._test_export_project(admin_user, project_id, format=export_format) + dataset = self._test_export_dataset( + admin_user, + project_id, + api_version=2, + format=export_format, + ) - tmp_file = io.BytesIO(response.data) + tmp_file = io.BytesIO(dataset) tmp_file.name = "dataset.zip" import_data = { "dataset_file": tmp_file, @@ -622,45 +730,23 @@ def test_can_export_and_import_dataset_with_skeletons( self._test_import_project(admin_user, project_id, import_format, import_data) - def _test_can_get_project_backup(self, username, pid, **kwargs): - for _ in range(30): - response = get_method(username, f"projects/{pid}/backup", **kwargs) - response.raise_for_status() - if response.status_code == HTTPStatus.CREATED: - break - sleep(1) - response = get_method(username, f"projects/{pid}/backup", action="download", **kwargs) - assert response.status_code == HTTPStatus.OK - return response - - def test_admin_can_get_project_backup_and_create_project_by_backup(self, admin_user): - project_id = 5 - response = self._test_can_get_project_backup(admin_user, project_id) - - tmp_file = io.BytesIO(response.content) - tmp_file.name = "dataset.zip" - - import_data = { - "project_file": tmp_file, - } - - with make_api_client(admin_user) as api_client: - (_, response) = api_client.projects_api.create_backup( - backup_write_request=deepcopy(import_data), _content_type="multipart/form-data" - ) - assert response.status == HTTPStatus.ACCEPTED - + @pytest.mark.parametrize("api_version", (1, 2)) @pytest.mark.parametrize("format_name", ("Datumaro 1.0", "ImageNet 1.0", "PASCAL VOC 1.1")) - def test_can_import_export_dataset_with_some_format(self, format_name): + def test_can_import_export_dataset_with_some_format(self, format_name: str, api_version: int): # https://github.com/cvat-ai/cvat/issues/4410 # https://github.com/cvat-ai/cvat/issues/4850 # https://github.com/cvat-ai/cvat/issues/4621 username = "admin1" project_id = 4 - response = self._test_export_project(username, project_id, format=format_name) + dataset = self._test_export_dataset( + username, + project_id, + api_version=api_version, + format=format_name, + ) - tmp_file = io.BytesIO(response.data) + tmp_file = io.BytesIO(dataset) tmp_file.name = "dataset.zip" import_data = { @@ -669,6 +755,27 @@ def test_can_import_export_dataset_with_some_format(self, format_name): self._test_import_project(username, project_id, format_name, import_data) + @pytest.mark.parametrize("api_version", product((1, 2), repeat=2)) + @pytest.mark.parametrize( + "local_download", (True, pytest.param(False, marks=pytest.mark.with_external_services)) + ) + def test_can_export_dataset_locally_and_to_cloud_with_both_api_versions( + self, admin_user: str, filter_projects, api_version: Tuple[int], local_download: bool + ): + filter_ = "target_storage__location" + if local_download: + filter_ = "exclude_" + filter_ + + pid = filter_projects(**{filter_: "cloud_storage"})[0]["id"] + + self._test_export_dataset( + admin_user, + pid, + api_version=api_version, + local_download=local_download, + ) + + @pytest.mark.parametrize("api_version", (1, 2)) @pytest.mark.parametrize("username, pid", [("admin1", 8)]) @pytest.mark.parametrize( "anno_format, anno_file_name, check_func", @@ -688,10 +795,9 @@ def test_exported_project_dataset_structure( anno_file_name, check_func, tasks, - projects, - annotations, + api_version: int, ): - project = projects[pid] + project = self.projects[pid] values_to_be_checked = { "pid": str(pid), @@ -708,20 +814,30 @@ def test_exported_project_dataset_structure( ], } - response = self._export_annotations(username, pid, format=anno_format) - assert response.data - with zipfile.ZipFile(BytesIO(response.data)) as zip_file: + dataset = self._test_export_annotations( + username, + pid, + api_version=api_version, + format=anno_format, + ) + + with zipfile.ZipFile(BytesIO(dataset)) as zip_file: content = zip_file.read(anno_file_name) check_func(content, values_to_be_checked) - def test_can_import_export_annotations_with_rotation(self): + @pytest.mark.parametrize("api_version", (1, 2)) + def test_can_import_export_annotations_with_rotation(self, api_version: int): # https://github.com/cvat-ai/cvat/issues/4378 username = "admin1" project_id = 4 - response = self._test_export_project(username, project_id) + dataset = self._test_export_dataset( + username, + project_id, + api_version=api_version, + ) - tmp_file = io.BytesIO(response.data) + tmp_file = io.BytesIO(dataset) tmp_file.name = "dataset.zip" import_data = { @@ -741,20 +857,39 @@ def test_can_import_export_annotations_with_rotation(self): assert task1_rotation == task2_rotation - def test_can_export_dataset_with_skeleton_labels_with_spaces(self): + @pytest.mark.parametrize("api_version", (1, 2)) + def test_can_export_dataset_with_skeleton_labels_with_spaces(self, api_version: int): # https://github.com/cvat-ai/cvat/issues/5257 # https://github.com/cvat-ai/cvat/issues/5600 username = "admin1" project_id = 11 - self._test_export_project(username, project_id, format="COCO Keypoints 1.0") + self._test_export_dataset( + username, + project_id, + api_version=api_version, + format="COCO Keypoints 1.0", + ) - def test_can_export_dataset_for_empty_project(self, projects): - empty_project = next((p for p in projects if 0 == p["tasks"]["count"])) - self._test_export_project("admin1", empty_project["id"], format="COCO 1.0") + @pytest.mark.parametrize("api_version", (1, 2)) + def test_can_export_dataset_for_empty_project(self, filter_projects, api_version: int): + empty_project = filter_projects( + tasks__count=0, exclude_target_storage__location="cloud_storage" + )[0] + self._test_export_dataset( + "admin1", + empty_project["id"], + api_version=api_version, + format="COCO 1.0", + ) - def test_can_export_project_dataset_when_some_tasks_have_no_data(self, projects): - project = next((p for p in projects if 0 < p["tasks"]["count"])) + @pytest.mark.parametrize("api_version", (1, 2)) + def test_can_export_project_dataset_when_some_tasks_have_no_data( + self, filter_projects, api_version: int + ): + project = filter_projects( + exclude_tasks__count=0, exclude_target_storage__location="cloud_storage" + )[0] # add empty task to project response = post_method( @@ -762,10 +897,20 @@ def test_can_export_project_dataset_when_some_tasks_have_no_data(self, projects) ) assert response.status_code == HTTPStatus.CREATED - self._test_export_project("admin1", project["id"], format="COCO 1.0") + self._test_export_dataset( + "admin1", + project["id"], + api_version=api_version, + format="COCO 1.0", + ) - def test_can_export_project_dataset_when_all_tasks_have_no_data(self, projects): - project = next((p for p in projects if 0 == p["tasks"]["count"])) + @pytest.mark.parametrize("api_version", (1, 2)) + def test_can_export_project_dataset_when_all_tasks_have_no_data( + self, filter_projects, api_version: int + ): + project = filter_projects(tasks__count=0, exclude_target_storage__location="cloud_storage")[ + 0 + ] # add empty tasks to empty project response = post_method( @@ -778,21 +923,26 @@ def test_can_export_project_dataset_when_all_tasks_have_no_data(self, projects): ) assert response.status_code == HTTPStatus.CREATED - self._test_export_project("admin1", project["id"], format="COCO 1.0") + self._test_export_dataset( + "admin1", + project["id"], + api_version=api_version, + format="COCO 1.0", + ) - @pytest.mark.parametrize("cloud_storage_id", [2]) + @pytest.mark.parametrize("api_version", (1, 2)) + @pytest.mark.parametrize("cloud_storage_id", [3]) # import/export bucket def test_can_export_and_import_dataset_after_deleting_related_storage( - self, admin_user, projects, cloud_storage_id: int + self, admin_user, cloud_storage_id: int, api_version: int ): - project = next( + project_id = next( p - for p in projects + for p in self.projects if p["source_storage"] and p["source_storage"]["cloud_storage_id"] == cloud_storage_id and p["target_storage"] and p["target_storage"]["cloud_storage_id"] == cloud_storage_id - ) - project_id = project["id"] + )["id"] with make_api_client(admin_user) as api_client: _, response = api_client.cloudstorages_api.destroy(cloud_storage_id) @@ -801,9 +951,9 @@ def test_can_export_and_import_dataset_after_deleting_related_storage( result, response = api_client.projects_api.retrieve(project_id) assert all([not getattr(result, field) for field in ("source_storage", "target_storage")]) - response = self._test_export_project(admin_user, project_id) + dataset = self._test_export_dataset(admin_user, project_id, api_version=api_version) - with io.BytesIO(response.data) as tmp_file: + with io.BytesIO(dataset) as tmp_file: tmp_file.name = "dataset.zip" import_data = { "dataset_file": tmp_file, @@ -836,11 +986,13 @@ def test_can_export_and_import_dataset_after_deleting_related_storage( ("Open Images V6 1.0", "images/{subset}/"), ], ) + @pytest.mark.parametrize("api_version", (1, 2)) def test_creates_subfolders_for_subsets_on_export( - self, tasks, admin_user, export_format, subset_path_template + self, filter_tasks, admin_user, export_format, subset_path_template, api_version: int ): group_key_func = itemgetter("project_id") subsets = ["Train", "Validation"] + tasks = filter_tasks(exclude_target_storage__location="cloud_storage") project_id = next( project_id for project_id, group in itertools.groupby( @@ -849,8 +1001,10 @@ def test_creates_subfolders_for_subsets_on_export( ) if sorted(task["subset"] for task in group) == subsets ) - response = self._test_export_project(admin_user, project_id, format=export_format) - with zipfile.ZipFile(io.BytesIO(response.data)) as zip_file: + dataset = self._test_export_dataset( + admin_user, project_id, api_version=api_version, format=export_format + ) + with zipfile.ZipFile(io.BytesIO(dataset)) as zip_file: for subset in subsets: folder_prefix = subset_path_template.format(subset=subset) assert ( diff --git a/tests/python/rest_api/test_quality_control.py b/tests/python/rest_api/test_quality_control.py index 925667d5d4b..6775bfe7eaf 100644 --- a/tests/python/rest_api/test_quality_control.py +++ b/tests/python/rest_api/test_quality_control.py @@ -616,7 +616,7 @@ def _get_field_samples(self, field: str) -> Tuple[Any, List[Dict[str, Any]]]: ("task_id", "job_id", "parent_id", "target", "org_id"), ) def test_can_use_simple_filter_for_object_list(self, field): - return super().test_can_use_simple_filter_for_object_list(field) + return super()._test_can_use_simple_filter_for_object_list(field) @pytest.mark.usefixtures("restore_db_per_class") @@ -801,7 +801,7 @@ def _get_field_samples(self, field: str) -> Tuple[Any, List[Dict[str, Any]]]: ("report_id", "severity", "type", "frame", "job_id", "task_id", "org_id"), ) def test_can_use_simple_filter_for_object_list(self, field): - return super().test_can_use_simple_filter_for_object_list(field) + return super()._test_can_use_simple_filter_for_object_list(field) class TestSimpleQualitySettingsFilters(CollectionSimpleFilterTestBase): @@ -815,7 +815,7 @@ def _get_endpoint(self, api_client: ApiClient) -> Endpoint: @pytest.mark.parametrize("field", ("task_id",)) def test_can_use_simple_filter_for_object_list(self, field): - return super().test_can_use_simple_filter_for_object_list(field) + return super()._test_can_use_simple_filter_for_object_list(field) @pytest.mark.usefixtures("restore_db_per_class") diff --git a/tests/python/rest_api/test_requests.py b/tests/python/rest_api/test_requests.py new file mode 100644 index 00000000000..f06e97ae7fb --- /dev/null +++ b/tests/python/rest_api/test_requests.py @@ -0,0 +1,337 @@ +# Copyright (C) 2024 CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +import io +from http import HTTPStatus +from typing import List +from urllib.parse import urlparse + +import pytest +from cvat_sdk.api_client import ApiClient, models +from cvat_sdk.api_client.api_client import Endpoint +from cvat_sdk.core.helpers import get_paginated_collection + +from shared.utils.config import make_api_client +from shared.utils.helpers import generate_image_files + +from .utils import ( + CollectionSimpleFilterTestBase, + create_task, + export_job_dataset, + export_project_backup, + export_project_dataset, + export_task_backup, + export_task_dataset, + import_task_backup, +) + + +@pytest.mark.usefixtures("restore_db_per_class") +@pytest.mark.usefixtures("restore_redis_inmem_per_function") +@pytest.mark.timeout(30) +class TestRequestsListFilters(CollectionSimpleFilterTestBase): + + field_lookups = { + "target": ["operation", "target"], + "subresource": ["operation", "type", lambda x: x.split(":")[1]], + "action": ["operation", "type", lambda x: x.split(":")[0]], + "project_id": ["operation", "project_id"], + "task_id": ["operation", "task_id"], + "job_id": ["operation", "job_id"], + "format": ["operation", "format"], + } + + def _get_endpoint(self, api_client: ApiClient) -> Endpoint: + return api_client.requests_api.list_endpoint + + @pytest.fixture(autouse=True) + def setup(self, find_users): + self.user = find_users(privilege="user")[0]["username"] + + @pytest.fixture + def fxt_resources_ids(self): + with make_api_client(self.user) as api_client: + project_ids = [ + api_client.projects_api.create( + {"name": f"Test project {idx + 1}", "labels": [{"name": "car"}]} + )[0].id + for idx in range(3) + ] + + task_ids = [ + create_task( + self.user, + spec={"name": f"Test task {idx + 1}", "labels": [{"name": "car"}]}, + data={ + "image_quality": 75, + "client_files": generate_image_files(2), + "segment_size": 1, + }, + )[0] + for idx in range(3) + ] + + job_ids = [] + for task_id in task_ids: + jobs, _ = api_client.jobs_api.list(task_id=task_id) + job_ids.extend([j.id for j in jobs.results]) + + return project_ids, task_ids, job_ids + + @pytest.fixture + def fxt_make_requests( + self, + fxt_make_export_project_requests, + fxt_make_export_task_requests, + fxt_make_export_job_requests, + fxt_download_file, + ): + def _make_requests(project_ids: List[int], task_ids: List[int], job_ids: List[int]): + # make requests to export projects|tasks|jobs annotations|datasets|backups + fxt_make_export_project_requests(project_ids[1:]) + fxt_make_export_task_requests(task_ids[1:]) + fxt_make_export_job_requests(job_ids[1:]) + + # make requests to download files and then import them + for resource_type, first_resource in zip( + ("project", "task", "job"), (project_ids[0], task_ids[0], job_ids[0]) + ): + for subresource in ("dataset", "annotations", "backup"): + if resource_type == "job" and subresource == "backup": + continue + + data = fxt_download_file(resource_type, first_resource, subresource) + + tmp_file = io.BytesIO(data) + tmp_file.name = f"{resource_type}_{subresource}.zip" + + if resource_type == "task" and subresource == "backup": + import_task_backup( + self.user, + data={ + "task_file": tmp_file, + }, + ) + + empty_file = io.BytesIO(b"empty_file") + empty_file.name = "empty.zip" + + # import corrupted backup + import_task_backup( + self.user, + data={ + "task_file": empty_file, + }, + ) + + return _make_requests + + @pytest.fixture + def fxt_download_file(self): + def download_file(resource: str, rid: int, subresource: str): + func = { + ("project", "dataset"): lambda *args, **kwargs: export_project_dataset( + *args, **kwargs, save_images=True + ), + ("project", "annotations"): lambda *args, **kwargs: export_project_dataset( + *args, **kwargs, save_images=False + ), + ("project", "backup"): export_project_backup, + ("task", "dataset"): lambda *args, **kwargs: export_task_dataset( + *args, **kwargs, save_images=True + ), + ("task", "annotations"): lambda *args, **kwargs: export_task_dataset( + *args, **kwargs, save_images=False + ), + ("task", "backup"): export_task_backup, + ("job", "dataset"): lambda *args, **kwargs: export_job_dataset( + *args, **kwargs, save_images=True + ), + ("job", "annotations"): lambda *args, **kwargs: export_job_dataset( + *args, **kwargs, save_images=False + ), + }[(resource, subresource)] + + data = func(self.user, api_version=2, id=rid, download_result=True) + assert data, f"Failed to download {resource} {subresource} locally" + return data + + return download_file + + @pytest.fixture + def fxt_make_export_project_requests(self): + def make_requests(project_ids: List[int]): + for project_id in project_ids: + export_project_backup( + self.user, api_version=2, id=project_id, download_result=False + ) + export_project_dataset( + self.user, api_version=2, save_images=True, id=project_id, download_result=False + ) + export_project_dataset( + self.user, + api_version=2, + save_images=False, + id=project_id, + download_result=False, + ) + + return make_requests + + @pytest.fixture + def fxt_make_export_task_requests(self): + def make_requests(task_ids: List[int]): + for task_id in task_ids: + export_task_backup(self.user, api_version=2, id=task_id, download_result=False) + export_task_dataset( + self.user, api_version=2, save_images=True, id=task_id, download_result=False + ) + export_task_dataset( + self.user, api_version=2, save_images=False, id=task_id, download_result=False + ) + + return make_requests + + @pytest.fixture + def fxt_make_export_job_requests(self): + def make_requests(job_ids: List[int]): + for job_id in job_ids: + export_job_dataset( + self.user, + api_version=2, + save_images=True, + id=job_id, + format="COCO 1.0", + download_result=False, + ) + export_job_dataset( + self.user, + api_version=2, + save_images=False, + id=job_id, + format="YOLO 1.1", + download_result=False, + ) + + return make_requests + + @pytest.mark.parametrize( + "simple_filter, values", + [ + ("subresource", ["annotations", "dataset", "backup"]), + ("action", ["create", "export", "import"]), + ("status", ["finished", "failed"]), + ("project_id", []), + ("task_id", []), + ("job_id", []), + ("format", ["CVAT for images 1.1", "COCO 1.0", "YOLO 1.1"]), + ("target", ["project", "task", "job"]), + ], + ) + def test_can_use_simple_filter_for_object_list( + self, simple_filter: str, values: List, fxt_resources_ids, fxt_make_requests + ): + project_ids, task_ids, job_ids = fxt_resources_ids + fxt_make_requests(project_ids, task_ids, job_ids) + + if simple_filter in ("project_id", "task_id", "job_id"): + # check last project|task|job + if simple_filter == "project_id": + values = project_ids[-1:] + elif simple_filter == "task_id": + values = task_ids[-1:] + else: + values = job_ids[-1:] + + with make_api_client(self.user) as api_client: + self.samples = get_paginated_collection( + self._get_endpoint(api_client), return_json=True + ) + + return super()._test_can_use_simple_filter_for_object_list(simple_filter, values) + + +@pytest.mark.usefixtures("restore_db_per_class") +@pytest.mark.usefixtures("restore_redis_inmem_per_function") +class TestGetRequests: + + def _test_get_request_200(self, api_client: ApiClient, rq_id: str, **kwargs) -> models.Request: + (background_request, response) = api_client.requests_api.retrieve(rq_id, **kwargs) + assert response.status == HTTPStatus.OK + assert background_request.id == rq_id + + return background_request + + def _test_get_request_403(self, api_client: ApiClient, rq_id: str): + (_, response) = api_client.requests_api.retrieve( + rq_id, _parse_response=False, _check_status=False + ) + assert response.status == HTTPStatus.FORBIDDEN + + @pytest.mark.parametrize("format_name", ("CVAT for images 1.1",)) + @pytest.mark.parametrize("save_images", (True, False)) + def test_owner_can_retrieve_request(self, format_name: str, save_images: bool, projects): + project = next( + ( + p + for p in projects + if p["owner"] and (p["target_storage"] or {}).get("location") == "local" + ) + ) + owner = project["owner"] + + subresource = "dataset" if save_images else "annotations" + export_project_dataset( + owner["username"], + api_version=2, + save_images=save_images, + id=project["id"], + download_result=False, + ) + rq_id = f'export:project-{project["id"]}-{subresource}-in-{format_name.replace(" ", "_").replace(".", "@")}-format-by-{owner["id"]}' + + with make_api_client(owner["username"]) as owner_client: + bg_request = self._test_get_request_200(owner_client, rq_id) + + assert ( + bg_request.created_date + < bg_request.started_date + < bg_request.finished_date + < bg_request.expiry_date + ) + assert bg_request.operation.format == format_name + assert bg_request.operation.project_id == project["id"] + assert bg_request.operation.target.value == "project" + assert bg_request.operation.task_id is None + assert bg_request.operation.job_id is None + assert bg_request.operation.type == f"export:{subresource}" + assert bg_request.owner.id == owner["id"] + assert bg_request.owner.username == owner["username"] + + parsed_url = urlparse(bg_request.result_url) + assert all([parsed_url.scheme, parsed_url.netloc, parsed_url.path, parsed_url.query]) + + @pytest.mark.parametrize("format_name", ("CVAT for images 1.1",)) + def test_non_owner_cannot_retrieve_request(self, find_users, projects, format_name: str): + project = next( + ( + p + for p in projects + if p["owner"] and (p["target_storage"] or {}).get("location") == "local" + ) + ) + owner = project["owner"] + malefactor = find_users(exclude_username=owner["username"])[0] + + export_project_dataset( + owner["username"], + api_version=2, + save_images=True, + id=project["id"], + download_result=False, + ) + rq_id = f'export:project-{project["id"]}-dataset-in-{format_name.replace(" ", "_").replace(".", "@")}-format-by-{owner["id"]}' + + with make_api_client(malefactor["username"]) as malefactor_client: + self._test_get_request_403(malefactor_client, rq_id) diff --git a/tests/python/rest_api/test_tasks.py b/tests/python/rest_api/test_tasks.py index f8f8510fdc7..6cb0ea0568b 100644 --- a/tests/python/rest_api/test_tasks.py +++ b/tests/python/rest_api/test_tasks.py @@ -18,7 +18,7 @@ from pathlib import Path from tempfile import NamedTemporaryFile, TemporaryDirectory from time import sleep, time -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional, Tuple, Union import pytest from cvat_sdk import Client, Config, exceptions @@ -54,7 +54,8 @@ CollectionSimpleFilterTestBase, compare_annotations, create_task, - export_dataset, + export_task_backup, + export_task_dataset, wait_until_task_is_created, ) @@ -275,7 +276,7 @@ def _get_endpoint(self, api_client: ApiClient) -> Endpoint: ), ) def test_can_use_simple_filter_for_object_list(self, field): - return super().test_can_use_simple_filter_for_object_list(field) + return super()._test_can_use_simple_filter_for_object_list(field) @pytest.mark.usefixtures("restore_db_per_function") @@ -653,26 +654,70 @@ def interpolate(frame): @pytest.mark.usefixtures("restore_db_per_class") +@pytest.mark.usefixtures("restore_redis_inmem_per_function") class TestGetTaskDataset: - def _test_export_task(self, username: str, tid: int, **kwargs): - with make_api_client(username) as api_client: - return export_dataset(api_client.tasks_api.retrieve_dataset_endpoint, id=tid, **kwargs) - def test_can_export_task_dataset(self, admin_user, tasks_with_shapes): - task = tasks_with_shapes[0] - response = self._test_export_task(admin_user, task["id"]) - assert response.data + @staticmethod + def _test_can_export_dataset( + username: str, + task_id: int, + *, + api_version: Union[int, Tuple[int]], + local_download: bool = True, + **kwargs, + ) -> Optional[bytes]: + dataset = export_task_dataset(username, api_version, save_images=True, id=task_id, **kwargs) + if local_download: + assert zipfile.is_zipfile(io.BytesIO(dataset)) + else: + assert dataset is None + + return dataset + + @pytest.mark.usefixtures("restore_db_per_function") + @pytest.mark.parametrize("api_version", product((1, 2), repeat=2)) + @pytest.mark.parametrize( + "local_download", (True, pytest.param(False, marks=pytest.mark.with_external_services)) + ) + def test_can_export_task_dataset_locally_and_to_cloud_with_both_api_versions( + self, + admin_user, + tasks_with_shapes, + filter_tasks, + api_version: Tuple[int], + local_download: bool, + ): + filter_ = "target_storage__location" + if local_download: + filter_ = "exclude_" + filter_ + filtered_ids = {t["id"] for t in filter_tasks(**{filter_: "cloud_storage"})} + + task_id = next(iter(filtered_ids & {t["id"] for t in tasks_with_shapes})) + self._test_can_export_dataset( + admin_user, + task_id, + api_version=api_version, + local_download=local_download, + ) + @pytest.mark.parametrize("api_version", (1, 2)) @pytest.mark.parametrize("tid", [21]) @pytest.mark.parametrize( "format_name", ["CVAT for images 1.1", "CVAT for video 1.1", "COCO Keypoints 1.0"] ) - def test_can_export_task_with_several_jobs(self, admin_user, tid, format_name): - response = self._test_export_task(admin_user, tid, format=format_name) - assert response.data + def test_can_export_task_with_several_jobs( + self, admin_user, tid, format_name, api_version: int + ): + self._test_can_export_dataset( + admin_user, + tid, + format=format_name, + api_version=api_version, + ) + @pytest.mark.parametrize("api_version", (1, 2)) @pytest.mark.parametrize("tid", [8]) - def test_can_export_task_to_coco_format(self, admin_user, tid): + def test_can_export_task_to_coco_format(self, admin_user: str, tid: int, api_version: int): # these annotations contains incorrect frame numbers # in order to check that server handle such cases annotations = { @@ -756,9 +801,13 @@ def test_can_export_task_to_coco_format(self, admin_user, tid): ) assert response.status_code == HTTPStatus.OK - # check that we can export task - response = self._test_export_task(admin_user, tid, format="COCO Keypoints 1.0") - assert response.status == HTTPStatus.OK + # check that we can export task dataset + self._test_can_export_dataset( + admin_user, + tid, + format="COCO Keypoints 1.0", + api_version=api_version, + ) # check that server saved track annotations correctly response = get_method(admin_user, f"tasks/{tid}/annotations") @@ -769,8 +818,9 @@ def test_can_export_task_to_coco_format(self, admin_user, tid): assert annotations["tracks"][0]["shapes"][0]["frame"] == 0 assert annotations["tracks"][0]["elements"][0]["shapes"][0]["frame"] == 0 + @pytest.mark.parametrize("api_version", (1, 2)) @pytest.mark.usefixtures("restore_db_per_function") - def test_can_download_task_with_special_chars_in_name(self, admin_user): + def test_can_download_task_with_special_chars_in_name(self, admin_user: str, api_version: int): # Control characters in filenames may conflict with the Content-Disposition header # value restrictions, as it needs to include the downloaded file name. @@ -786,11 +836,14 @@ def test_can_download_task_with_special_chars_in_name(self, admin_user): task_id, _ = create_task(admin_user, task_spec, task_data) - response = self._test_export_task(admin_user, task_id) - assert response.status == HTTPStatus.OK - assert zipfile.is_zipfile(io.BytesIO(response.data)) + dataset = self._test_can_export_dataset(admin_user, task_id, api_version=api_version) + assert zipfile.is_zipfile(io.BytesIO(dataset)) - def test_export_dataset_after_deleting_related_cloud_storage(self, admin_user, tasks): + @pytest.mark.usefixtures("restore_db_per_function") + @pytest.mark.parametrize("api_version", (1, 2)) + def test_export_dataset_after_deleting_related_cloud_storage( + self, admin_user: str, tasks, api_version: int + ): related_field = "target_storage" task = next( @@ -806,8 +859,7 @@ def test_export_dataset_after_deleting_related_cloud_storage(self, admin_user, t result, response = api_client.tasks_api.retrieve(task_id) assert not result[related_field] - response = export_dataset(api_client.tasks_api.retrieve_dataset_endpoint, id=task["id"]) - assert response.data + self._test_can_export_dataset(admin_user, task["id"], api_version=api_version) @pytest.mark.parametrize( "export_format, default_subset_name, subset_path_template", @@ -816,9 +868,17 @@ def test_export_dataset_after_deleting_related_cloud_storage(self, admin_user, t ("YOLO 1.1", "train", "obj_{subset}_data"), ], ) + @pytest.mark.parametrize("api_version", (1, 2)) def test_uses_subset_name( - self, tasks, admin_user, export_format, default_subset_name, subset_path_template + self, + admin_user, + filter_tasks, + export_format, + default_subset_name, + subset_path_template, + api_version: int, ): + tasks = filter_tasks(exclude_target_storage__location="cloud_storage") group_key_func = itemgetter("subset") subsets_and_tasks = [ (subset, next(group)) @@ -828,8 +888,13 @@ def test_uses_subset_name( ) ] for subset_name, task in subsets_and_tasks: - response = self._test_export_task(admin_user, tid=task["id"], format=export_format) - with zipfile.ZipFile(io.BytesIO(response.data)) as zip_file: + dataset = self._test_can_export_dataset( + admin_user, + task["id"], + api_version=api_version, + format=export_format, + ) + with zipfile.ZipFile(io.BytesIO(dataset)) as zip_file: subset_path = subset_path_template.format(subset=subset_name or default_subset_name) assert any( subset_path in path for path in zip_file.namelist() @@ -2223,6 +2288,7 @@ def test_work_with_task_containing_non_stable_cloud_storage_files( assert image_name in ex.body +@pytest.mark.usefixtures("restore_redis_inmem_per_function") class TestTaskBackups: def _make_client(self) -> Client: return Client(BASE_URL, config=Config(status_check_period=0.01)) @@ -2237,6 +2303,23 @@ def setup(self, restore_db_per_function, restore_cvat_data, tmp_path: Path, admi with self.client: self.client.login((self.user, USER_PASS)) + @pytest.mark.parametrize("api_version", product((1, 2), repeat=2)) + @pytest.mark.parametrize( + "local_download", (True, pytest.param(False, marks=pytest.mark.with_external_services)) + ) + def test_can_export_backup_with_both_api_versions( + self, filter_tasks, api_version: Tuple[int], local_download: bool + ): + task = filter_tasks( + **{("exclude_" if local_download else "") + "target_storage__location": "cloud_storage"} + )[0] + backup = export_task_backup(self.user, api_version, id=task["id"]) + + if local_download: + assert zipfile.is_zipfile(io.BytesIO(backup)) + else: + assert backup is None + @pytest.mark.parametrize("mode", ["annotation", "interpolation"]) def test_can_export_backup(self, tasks, mode): task_id = next(t for t in tasks if t["mode"] == mode)["id"] diff --git a/tests/python/rest_api/test_users.py b/tests/python/rest_api/test_users.py index bec563e2e79..eb04149fa7c 100644 --- a/tests/python/rest_api/test_users.py +++ b/tests/python/rest_api/test_users.py @@ -112,4 +112,4 @@ def _get_endpoint(self, api_client: ApiClient) -> Endpoint: ("is_active", "username"), ) def test_can_use_simple_filter_for_object_list(self, field): - return super().test_can_use_simple_filter_for_object_list(field) + return super()._test_can_use_simple_filter_for_object_list(field) diff --git a/tests/python/rest_api/test_webhooks.py b/tests/python/rest_api/test_webhooks.py index a29bc0c19a9..3c528bc78c1 100644 --- a/tests/python/rest_api/test_webhooks.py +++ b/tests/python/rest_api/test_webhooks.py @@ -565,7 +565,7 @@ def _get_endpoint(self, api_client: ApiClient) -> Endpoint: ("target_url", "owner", "type", "project_id"), ) def test_can_use_simple_filter_for_object_list(self, field): - return super().test_can_use_simple_filter_for_object_list(field) + return super()._test_can_use_simple_filter_for_object_list(field) @pytest.mark.usefixtures("restore_db_per_class") diff --git a/tests/python/rest_api/utils.py b/tests/python/rest_api/utils.py index ea1266ac5aa..e3fbc9d1e97 100644 --- a/tests/python/rest_api/utils.py +++ b/tests/python/rest_api/utils.py @@ -2,31 +2,54 @@ # # SPDX-License-Identifier: MIT +import json from abc import ABCMeta, abstractmethod from copy import deepcopy from http import HTTPStatus from time import sleep -from typing import Any, Dict, Iterator, List, Optional, Sequence, Tuple, Union +from typing import Any, Callable, Dict, Iterator, List, Optional, Sequence, Tuple, Union +import requests from cvat_sdk.api_client import apis, models +from cvat_sdk.api_client.api.jobs_api import JobsApi +from cvat_sdk.api_client.api.projects_api import ProjectsApi +from cvat_sdk.api_client.api.tasks_api import TasksApi from cvat_sdk.api_client.api_client import ApiClient, Endpoint +from cvat_sdk.api_client.exceptions import ForbiddenException from cvat_sdk.core.helpers import get_paginated_collection from deepdiff import DeepDiff -from urllib3 import HTTPResponse from shared.utils.config import make_api_client -def export_dataset( +def initialize_export(endpoint: Endpoint, *, expect_forbidden: bool = False, **kwargs) -> str: + (_, response) = endpoint.call_with_http_info( + **kwargs, _parse_response=False, _check_status=False + ) + if expect_forbidden: + assert ( + response.status == HTTPStatus.FORBIDDEN + ), f"Request should be forbidden, status: {response.status}" + raise ForbiddenException() + + assert response.status == HTTPStatus.ACCEPTED, f"Status: {response.status}" + + # define background request ID returned in the server response + rq_id = json.loads(response.data).get("rq_id") + assert rq_id, "The rq_id parameter was not found in the server response" + return rq_id + + +def wait_and_download_v1( endpoint: Endpoint, *, - max_retries: int = 20, + max_retries: int = 30, interval: float = 0.1, - format: str = "CVAT for images 1.1", # pylint: disable=redefined-builtin + download_result: bool = True, **kwargs, -) -> HTTPResponse: +) -> Optional[bytes]: for _ in range(max_retries): - (_, response) = endpoint.call_with_http_info(**kwargs, format=format, _parse_response=False) + (_, response) = endpoint.call_with_http_info(**kwargs, _parse_response=False) if response.status in (HTTPStatus.CREATED, HTTPStatus.OK): break assert response.status == HTTPStatus.ACCEPTED @@ -36,16 +59,337 @@ def export_dataset( False ), f"Export process was not finished within allowed time ({interval * max_retries}, sec)" + if not download_result: + return None + if response.status == HTTPStatus.CREATED: (_, response) = endpoint.call_with_http_info( - **kwargs, format=format, action="download", _parse_response=False + **kwargs, action="download", _parse_response=False ) assert response.status == HTTPStatus.OK - return response + return response.data or None # return None when export was on cloud storage + + +def export_v1( + endpoint: Endpoint, + *, + max_retries: int = 30, + interval: float = 0.1, + expect_forbidden: bool = False, + wait_result: bool = True, + download_result: bool = True, + **kwargs, +) -> Optional[bytes]: + """Export datasets|annotations|backups using first version of export API + + Args: + endpoint (Endpoint): Export endpoint, will be called to initialize export process and to check status + max_retries (int, optional): Number of retries when checking process status. Defaults to 30. + interval (float, optional): Interval in seconds between retries. Defaults to 0.1. + expect_forbidden (bool, optional): Should export request be forbidden or not. Defaults to False. + wait_result (bool, optional): Wait until export process will be finished. Defaults to True. + download_result (bool, optional): Download exported file. Defaults to True. + + Returns: + bytes: The content of the file if downloaded locally. + None: If `wait_result` or `download_result` were False or the file is downloaded to cloud storage. + """ + # initialize background process and ensure that the first request returns 403 code if request should be forbidden + initialize_export(endpoint, expect_forbidden=expect_forbidden, **kwargs) + + if not wait_result: + return None + + return wait_and_download_v1( + endpoint, + max_retries=max_retries, + interval=interval, + download_result=download_result, + **kwargs, + ) + + +def wait_and_download_v2( + api_client: ApiClient, + rq_id: str, + *, + max_retries: int = 30, + interval: float = 0.1, + download_result: bool = True, +) -> Optional[bytes]: + for _ in range(max_retries): + (background_request, response) = api_client.requests_api.retrieve(rq_id) + assert response.status == HTTPStatus.OK + if ( + background_request.status.value + == models.RequestStatus.allowed_values[("value",)]["FINISHED"] + ): + break + sleep(interval) + else: + assert False, ( + f"Export process was not finished within allowed time ({interval * max_retries}, sec). " + + f"Last status was: {background_request.status.value}" + ) + + if not download_result: + return None + # return downloaded file in case of local downloading or None otherwise + if background_request.result_url: + response = requests.get( + background_request.result_url, + auth=(api_client.configuration.username, api_client.configuration.password), + ) + assert response.status_code == HTTPStatus.OK -FieldPath = Sequence[str] + return response.content + + return None + + +def export_v2( + endpoint: Endpoint, + *, + max_retries: int = 30, + interval: float = 0.1, + expect_forbidden: bool = False, + wait_result: bool = True, + download_result: bool = True, + **kwargs, +) -> Optional[bytes]: + """Export datasets|annotations|backups using the second version of export API + + Args: + endpoint (Endpoint): Export endpoint, will be called only to initialize export process + max_retries (int, optional): Number of retries when checking process status. Defaults to 30. + interval (float, optional): Interval in seconds between retries. Defaults to 0.1. + expect_forbidden (bool, optional): Should export request be forbidden or not. Defaults to False. + download_result (bool, optional): Download exported file. Defaults to True. + + Returns: + bytes: The content of the file if downloaded locally. + None: If `wait_result` or `download_result` were False or the file is downloaded to cloud storage. + """ + # initialize background process and ensure that the first request returns 403 code if request should be forbidden + rq_id = initialize_export(endpoint, expect_forbidden=expect_forbidden, **kwargs) + + if not wait_result: + return None + + # check status of background process + return wait_and_download_v2( + endpoint.api_client, + rq_id, + max_retries=max_retries, + interval=interval, + download_result=download_result, + ) + + +def export_dataset( + api: Union[ProjectsApi, TasksApi, JobsApi], + api_version: Union[ + int, Tuple[int] + ], # make this parameter required to be sure that all tests was updated and both API versions are used + *, + save_images: bool, + max_retries: int = 30, + interval: float = 0.1, + format: str = "CVAT for images 1.1", # pylint: disable=redefined-builtin + **kwargs, +) -> Optional[bytes]: + def _get_endpoint_and_kwargs(version: int) -> Endpoint: + extra_kwargs = { + "format": format, + } + if version == 1: + endpoint = ( + api.retrieve_dataset_endpoint if save_images else api.retrieve_annotations_endpoint + ) + else: + endpoint = api.create_dataset_export_endpoint + extra_kwargs["save_images"] = save_images + return endpoint, extra_kwargs + + if api_version == 1: + endpoint, extra_kwargs = _get_endpoint_and_kwargs(api_version) + return export_v1( + endpoint, + max_retries=max_retries, + interval=interval, + **kwargs, + **extra_kwargs, + ) + elif api_version == 2: + endpoint, extra_kwargs = _get_endpoint_and_kwargs(api_version) + return export_v2( + endpoint, + max_retries=max_retries, + interval=interval, + **kwargs, + **extra_kwargs, + ) + elif isinstance(api_version, tuple): + assert len(api_version) == 2, "Expected 2 elements in api_version tuple" + initialize_endpoint, extra_kwargs = _get_endpoint_and_kwargs(api_version[0]) + rq_id = initialize_export(initialize_endpoint, **kwargs, **extra_kwargs) + + if api_version[1] == 1: + endpoint, extra_kwargs = _get_endpoint_and_kwargs(api_version[1]) + return wait_and_download_v1( + endpoint, max_retries=max_retries, interval=interval, **kwargs, **extra_kwargs + ) + else: + return wait_and_download_v2( + api.api_client, rq_id, max_retries=max_retries, interval=interval + ) + + assert False, "Unsupported API version" + + +# FUTURE-TODO: support username: optional, api_client: optional +def export_project_dataset( + username: str, api_version: Union[int, Tuple[int]], *args, **kwargs +) -> Optional[bytes]: + with make_api_client(username) as api_client: + return export_dataset(api_client.projects_api, api_version, *args, **kwargs) + + +def export_task_dataset( + username: str, api_version: Union[int, Tuple[int]], *args, **kwargs +) -> Optional[bytes]: + with make_api_client(username) as api_client: + return export_dataset(api_client.tasks_api, api_version, *args, **kwargs) + + +def export_job_dataset( + username: str, api_version: Union[int, Tuple[int]], *args, **kwargs +) -> Optional[bytes]: + with make_api_client(username) as api_client: + return export_dataset(api_client.jobs_api, api_version, *args, **kwargs) + + +def export_backup( + api: Union[ProjectsApi, TasksApi], + api_version: Union[ + int, Tuple[int] + ], # make this parameter required to be sure that all tests was updated and both API versions are used + *, + max_retries: int = 30, + interval: float = 0.1, + **kwargs, +) -> Optional[bytes]: + if api_version == 1: + endpoint = api.retrieve_backup_endpoint + return export_v1(endpoint, max_retries=max_retries, interval=interval, **kwargs) + elif api_version == 2: + endpoint = api.create_backup_export_endpoint + return export_v2(endpoint, max_retries=max_retries, interval=interval, **kwargs) + elif isinstance(api_version, tuple): + assert len(api_version) == 2, "Expected 2 elements in api_version tuple" + initialize_endpoint = ( + api.retrieve_backup_endpoint + if api_version[0] == 1 + else api.create_backup_export_endpoint + ) + rq_id = initialize_export(initialize_endpoint, **kwargs) + + if api_version[1] == 1: + return wait_and_download_v1( + api.retrieve_backup_endpoint, max_retries=max_retries, interval=interval, **kwargs + ) + else: + return wait_and_download_v2( + api.api_client, rq_id, max_retries=max_retries, interval=interval + ) + + assert False, "Unsupported API version" + + +def export_project_backup( + username: str, api_version: Union[int, Tuple[int]], *args, **kwargs +) -> Optional[bytes]: + with make_api_client(username) as api_client: + return export_backup(api_client.projects_api, api_version, *args, **kwargs) + + +def export_task_backup( + username: str, api_version: Union[int, Tuple[int]], *args, **kwargs +) -> Optional[bytes]: + with make_api_client(username) as api_client: + return export_backup(api_client.tasks_api, api_version, *args, **kwargs) + + +def import_resource( + endpoint: Endpoint, + *, + max_retries: int = 30, + interval: float = 0.1, + expect_forbidden: bool = False, + wait_result: bool = True, + **kwargs, +) -> None: + # initialize background process and ensure that the first request returns 403 code if request should be forbidden + (_, response) = endpoint.call_with_http_info( + **kwargs, + _parse_response=False, + _check_status=False, + _content_type="multipart/form-data", + ) + if expect_forbidden: + assert response.status == HTTPStatus.FORBIDDEN, "Request should be forbidden" + raise ForbiddenException() + + assert response.status == HTTPStatus.ACCEPTED + + if not wait_result: + return None + + # define background request ID returned in the server response + rq_id = json.loads(response.data).get("rq_id") + assert rq_id, "The rq_id parameter was not found in the server response" + + # check status of background process + for _ in range(max_retries): + (background_request, response) = endpoint.api_client.requests_api.retrieve(rq_id) + assert response.status == HTTPStatus.OK + if background_request.status.value in ( + models.RequestStatus.allowed_values[("value",)]["FINISHED"], + models.RequestStatus.allowed_values[("value",)]["FAILED"], + ): + break + sleep(interval) + else: + assert False, ( + f"Import process was not finished within allowed time ({interval * max_retries}, sec). " + + f"Last status was: {background_request.status.value}" + ) + + +def import_backup( + api: Union[ProjectsApi, TasksApi], + *, + max_retries: int = 30, + interval: float = 0.1, + **kwargs, +) -> None: + endpoint = api.create_backup_endpoint + return import_resource(endpoint, max_retries=max_retries, interval=interval, **kwargs) + + +def import_project_backup(username: str, data: Dict, **kwargs) -> None: + with make_api_client(username) as api_client: + return import_backup(api_client.projects_api, project_file_request=deepcopy(data), **kwargs) + + +def import_task_backup(username: str, data: Dict, **kwargs) -> None: + with make_api_client(username) as api_client: + return import_backup(api_client.tasks_api, task_file_request=deepcopy(data), **kwargs) + + +FieldPath = Sequence[Union[str, Callable]] class CollectionSimpleFilterTestBase(metaclass=ABCMeta): @@ -68,9 +412,14 @@ def _get_field(cls, d: Dict[str, Any], path: Union[str, FieldPath]) -> Optional[ assert path for key in path: if isinstance(d, dict): + assert isinstance(key, str) d = d.get(key) else: - d = None + if callable(key): + assert isinstance(d, str) + d = key(d) + else: + d = None return d @@ -113,12 +462,27 @@ def _compare_results(self, gt_objects, received_objects): assert diff == {}, diff - def test_can_use_simple_filter_for_object_list(self, field): - value, gt_objects = self._get_field_samples(field) - - received_items = self._retrieve_collection(**{field: value}) + def _test_can_use_simple_filter_for_object_list( + self, field: str, field_values: Optional[List[Any]] = None + ): + gt_objects = [] + field_path = self._map_field(field) - self._compare_results(gt_objects, received_items) + if not field_values: + value, gt_objects = self._get_field_samples(field) + field_values = [value] + + are_gt_objects_initialized = bool(gt_objects) + + for value in field_values: + if not are_gt_objects_initialized: + gt_objects = [ + sample + for sample in self.samples + if value == self._get_field(sample, field_path) + ] + received_items = self._retrieve_collection(**{field: value}) + self._compare_results(gt_objects, received_items) def get_attrs(obj: Any, attributes: Sequence[str]) -> Tuple[Any, ...]: diff --git a/tests/python/shared/assets/cvat_db/data.json b/tests/python/shared/assets/cvat_db/data.json index 39b63b2c60b..1f3b9cfc0a1 100644 --- a/tests/python/shared/assets/cvat_db/data.json +++ b/tests/python/shared/assets/cvat_db/data.json @@ -11276,7 +11276,7 @@ "pk": 1, "fields": { "location": "cloud_storage", - "cloud_storage": 2 + "cloud_storage": 3 } }, { @@ -11284,7 +11284,7 @@ "pk": 2, "fields": { "location": "cloud_storage", - "cloud_storage": 2 + "cloud_storage": 3 } }, { @@ -11292,7 +11292,7 @@ "pk": 3, "fields": { "location": "cloud_storage", - "cloud_storage": 2 + "cloud_storage": 3 } }, { @@ -11300,7 +11300,7 @@ "pk": 4, "fields": { "location": "cloud_storage", - "cloud_storage": 2 + "cloud_storage": 3 } }, { diff --git a/tests/python/shared/assets/jobs.json b/tests/python/shared/assets/jobs.json index 82dd91cc0d9..d4add795c78 100644 --- a/tests/python/shared/assets/jobs.json +++ b/tests/python/shared/assets/jobs.json @@ -664,7 +664,7 @@ "organization": 2, "project_id": 2, "source_storage": { - "cloud_storage_id": 2, + "cloud_storage_id": 3, "id": 4, "location": "cloud_storage" }, @@ -674,7 +674,7 @@ "status": "annotation", "stop_frame": 10, "target_storage": { - "cloud_storage_id": 2, + "cloud_storage_id": 3, "id": 2, "location": "cloud_storage" }, diff --git a/tests/python/shared/assets/projects.json b/tests/python/shared/assets/projects.json index b1d06e75ef9..372e859dcbe 100644 --- a/tests/python/shared/assets/projects.json +++ b/tests/python/shared/assets/projects.json @@ -470,13 +470,13 @@ "username": "business1" }, "source_storage": { - "cloud_storage_id": 2, + "cloud_storage_id": 3, "id": 3, "location": "cloud_storage" }, "status": "annotation", "target_storage": { - "cloud_storage_id": 2, + "cloud_storage_id": 3, "id": 1, "location": "cloud_storage" }, diff --git a/tests/python/shared/assets/tasks.json b/tests/python/shared/assets/tasks.json index 238df8007b1..a7219503004 100644 --- a/tests/python/shared/assets/tasks.json +++ b/tests/python/shared/assets/tasks.json @@ -675,14 +675,14 @@ "segment_size": 11, "size": 11, "source_storage": { - "cloud_storage_id": 2, + "cloud_storage_id": 3, "id": 4, "location": "cloud_storage" }, "status": "annotation", "subset": "Train", "target_storage": { - "cloud_storage_id": 2, + "cloud_storage_id": 3, "id": 2, "location": "cloud_storage" }, diff --git a/tests/python/shared/fixtures/data.py b/tests/python/shared/fixtures/data.py index 2d650439f7a..0dbce65e47f 100644 --- a/tests/python/shared/fixtures/data.py +++ b/tests/python/shared/fixtures/data.py @@ -3,8 +3,10 @@ # SPDX-License-Identifier: MIT import json +import operator from collections import defaultdict from copy import deepcopy +from typing import Iterable import pytest @@ -60,6 +62,51 @@ def tasks(): return Container(json.load(f)["results"]) +def filter_assets(resources: Iterable, **kwargs): + filtered_resources = [] + exclude_prefix = "exclude_" + + for resource in resources: + for key, value in kwargs.items(): + op = operator.eq + if key.startswith(exclude_prefix): + key = key[len(exclude_prefix) :] + op = operator.ne + + cur_value, rest = resource, key + while rest: + field_and_rest = rest.split("__", maxsplit=1) + if 2 == len(field_and_rest): + field, rest = field_and_rest + else: + field, rest = field_and_rest[0], None + cur_value = cur_value[field] + # e.g. task has null target_storage + if not cur_value: + break + + if not rest and op(cur_value, value) or rest and op == operator.ne: + filtered_resources.append(resource) + + return filtered_resources + + +@pytest.fixture(scope="session") +def filter_projects(projects): + def filter_(**kwargs): + return filter_assets(projects, **kwargs) + + return filter_ + + +@pytest.fixture(scope="session") +def filter_tasks(tasks): + def filter_(**kwargs): + return filter_assets(tasks, **kwargs) + + return filter_ + + @pytest.fixture(scope="session") def tasks_wlc(labels, tasks): # tasks with labels count tasks = deepcopy(tasks) From 86c1b222542d25f4e495e97d4da37b2b17b8773d Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Thu, 22 Aug 2024 12:37:36 +0300 Subject: [PATCH 12/22] Fixed `undefined` active control (#8334) --- changelog.d/20240822_111601_sekachev.bs_fixed_minor_issue.md | 4 ++++ cvat-ui/src/actions/annotation-actions.ts | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 changelog.d/20240822_111601_sekachev.bs_fixed_minor_issue.md diff --git a/changelog.d/20240822_111601_sekachev.bs_fixed_minor_issue.md b/changelog.d/20240822_111601_sekachev.bs_fixed_minor_issue.md new file mode 100644 index 00000000000..7ae4d38588b --- /dev/null +++ b/changelog.d/20240822_111601_sekachev.bs_fixed_minor_issue.md @@ -0,0 +1,4 @@ +### Fixed + +- Sometimes it is not possible to switch workspace because active control broken after +trying to create a tag with a shortcut () diff --git a/cvat-ui/src/actions/annotation-actions.ts b/cvat-ui/src/actions/annotation-actions.ts index 6a121a60521..a5594093db5 100644 --- a/cvat-ui/src/actions/annotation-actions.ts +++ b/cvat-ui/src/actions/annotation-actions.ts @@ -1412,7 +1412,9 @@ export function repeatDrawShapeAsync(): ThunkAction { return; } - activeControl = ShapeTypeToControl[activeShapeType]; + if (activeObjectType !== ObjectType.TAG) { + activeControl = ShapeTypeToControl[activeShapeType]; + } if (canvasInstance instanceof Canvas) { canvasInstance.cancel(); From 9393681b7d4aab0051d15aa06969aff696c539b6 Mon Sep 17 00:00:00 2001 From: Kirill Lakhov Date: Thu, 22 Aug 2024 14:37:05 +0300 Subject: [PATCH 13/22] Docs for `Requests` page (#8327) --- .../en/docs/manual/basics/requests-page.md | 37 ++++++++++++++++++ site/content/en/images/requests_page.png | Bin 0 -> 103720 bytes 2 files changed, 37 insertions(+) create mode 100644 site/content/en/docs/manual/basics/requests-page.md create mode 100644 site/content/en/images/requests_page.png diff --git a/site/content/en/docs/manual/basics/requests-page.md b/site/content/en/docs/manual/basics/requests-page.md new file mode 100644 index 00000000000..510ca86d772 --- /dev/null +++ b/site/content/en/docs/manual/basics/requests-page.md @@ -0,0 +1,37 @@ +--- +title: 'Requests page' +linkTitle: 'Requests page' +weight: 7 +--- + +The Requests page allows users to track the status of data processing jobs such as exporting annotations +or importing datasets. Users can monitor progress, download results, and check for errors if they occur. + +![Requests page](/images/requests_page.png) + +## Requests List + +On the Requests page, requests are displayed as cards. Each card contains the following details (if applicable): +- **Operation Name** +- **Resource Link** +- **Status of the Request** +- **Timestamps**: + - **Enqueued Date** + - **Started Date** + - **Finished Date** + - **Result Expiration Date** +- **Annotations Format** +- **Menu** to download the result or cancel a `Queued` job + +> Currently supported operations include creating tasks, importing/exporting annotations and datasets, and backups. + +## Statuses for Requests List + +The following statuses are used to indicate the state of each request: + +| Status | Description | +| --------------| --------------------------------------------------------------------------- | +| `In Progress` | The requested job is being executed. The progress percentage is shown. | +| `Queued` | The requested job is waiting to be picked up by a worker. | +| `Finished` | The requested job is finished. Downloading the result is available. | +| `Failed` | The requested job cannot be executed due to an unexpected error. The error description is available. | diff --git a/site/content/en/images/requests_page.png b/site/content/en/images/requests_page.png new file mode 100644 index 0000000000000000000000000000000000000000..cd2e84f2493a83f30b16fac587bce08fef9e3dd9 GIT binary patch literal 103720 zcmdSARa6{Zw=UcSOGqFgSb_&l(BSUw?lc4l5Zv80IKhJl_r{%u;32rXTceG;{FS`_ zyT5&I&NvrmUz{}tQe9oER!v)TKJ%$iMR^HyG<-A=2!t*rDXI(tp>l#ih}ln{0G|Z> zPP7O9JhPS5asYuay8iqiB+z0IfIy@mDN!L+*R=gbFWtEF2Z1BJibiBuA`+b>Aq6Y% zt5a8wytwCOxOwslQ~U8_T#0FMPFLwe=ml}xwnOzcTGYeuYHc(#PQ{vIF!G-(G{{<} zz+{WwQt+(dQ9s3ax`@5sVRw3K@;qmL&j`tNXr8Y#;Bdmy@D^ujze6y?lO$2loh5r}xdrLJD``Tht*`gF zm^Ia@v#gvy@WOb*Vlr@lxh!D(JHB51-=0APK2vGbpTsEOeRqW{Aub{DxU13DG_hFQ?0oRMug}cN3jOKR@2qCA+GWIi z?vAdmt{xs9A|jomITCGGYvK2|m-V%^QWUWTncnUlfzOg^hNv``d;Ug zHf{IL-_|7}-ozv)zs16G2;Keg{=F+4dI>6=+|9h-H@y$)=#3_M>=k%WQddX$G@302 zJWzw^x$EuOxWcmAHj%=3rE-zy-El9xVLht9Vwn~_K>J^5WM?^^!AA3Bk}~)_F0!Z4 z(9qnkHCc`}=z~{7Cr%tWZ%lHa6?}+miBDX(_4W z%S)0D_ubuHJ=fJBU^z=4%H|Is5QsB(dV72OBG38hsf@HVx6|&#&!={H!lFSPVG3NVi0n=9{rdx#YfA_LBeV zrw8svcR1c-+vDZq4MApXtg?Q|k)0fd~aI^FM-L}BvCZiA6PkEoq=4G#| z)vut#meZjGS`CbEMFn|zT6Bs0Bu!}CAuqieJWCvxVm)#^!M_r+Mo-!rB__hRqnp_o zI9<<4q7<7R&Y>IvO_Br;P<97c=YH8*ql~g!%J`(v`}^rVk4_ZMV+;PFK8uWtfd$uN zo0~{!Ww+0HmzdBxLVq&Uaaxd9F z+|KmDSOLsdOib)@$pijs>h!P~URPJQLdV=&yIo*ci_3B11DinP*7G@R(b1xG8VPal7Mu6j48W4xrI(4xk*a|v%_HR@VX*Swv;+G8*Yf*&*^)eiID?n{t( zO&A%#;YclmXY<<@|l4;QY1)$=`Et>2>) zH++{V9crO+u`#*?D=E$tUTt{5BjV66#yuv8g1NH8+aMW=YVM-hVK3XVh5d;Y#On(u z3bv^AYL$#L7d8nruvKfGJWZ{G*$t}s7>K&8XRQ%=SgD;u64j*?B}MnIHa6^Hk{;MB zisv*`J%2RSRW_k|C`u@wW6m z=*!Z*`3=F8dE_*!Jlj0;eRX?2M34mY9)YN{DqgMmaguL zl~Dw(|9}7zw~ZLUZWH6w&z+#gO^b?Xl>I>8Dx#>PsJJ3!mg%;Y z4A{g^7Pl_s`>S37J*a!_tfjT3<%?%rjGc2828g~NRPq<>`{@xaSM?un@O&d8A}T5> zwlh2!1O$A1e*oI7%^zkPlPn6@Cc@HvCtAN#-dGnq)5R5Z&XIdU7cp ztFk>D@$_{q`?SVB%|Z&JB<^5ToL;hN10#|+MP%eFMB7ZW3Jt=gYHv6ChrP{N>18#? zixgqO0v=h+QygP87jbc?OkOABuc?sg(5S!O3A3NAx6!w-u($*ljB|ajZ7Jt+ z-km54Kz)fZIZ%-yz6eOlRiAJFY-gM*&TOJksqOI|E)hleYz5G_ClETr!^80ybZ650 zqKNspnV2eTYR*v#vPh1jABsoRn5lwg>wWES? z=`@eR_3sRiEC63$1l;@2R`TZtfEQmZtpFa(`E$27is)3<_-LKvk@MrnqsHxwlG0K- zdU`oo*_N3b#+OQpwzd`2VJ{NR4xJLpKU7XCM8|7CaMaVRW~?BJZIZ*iOsskF`|EA! z8Ooh%ktM~&<-TVWhsI_Iocrx5j`s5Omw@RJVq-?*;EgVZx!9l-n!%v?vdKs*gm*a; zAByIbCJ7wSXtZrN&i79|aQU#+?(Gch+^meAQ@q0rb;*~HCMc#l4yQs-bfaQ2iv^J- z%Tk^4(nCbVf_P;3D5I1Ma?OqWo1;ofn=0SF*3#6q-(3uSx1DDnssCVcuQz&j?XZr> z@No>atDRY^f|Tnr@pbV94(>r2_vjcdX?Tc2D>Y?nNl&8rXrnio0FdY>EgNNU(#wfk>3(8|iB3RmvZ9eQ*s5|6S zU0%R_g^kN`ozdJI6h3;kc{4c$s~LhAmU{>fwsz2WOXxmD3s+FDMS$utXI)5oDV7Rh ztzR=s;IlC}$$uN93~0VDkKc&A@JP5)H(y@Y&? zATu+!!5m*k+?bkocS|a_*nsBQ-o%iPZt5lU;_QS^!iA(Txhz7o`aaHS5A8yYwV^w! zQf2$mvVG#00Vw6AcZ1>JS5yL05*{O=u~D2ojKCr7i+YK8^q^q@=;09znO0vO9eL=` z``ncF0(IzWeOJtAbU7dYq6}Q^ec7&^q(euL9?$JFYUyYZ%L;VQ`6b?P^LQ)?pU5L~372x-{Ot@z(;b2^l<^(y<;Sy6F> zlt38XdVA>a?++w2u7^OVd!eRiQLBw`cl1@@OivRGMi(rZbXV2}0oz$$UmpRxr7b7n z(qQ#^Pb4ipeXVv`5?SH-eqm*0G>K6ruiIt=6o)23xC20Kw6ifozAGa6S=CteX^&Vr{9zv70JgcoIB{oBnYgT5r*JzU%efFxQ zy9Qj>8cj__0p}6AP7P0Nd6YAsk|`cmFS{+RqEzNH?StbLqng(NX_}>;o*sPp{`UHs z@8lObnG#9Sz8GpP>PupzAJcn`&SFr#$4L!+9Bgb?)IZ56xA$FxELHR4!!4?&3-BzU z8tpRb+v&Y?@f;x2)vFY;Qb<>*elHk!gP5Rn$@ict`C|sGQd3mnJ~7Zs5)oHBc|fb# zR;gI`#kyr0^@jVjr=z$~%gn*he}1=>ZeW=4g?uIfqt!xv(TeRaXfz+2_!Ji>!J`M~5Ys`Y3DoG^IF@Ot|QQN*1>fu2Tt^vtj(-;bKBXLR&c>q$gT|gCZ0DT=CPt- zwXuwy%-CH4Rj^E|`2~*koMGrqYurz77l=rmyVPCF>306No-f-yAm0(FWZ@l!kAzXj zf>^W|GKX3+$L6*VnNHfAO}~}g@m4x@_BNd!Ho0AbPptlo|5b)aF1d)mF3zpYo-l7U z5oqnGq01Jus%OeM(qJ=AH6P-!B@p+$l6mM`XIu#oo91PS-RM9~YMh+XgxIF#~*|Zms z6NM=}%S1kYpS0W-n6wWEf-mRCr?!XQ>x_>xjKn+^qj$ejtl&7MnhY4=x{p&(c%;TO z2`4!LkI@CvvEMlXMiT-O`W;kUAk`kHwa&1AG{o%GhoAjdBfJxRFRA?L-F&sw%OUO~ z@%DC>oPvAH3v8zqtvvxfa6PJ2H7@U)&@hE58xJEc;Uq@@jj-*)(k5 z=HmU3_wIAQipewH=G^ea(T1Z*Yg4_+7!%Dt?C+49QXd@@^eOn=*e1ZRH&y0@84So7 z<#}Wt$u{0$gmxPK`kqbfKO1Tlm-n6PwSXR%ov2Zxam(W(DRl@Tkuv<_9c%3rV;OA^g zM?D(+dkptZY74ko9eP!AHInmGQ0w`k+=IbvtGh$=MOr^u&uO>41z3>I2r!7AcyRP^p>ACOh zmA64kOV3(*U@4a?eyGH6K6L|C&)uD;tDGT^!L~XU0ofedsu)&2xiG@NCn9-5uZkl@h2jHjy#LTrmi-(I(C(cqRl(HeVd14 zG~)Bas?5z=3=A8ved!q7JiB(G#%y!8A3#zAfg6RNm|k&td)62`h?Vgu&5@D*ba>Xi z@+bl30aeLn7P{c z{s~ldwLu}Xv$PaI22XQPP%%(Cv=FNb2`LPL!Dbqml%V-8)}`-DNVvQc3&Oc6jxi>e z7OF%bp;%&qC9LkAhOQq$qIq=g#e=75VMedTkrO7TnlZ^B0=kQ@`S`e)-v|z*#~J-D zF-Q1Oxu8wJ@0M@MV5a4Ha&>R<8>6wOTi*|!KSyMt$M+ruRDTL4#G^AfowO}SYMvPt zu#vKMY^zKMQ&J&8!Oo%~;Yl4}u=+cx?;NkyGI{|=6sy}!gzfA~e(*iAi!-`ZCq%g6 zQ)_8eF}L=6FG+(yiY}Fw?w#P7DfxFN73tozcUo4bNeIwq;^uC;bq25u5IN)TB9#Kr z3le@$UnZbXpPikZoSXzUPrj|}b5=PF1HL7o=AJM=H_i0ASPN(5Kc7&^*b7eQbvudV zSek))T*iM;LAgbWlGMJ^O{B0N%YF0gXf_|Q*>Xx+HtepY@Q&458hm0JfdAc2Ww2AL zS(}$u60|ZQSJdWn@_>oXd30Tj$QHZ9$mizsrLzJt^YK8?*qG9LBBxFE$q!2J_DsBD zzg(3m>ZwW9IrAUHwhv>=4;S?qUpvPycA1>_XZ{@BN4CL!sMFy#FIvMJ*-^~8LNI~) zX}->CJ{X;V7df5ZtEsT?sqZD_w)^dV^)Nm@zIDrS&(3&(N1|*xHybZ+3j|_)dvPG* zfz$GCOuN}@@FYDWwX7`m+e|#yhx9n|Fn@99hX=52qqhHN@@9|I&*a91*1VT`!L}Wn z=3G@fHtgl}UmH{26*(A)UgE$%D;+a~z8n%`5oYqXTCVeyYlw5i7DY1}5>{wRs zyKLZeI&~mx3A@CBD`as;_PIBMv0wEGH*X2P>i5qpw6Ydj2sBf=)|vhcj|_<-#>)Km z2Ji3`v1vu;E~0;uDjd$=*u|`+r~@rQzDKGZgiu91-Q68SkMFy=TPMdL;kZ4vyJS~a zQ=23%syg_!{9xBr|l^vdP~sv;|TM;g)PEsK~}QdJt>z=ZsnFvWu^HA zG!Xq@2nwiua^f{MbM_wc(`GJF)%nK3sp5`leFon8)E^c3v?q&>TQI+SiRMG3sPyw& zQ{uO@s2IdnAX`VTE&uaH5eq;qL#z#cA)`3~pikfJEh#P<+-=Ty(Y_sMhLXH4el8$Ddl9ZW4&Hu z>=e8DXU9Ubzu2EWoCjkWVT4_-9D%aWo1N z66gv;YK8QE^`(#NRDw6< ztlcQpc#=AtZ9KTzu!p02{IG)nqp+KSw_UZ#QqC!P07BN8ez}J`ZTBh`iS}!K4Yv(R zF*hlp!zEa9^iUMy$eiYUHa;bvuC6X#89Lr2$I0(!A`DwrlqV4wQfaBg^O{b%RoN|; zdpzA!9>hNHM>QO{eu@89R06~0U$hERxsCepqsZXD#=(1y!yzA#e-hak#M6(p72W)k z2D+r-)SZe(E3C537?l+2my(^<^1M4PdxmRxAac)bKx3)dY`{N(F*kT^znyLP!B|J6 z^R4S+h{>5(a(KQom_~*04NYehaY^QZ9myet^6)h2a!FMYmTlZ0=WH|I*0%Pk+s|t6 z<^Bv`+okb$J>u~k$;9J@I;VJ>QR-w-DmA|+;SO6%3c;!k#Q6vKV zS4Ae!0-lD#K$lPUjX)$D+Ht)2l5kLGHl7cFfl2z_jlE}2i8GS1{sg_O;sVtAc|P|u z7y!5f@v?e$0qO}r7fUghKTAuh)N#_K-LHm+7fC);LbT87hlf2H954cwDtAAIPWu`J zG;Gv#c?7k;<@dazEpIuQP*Ff|-AvFxv9`J(xpU@QTVF@O#>Up&5BuX4Oom+{0Ge+y z4FLn$&!?6CkaSKvLm)+jes!bjTWs~Z4aK5v6`@Q@`4U(v)25Z=m`hhU>y+)$qo5J+ zoV>$$2~pTzdzYZOO~{Ue?HP#ldD3cEs0z45?Qq@&cE^>p=)Z!C`n5N5Jb&nMc7RDP zPyglw&CTmeX_-y9VhNv>$>Gi14aCM6Nl4JvRW^HT=_~G#%6i3l2nduNVpeT3oLpp0 zJKIv-!#5(SZ<~PkP*TxyYHV(XNt%EJs`~g=_k3~mQ(Q?Uy(GKx3s4yZ3*5g5${w*sy=Tyc3q?FfT| z1x@msSm$`0YD0F<{dMTVBQiO?E{x1BMJIIncNu4u{+jn_LNAMkZ|_h#kB7cYu>uhh z5s!14MeQ}leawC1PJSwA=HTK1C5_ijIZN>@dF%sHs;V`%tHR;k_5h?x z@FEyM%O9?RUjVM}a}+?b58$e+9YIT@2O!IY0^jesZ{rNf$aUm1cYnK(liT12SF^Dz z46heS>li<#HpE*uZYKUo;Wv}?TUT_e0ASAj^_a`D*X8-NetB_m)6AslP}(z64vzZ5 z!ouuqVX~Oq{Ctd$4?wF0>7(@mC)|x2v;g!Xo;`a8YBJSoy%8-|tJ;O~(4;Q-DjI#@ z4`kZ?0#}FrLP{w0wox+6A$Wz~FW_c6n9m3nl(ZC{3BBi$-b)G0Z(qe!GRY0DY-yFIS>>=4sTIF-%CtaLySE0`}xBcE03qnTWeBN!>m%ZCNMG- z4aGlaa#YyuILYbBigN;dK}SG2eZe__M>N(=pmJ-Da+uT7K{ z=gH|>q+kAwYyHVtR4U7Hgv#|!_cz&Nzngk`no>k=`8}!I=mgo?Z;@~$sCz=Au<@o@ z5f1Zz8}`TixC&G)_#U z0pPe(q*{D95A^vm^uzSO^?fdeKn`2~>`ht45@qU$S10|9w*=qP|BS`u93X>0_O)AQ zIFp$UzuYOvYz3O0j}MpB8Ln%P=EJ5vNOOkgNgoca`o_iv5;F4rw9ozY zpPp#DoY)ct0*wIU!o{paiMrR+>t|^+%ja5@RC3WI!8Ube?o99;xss|A8cJA6`Gx%( z9_P}_(9z8BhXZ?a^WBSsc>wM9Dc5ZT`hE#sQ9vK}8QLz-;CBw2rDk|*v&A%ng37_4 zzOQX>dBL9ZQi~guWNh!;*w`3os=4o1_A*?&hrC$rJ5|;}tA}Wf{)Kn>tcIX=+mE^2 zx;!=}^SwOFdP+0?KfWnFt2T*>LwK6ZX|Yul?E=vU&)~jdmgckAYWPIT$^Fjoai}G2 zmxm;kgZg^CEOmO{5rBBZLz+!Z>>F$_Dg>&!;y1`spKCBOz6cA+bRI5+`JSgHXHlE| z%LRBog)*eW>2O1>11x3L+)or2H=UQ7QaZxnK4I+_k7-o7dx5QorM9#_xtAy*o0Ume zMJ=a~oP#eN4ZqV=*DKJ|D24Rtv*W3|>H2yAUB(e`faISVzlv1u<**qZgjiWW4^u2y z!eT_G8j|KyYdH1MurxQrK@Jk59%!OoN)jl?`)ctbI+AXwvcBQxuDMylua0a(6%x6q zmw1R)+v42|d^W)1ILft_{>TZ_XP9@m|5XAnpSRKIBV>+%j)2SLqpb>g)N_7hB{O@d4nJ(A&~MRWHLG!tFD#~i z*VLO@lF6b)=%P^UetOX|E%W3v=5BnSl7#e}ueUr-Ikan;rNnwK>Tdcu(7uTMrW9IK z1eI6#+wX5mG-Jhw*L9#1u-DYp;lBue=&7%(gXq)`od%p(s}#r+6}wZ(em?@5!iV$q zS91%CGId(tP^_<4Rog(p&)9Znuc)Z#?CcE02si?e z9sm$P+f_HCp4(=^ymfQgFw>&<*+%Tna?3g3#!F!e;MjBJex2dgtJ%7i;rzDNqsw6; zH?v8J%uVRyU^FgWvdYzM{cnU!irRp6)9P6U-r}Um5{<__{m0^>qMg%G1b=N%iF%oK ztvQY_&<)+%*myD0Z&j;}jv@l!@epwH6GTK0-d+1fDus*}pH8<1fi-zqzXQ6DJ7akO z@_z@+WI{rVYR zVvddf5LCe7%q%Pi&bL6^cCA_-0w7?v-EA_8==@CB;n73? z0#t9|c#IF$dkk7tp8>i>z}q7A64YNY#>}(GDq8?fcs?8HfsC~)pXST@8|cGOHzhWf zplNJRQx2pi5mIug=;UD zwa{=p!~;JgrizgSI3wCakMymQ6xtP^+DE2Nf00K65N&*7qV-HgmWv`TF76h5xp$)w zNZn6mb+`q2j$9mW&US_c9(e%<$x1x#Pa>BEOguc(O3jp%l;EQ$78<3#LMHi+ZmM2s zr}Cj5jwg$8Hixr9vZ5lSg6z_(`uE)h^L1WL^%G2V7l#uYg@T!bww|-k5_lf&jRSa( zmzveQ^!MZbmf#_+^&=xAfZl=Zju62~fMjMaN4#apSI9hEl%HQ(dbpUkNtLAnwcSjX zTSL#MvdRi-Y8-**12`oYy1wS0A|oT!OPl~&&(h$p>0{?I#*j;R1AL3%z`!%E6M*jl6x~zp0jLPMd3oq=<}X&d zLa{*N(Zqb;+cUmjZS=*7==I>hc!rezL*u#!=t|%=Z?DUsZov6l|1^o;oC1okzGXH9 zcFu5I`dtTH|EwxN^&HEd)n#RJ6t4kmH@mw!u3xa10mw~2pIbyk1lT>z%yxjBd1+~h zAx1L8`(_6qW9c>-7IHZMhvS9L6eD>yV-)?zUjP^o=&~V35CY}idb9vPz`~LNkJXrs z1I3^KKvn>FHU|Lz>+*0B+6$l*Zvk#o248bqqE3?|Ak}~DR8-Wl2nc8?viYl>Ulmf> z%X&osO>;VIMW~lRSAz){Or;9s)931KmVucF@bP)EFe3=LP5@WU0Dwq9I*5pFn-t5R z{uf?blI`5{e_2vwWcV-s8};jdOJ95QA958L8Oi^BuH0QpYN{6)@vpu7`8DbkN%Vin ztoxr{+W$i1`+wnPp`KvSzk)>fo0^(xe~BCV1`QLe_1|pk{{qJmmZApGXZBu)d{O+5 zjKHb9ALy&KvrGK-9UQ6whGI6nFefJm3MKO0e*A}13BzB9!{OIH0s?K%F|4hu&Rgfn zGt=2D5--6qlC0RsuSdTbD9Sk2n%mgeSXjhK=RG_;ARr)Q__X~0*QcyrFH8f^H8#fl zeb{SB$SM4P1ejeCTM)#~$vHC}^bg^3baP~4A`pTy|8NE?Bmy)Dd`;&l z1i~+C_RoY6j*Ma7w5uo>K0x4v(TtCdj!sSW#Q($Pgt=o85-tFD8y#KU6AppJ0uJSv zUkgwK%YN4x1+0Gfr^YqKUmwl3z!7-)8r*qxLG@ogd-8wGIQ^e9 zRZmFgKV;Vbuv$l*3K8YL&J$%ir(H3Vp6~|_`@SEUru<`u*ROg0A=5&{Ol9UT1ooFa zOFY);t;hGLs@D1^tDD-+a}+NN8W9g5IkoE>cn3~Aq|c6{hXuRt|4|TNT%+fG8Agh4 zPtsZ$*2lDu@wfI*oG?X2H?XYotTK1zOD>FZ+aITbyw`T4L|)c=>p6-IFpdCf{0#px zgWf0Soh5KWK>WN7q(A*9wr+R;^j^sGM7qc-JhdjN2G(W=%?MlxpFFy8nuHYDe#)7$ z!Ot~%DN6Q44tpT+L~HXG72skyxW z-u)$Z?_Ap9aO7)OVvDicl%LLLtDb#A2UBDF1Z_95?HUkiXgfsU$JxvG_e3Wq+1`w} zDU62a6;Hb`+F; z%UARW$41;3e6w3Hc;;@ac-?^wty@-(m;Ym~#{FEqOU(<-`Fgf5W=G9T&(<)PuR(wO z7Q**Oiv;#T%69x*p!T$)$=p=#Q@K6n^to>qr~_}!!HGuOWTv8_VCE zqwlXV7l|VB?QEfwD_1H~rHSMpWa&!GnKsq2YDB5EPCjobdTH4s8z7&fT*S&Pl0O=Z zaB0n9^48Hj!E~D|QHIye|EWPICOe@4hoR@|426zqR7K}rPUKT3eXgqTSj*}%3 zd)({{iIcrwB?r2<3lpf;mcQXfzKYS?`Vt+dA-_8eS#*ESO;^_7!`mVQFK=LeWnVhJ z6CnJ^#$UHV0ZC!&rA7xaJtc@b#)tXaH6VbBOFkN6Zv`R<&R9vljq1dkd?70;x~%ZE z8TldSu4UT`J?x3LZGLO!>`Upnz2bK|Z$8qD`pBka<~HRNe?B6gSHN?r0dYL`2#|5D zT$tXW+70#LyRa=zZ8;mR3Hb)&?w*j23;f))6Kzv-$ew=qSmqlDgZY|PR1 z*YG)Sv2JO{bEiuejTe==iI>k}BFR9MGLlVMljYvptEkr@`(nScbaNBd*30O#98%Xx zGEPMTT`ZhA=CJZjn^$W&jOX>7o$;cwYPo6Q7g3E{4$5eBm<>ED;sP=5rzAX`S z)gZMdCLL***ss=$6J5L_j(vS^!Lk8ueN=CC;KuxNm@ z-JMvJf}>5PU)>@OgzgfmDETYs*y+CjQBVkJhsvFU6dy^jedZg}LP8kgKtJwPB~qtj zs~JE?NoRbmgUB2!?yZxq;qzCv63ESq;8z+cj?;F`q*IyQL98+HC_b(zyFhJ|3qCwxV2#(WtRic= zl7j*%S8FNmE^M9-i>}Rwq=W_;B&|rjBg^)_LhgNHOthSK>m=ICLHYA(TTG=l=+5s# z1vH}ma@fTKfjIc& z`nlp`A3YZ??qfB{bUCJL%-##O)(TciYL$&Kkkv@{(46H9DiMD7wR6>Xam!b@{Dj!R zg%bf8l&9?Ghh{0BBbSi49%|&|eqKeG!v-y>SA)QcQIvn&o~E06qkz6i^Ca;cI2Q>| z@@W^T{9afZlWc^=C3J|3;0`!Fyc4lQ}{7DXjk z-bmG=hC5xpTt$D(%~}WJ-LV@;kXGB5B&v7!EjU_=iRKMxB#>sMbp*0$7=a%Z8>gpZ zCY%6TPj&6Lm<3a>zRuALDRJk(K?pCpdfBsp0usj22AfVYoJiHu&H=`;e>ezVI`(`eRclmg7+-eN(PU5%X?Q~tI z@i)4&=x=17;`K?n@$KhJIu+}iEW87bA69eb%XWS6zjKH&1rT$&3pXPX z^*+w&-BawUJJ#qI^1H3&@i+sST6PIGkYw%!>t?CGzu9qh;!N|`-L2tCQ*=U&7V*qu zlF1m|a{O$js+Im6sXfuOPt|8QN)-W0i!+eC(@$vDn6F!l{VKw2cL83H#b|GpIGSS3 zmO4na&Me|=Ev}cL(Zn}Fk)wF^lvhJL23drP|IxsXq~uI4P~h&#uxMKSfEqJwa<9^r zPg-8s{^=zGeSs2Vk5_`LJ;K%tRM3uX7lwo-QsBC#b_DfyyJmz^-}Y9a7dOhV=ASGc zEZ&J1-jh)4ojh&L?wp^$hW+?~@d_0rd8Xi61^sO0(N})I=CenlIkvR)MDl*p#<{^~ z{2{Lu-5}&j?kDU$D1hkgE1ZvqeN&9nbxEq%ky`tBn)a)uf0OB0(Ps4q`#Z2^~ z*!WobEq_nB+ba|d9tMhIfSj8VrE8x^CpZ|PRkqB)2&6zlm0kQ2wSN9W5qeVHX(zC4#nf+6B&q=+6Q2q%SNuC32PTe zrwt%-be1GFl#^qUsuk9v)J}baNR7MJBkMom_%so@eGa*J_eP0~&>F<+@Z2*zZ2S$5 z`;xCFW~()*$$#!9?rAe-6#*vIk2S~b_@_AfM)gmh+w5838ED?i@IF%E@`q}E8t1MV z@H5~9wOn&cvug4!SyVJfqrY%Hw;>~IqPP<;#|{#hwR04kpB7)Z5*9o$d^7;R6uYbE zTx;WjFR)YpbTFC;^xaTEbhq4Sj+B)o7!!F17*Z31El?5c1+T!`cd$V}Gg6boYm8A6 zs902t`+iqB%Dw)?v871}N=wwuk;|K;c{hz;BX1<1AqQId$k<<7bl${2CzYq%Ztdc# zz`nw|nVcoVQi#16?;ap;J;cVvc)}U)4xLM^AJ_nq&P2|NH~0KdaW$c~ojLFay^pt_ zp~s~LHJfQ=$N|ivoe#3Rww!6(asfZsV0robge9P#W_?4o@6T?$n>~?Ln$(jFYMz5u zYUa|u(V`})>I)mvT_MzcSF*>q&6x>Y>^!y|a6=@1K&Z3qyoduSnh5P$%$(4FM7vpa z9S;n%QPL01yPjD@1HHy9R_K@{x8V!j{S@&8wG`I9&25=a1roKxe7{D$T9UJ_>)-cKPSAm(Yx;S6 zTHa)uMA;x~XG~Hu`5Wf(1kgrP>5IG%7Rl7bYG2N2jDpq6#y)GRY4wk(0jKuU9E+I1 zKz?>UD-V`;))%;)hMk#vB&sJ_@1q$I!+2trEj&wEQ z%6^Dgo?1)jjiEc_TB&_c=Y_76EIA)ejknTD0AljK=j@p#KOg>Hfx%3#cZ~oVkP_~0 zW4gDR@URjpM9wBfw#CO$j@Z%We?TaGK$wDb20r$rSxJtpCgegk?XLDLLVk-%^NSC{ zbmc`wlfb2qxotPdnY)}P(jJor`gkIL>Z$t^3v25OCv;Iq%mwRuS!ch@j+Y2B!Q3ht zMng9lSifRz%3RSiqk8jsJi&kS2TXLLq<%`=ORx5@sVQ~PpBk9HaMBz^hPmBI(NtBD zlNbV#ki+eGTD%&H$&1UWP!JlgxGz z|0$K5$n16zfdaaK|G(amB>Nw#oxzoaTML$P&|cwGCZ6HlmNbw3Val?=zvVx~-2>oC zRhveamyl+c1pj#nbaZp&0^V zDxrcw%et9;c_A7N8aD0p;#)yN_PD;!aeHIu#eOcE6|;FX#&GJN25iY?&R|QPfPg<&+qnRV8 z_I<%rL35ZUifM-3**$yRhxS>6~#IzFIeU&GME~L8m#hmSxl-M7sGyFWsce0gC_NcMGLAV?)EN&=kpGoS9)_ zWD7lrNM0Nd9Xp^rnp*D23G=#{ba_cNZK(POWXm4Q66V1liSu$?xIyv<71~7d!z=vw ze8|n+mX%I*{(uG}mC-oq8pR95sq1z2F$%EI66$V4Ios@0RfN& zC302GCY8hcigqxixRHy5b0%qUK)KRo+A&5BoLS#QzR@_D(G;-_n~rY z{Y0HbzT~d~*ak1chQ!HAcbzomBRQBzO za-Mij)mtm)_X)P&9Zu8m6}HQC%chIs(LjoY6Vfsn-!BUi&QtwvCBIGj z#0bchrhrb;{M_8`9!&_03;?`=*ShKf8hk4Q;6a|j`cs(T%iN4$xrC)pk}}h@$knqn zAX{xYs;|1Ki7^@Z8eFC9y+8R9TVS?< z5jV;*yLZh~JA0FXFlAQFrFL@`@ghL{@MCh>lX#lEPu0!`N)_{j_D<2I6J~75`}S7a z;^N`}F{kGHKor8|dz^jtW?>6S8JTfut1aGYkvxvvytfk-+U}E-oMr4nIly7w-Udc> z-gG5TnRpyJzLz_2xgH{5R#+fOgm&v%+8PuQ?It05%|ToV__~v(m3*G`2eo3g3LOut z=4nV%v}8$zR_V|ZW18+HkqV+MpekJkR8UY=^fd@6b#`A7HparOuN~KokT>*0Nw8=b z#wFyhyP2-clkTs(ls?vBz|NB}Csd1csTDfD>Z|)sfI}SN>mq_6WbBJt#{I8^^bho) zf<<$@cp6Gjkn{%^O`Za5=b3#B;0{^@KwNzX^yyj|{6ZM;mm8-ufR4!NAjk5V6OO3k zKXe7pKzF2E*D?O3d7{d4mK z=?`p9hktdEP$%(%T7!UjBE23*I$^=m1x@jjFQ0UR`LKPEFvHXI_AK43Z`y=V4<;O% zTGLSF4|sGK)I;G2>CUMlkSUMEz6AVQ-f43zRdYw;z#4~f`+^Z3^;#V(#oc%iHVtJ2 zB5yf}loJqu_Vo4L7qz-H{%n0ljPO~i$u}r-*`!ZlmOY6jj%LLO_1m=>iS{Li3Y)`ef1!SRAcu{ zgeeFKa};M_c{wxm<*3@%Q{q>c;O##}G>7}3k_W>{tot$&5`ryv+P4gg$oPKLj5^g< zX~NMInydTGB)GogtJs8FLnEaQZ8X%RL<{nHC#S)jX-|Hr(&nYV&0jqHR^sS2!1sww z#|lFsplBs9yE@kU(iMWEN)vRpVBC3=@~ypuq6izNRX4sG@9vcNd($2WeZ%=1izU)| zP<5nF^|N`>(AS_JiXZaCOXL(Ue9{>UI(O6}RhdSfQ_l>F$BAT(m!P82Z_-50nfxBC z1agr)={#i&6LY0Y@2Ki^pODV$4q?|2QSn7?X**G zCYQss-QsPx;-RQJZUo6ibk zM&7i-%jfO=hI2$VdpEE2XWA|+d-awev>$$ND6JMTYjMe@*Ve&P#Y(EGJik=YLP~b! z^Nj7V{ca%ugSDb*u{SFojanwEQ(V~FT%(LEeIq$_GF?V_`J9Hh)Vv{wlj$HwIu2na zw%HC?KLPI~DhSjd>w1B*qRTzD7G+(J-Ex_Ftgjj5X8>j^iN9Dv)xy&u_R;h{EqoOf zF6ptoS%|)4erfErFBV4}?cho99)8y}qt+X$cLt_r4;$X7Su2f$6?zoAlsE}&Mc5=o zIl}I5ibGxW97{>mUpKAY4v9-O9&>putA)2-N@X)P)k)t-WGGcQO-ifdZZ9&-{hZech`7?BdH&rxcCcS>P*K+$EzRTXF8KgXM!x*f3Q;rNB2Avq1R5^UPV@6^jQfJvbS z9T5^1~qI{qV@Q^pX?Sj`+~ zk`FO;$aBJ`mik;4xLMX0Hc}I=g(E6PHJHQ}$t|TaO=prm#Vs8-Q`bK#r}x=OXPAH_ zp50EJE|rQ@Hp*T&UFiu+LHqPXY2dAh{t#Q(cbh?gJ(f^K);f?KqVjl z=vGcZoD|G+^2EELLwq@IuO)@7wtX0HE9z}a<2ueVf~0wYA_Wrpt7ZQ^W&Gd^GSmjH z{hL1Wv*%swtTx@ z8q)(5bol;g38k{$ZS-*Q)c4JWc~|{g6C+TMzAiflH=!_j`3&KK%IT2D>LinMAdu}1 z8EfF@Bd>bmm?!RYkPY4_H=ya=YtEl`hrb%e%B{dTij( z1#b_$spKNh-xuQVf4N6%9?(R1Ne`*tHCh;8rLKCcFCROqk5}S9&HGHvCiyV1uz`*4 z5+1Tcpv;tS(PMrvE@UTC9@Pd5-l>a8seDJ{b_>gs%^|(?2_Ry^&3~KX#+Q344{4ly zhXkbPm9|Xo;@>^6CWg`>?IuRaBcAv_y2Z4knO3^Y0sfTl3uz^$Wxm{tOx^-|qv7-a zGzR0rv{lT*ZM&&&MSS73+v^#U^vU*L$|4!_zLn(rN^_e&pgpj*SC=UnuJ}hcIOxmG z;nq~DzCZdD_yBno`Ol2G!{IdC1dfj(w0fSq`}$iLTo9a0{Sb_@_*(V(|bI?=Yc>NCtP1@vY12K;U6HR z5+n;>9~k)$M{v+&?X%sxTRuN`fiCbj+1~lvTxZAszaBc0r9HRaVP5|GQRhl2KJkYA zV5l_%Ae2;8RPctgXknmV3@j`xYa6snAYD5Dnoln+EG+{)UY}KE)=N@cTwGdu5~#m4 z{l)5E4=ND|=I3MoA&0@%831v89T3JqY1aT5EE>RLj$<(1$Ghm7hMRxOezFUMmf8fo z(AL&$z3=~MS%Y7wZBYjAOF_9paJmk0(__e zFJvI?qnXZ1Ecp%u9{@fWsMx(PzwolNT>x(sC^Uzz{{?O#+SgmpvdyaKf2$O;pZtM6 z=eS5Sd_G~U!nio-&&#`KH$k&+GA;f+0)gNFMgCs=Uq4)HgEN9=OEFP>%Dssai-dk6{Be0WAv0JZnAh@tlPKK_QGY$bzYSUsoBPTLG(B+ftcBDsJ>z%m1gJS#?`DH3 z{QIOSp`v&+FdyZ}`t}c-MX9j4Pg{|F;hfO#%Z0fyp1BDV?J!hVty4!1NC^%*-*u zpM+^V#x#<=8%YC^emm@7a);FPc7&ZEFbrnbGcx`%H0w!Zn5v|8O+!TP?8)&V6z#jn zYfNVL$2uJCsGsh%%kDh}qIQ5BMQ#qCueF^x=3q8%WlU3_wv=NBpLW3+&w+l$xOjT* zuNId2t=kwMZfu@xQ2Q>wm;r)>H6prsflllQ*{Zr+%P`?&-4_(-<|#Q;2e+O|EcVlA zYmecwGKyH5($sHSB(3T$WJewKK53NQTjY|Bi9J=3m*42@KPo`-u=(EyOG#MO^|dx0 zl`gQG!mKz1Pos}?XKtdrL5_E-u)`dOp%!l}lv5TC<7BqWjAP3?6)RWnsn3h^auv(T zT8o0sI)7Ya^A+K#dc0M|Oi)m9 zyLnoRQ#5r>cX_t)8u$i$8a~ahMsPMvH^i0$1KIZ&Z_F44J}d`&1HqqERRGGD(58)p z&|g*B-MpVa=^W8a>=GO5%xC;$v;A(s9g24AHIWRe@BC+Qj1sxyNABwHM*AC~POidA zJN!>bi+DLf<5L)!ahxt~xmqp`BN!aik=|{y8J!@%aD&{8}_@IGify@~BKKu}=JCSB+l9a)U6-;-J#D!YPIrt%A-fiPNcHa)*_2 zhosY!nMT0>cGCwZm3|Nhi2j~%V2IniP~?4yUarS6t>)CN`OAZ_Y273#I5{*_6LEqHddENBJ*1Nm}>meT}FZ)mL=V zrxL;z?{c+?I7P>0TFa#FyG!y7ibo8Yh#SOBvQVyPDRt{{4X(A1XBjxi9TcRRt(;`s zO@O3jTgiT}PRO~k!%7%L=rr0-{lkd%_uEGK?CCCENRLlYAk3X=ppF;|r;HC@kSQA3 z^{EO~d1Q zUrqT1b)&dVpGco02u&2D|65st8W~&Jd=8ZMvg_7ifH41M7+cfClXUNam6JyzPLDfG z%b|!94{;s^?6asOcMXF$Dn&?=Me{Km6?E!txt&ext;1$M{TC{#x z!j$apU#BPkTvxXUOt3ibcsC2reA^^T?5^=@fl;LkZm^vlG%nYj}9$ zOh>ycyWF8FIl(wK(pTYRFJU3sI4;s(`8TsO4s|q9+8NkRAP9DHC2J!O35W8&2B@8+ zpbi^oD+6WIR{;j>BdD}A%N)SK*Z7y|HWX%Bl_Z=gd&z24;7ZbS3%SoVt34~%=>VvMnZ z?zr6Pp|EIBG`#-i0b6t|M5vGn0d@;jU`5pwD-ToPh#RY&?sECG#6r| zxjOyQ@Rdd3%Am&7rEI`pP^Fal%mETsyee?_)ssy5d1s-KDJQpZhxWrKBBm0+|JUp83s!yAvQA6MH3bpO=-u=5+w?SB#48t1Gj$ zne}N-CYiff#uiY+KAuBl8_B)AU-B_CbJ^$jzCWT_5Q*)g=fV+}>)Y z@d{~&A-f8v_k@^KlvGWXArOb1>V5=Zf&&7DsxPVlq_?F8WC3WNtClLU_qApzud%D_ zcqw6yW1{q1yIzT?MT5sJE2qBT^;CHXvM$igh=Z2Kw8ylFO~8`)lZNJMiGxK<#ZGAJ zub^6Y(X^2UN;X;~rPWOu%R@K*$e?A8ejWeo$3AbxtluU?KDII&<<*HA=u6v3cdP@L zh4InM!l{qy^sT5vLPB@3*rFyluF0W;VwxX(Ap6iyUbZ6Q_2yC3^%Eu89nvB@4o*0H zQb&{_WCS7a&(z*;Ri%l-Y{rzL92%NK3FC5cr^_~kLph$A@vi=pPe+yhd|-bKBY<@+ zICSLBJZMzX;0GCA`k5GR6F07B*Lr>EZ5GByoZm%LyK;byC(~Vjm*);n;Z^kQMBO1h z$*`nxvZ^_jFVwO!$GTEz6*nY#D<(0~GC(vjEU;WCih({h+Ft)UgO|UgN_9jmZML9o z)+ykhD7P{Ma1~DhKu|z`i+FVlx|Pu`0j?s9!h-u5nxzM{mp|+dbZ|4hdXXH4TYtyW z`>k0rtv8vEHp)IaZf`k=XtYFSpiE`5UOKpMw5l>J!ji#J}WIx$ab}j7bHVx>J}}z z*y#e>^l8}H<|iPmgU>kGG$hi(RYmyPG9^>_GxZ<5XKj|m#hatLOCuDv`pDdVyiFU# zCBX$!KrA?((IF(r2d@otIqpamTiBCYWY72Eh$FltY?pQlA^|GC0LS&82KgLR5}s5a zZTjO{D3{fJo=+2PoN?GUC{3f|kRzf;r{xSU8=-h6PVgx1=yzm^`$>!(JuD%`&UtGo zkstICrzre(CtwjAqhYFODVCl!blmCaZ`!=1!N%6zchNF3Nt?YGlEvB!sd~dUUR9GU zmH0$aVf;gy9hCr^Bx_&6oM{SRRrwXy+S~HchP_r3NzKOQE}PW-?oHduhg&sRCq#sl zorq0p(7wCTu2cM-0lh@BHGDZBsMbPgGlng)UoBg!Y&6xNVYPt>tS3Wf1Qdo8a4mf#uO&qOxm4*v z_(MCH!gc4vjPQyzA>C!9rqM0(Y7Io*=s4$y=+Nykoq(RcN!BFq1A^Ge&x)4=#hID*UAdW-< zc2+_Gs4w+bdc|L=D^+N)+doeF>MShGmOLr+3~-?sj{nFnRT2!}&S@ESGLCM4ZoUMojoFBxGI7JvYL}_$Vhc9Fs zM}Lj(KV1l(*V~Qu*;8CF_R~New6%?jEvUyHeGCdZI>7^?SFz}fUn!q68GHIdc59|< z<|G=4%OsGs1V{!7_zSbqFok03zo-2V^uZuq zdipb?;aMs(|7)`PzjkP-v(EiExk9TiIF6OMp6r+Sz$wLSF>f^z zj z!@!9N>ig2M8NBOrcDmYxYZX`yF?8u;Az*d*<1=U0%LLCsg3BS{KLQgc}+HIBFLe+_n8Eyh?QCBrn!WgCfd8BhbV@ z_D*r8>!Z8hEsVU&8-~hhqPRtW);yDM>1$J!*1y~=j@~@#`}Ys7dv12H_RKWw$pF}e zi{vU?`2_Y{D!+~x(s1&!XQ%fX{w%eq4zGBhn8zjbV>3Bty_i~sImQC!8W)+nWOv_!swM{!yT5drJTXLNc;nas9nmCAF)AWt$Cu&8jZ$ksmr$h zlCRQxG)D>-&Yb%p3jyGxS-{FP>wGWoQpt^&8wAY(X7#LTs_aZlw{yNT5_Ih)n?8gN zAL8S&dTV57==-byn}P!CoqM73KE}FsL2Ue%Z;mTXT~)#9SFK;1T_IKk#nz}oN7jRt zM8|U6*dnC^DRoTn1vji7bGtf1HFc|*QoKr=JhVsfNbYc*X;HTZ*_z!es;1;73P;v2 z-qa_E(rGfd{#2cc+5_-qzPq-z#&Y-W#zx0amW{O_mb=F&7jA06M5kjQdRInzx`I9M z>?}U=ZrZfV?s9OH*j<@Pad+ZC4A!9sAwg)H=<7j9?s@u1)SYPZCb-}L$8#>-TJP0^ zBE@OwdZUYV+Y}vE`k-rbV=YCd^o7KvjAUsH9=3@<;lamS{`)egO&-#RU><2Zz+mRt zE;U__IMk8bW0GgPn&BGI^8BCnYAEv@MwFCbrdhSM zC%i`U=D8y&eMKBHogIaSCb`*UjAk!A`DXvv!>I&{kpjnd613b@Z5kNqEg=@MO2(>E zPq^vq@OsRP!FWkrf>gTZ@*P`D)p5ZsVYSL(ZW?qSZ~D>%d=w_Ak-`cMis9XCP!6q4 z7fXGKwuH@9%Sf`*Sq59%q*JNqB+BM-_X^Pk2MOuX_IlwS4ARDSQk1VUH28MJbw_uM zR2EPVsuy=sBws3h)>2gXAyYgYQ|Iz*$BP0ZEUTtU>cFU`Afo?x=x~|SKBO@p6y5;b z+aJ#-k$>lQcNwJKg4wbG177VFV(Vo8Qm~D_%^Ooh<&*Qj=!q_kE2jx^wOni!mF&*k z6)G2~6JU!!I|wB6dY*QxA3%5O>gyvknWO3|C#$60$$3FV@KS|oy+|aBqMEH-mohcA zxdh>00J{wt<&9l@?ygqsBSYxxj4N%ENb{)s-AkO%OGVe=8Rq zbq+h1ai`?O_VSQZ!n)wZ_FMTrB7_z#@)%l1a ziX%!H$1(TfC1^|ZOx8}*XRu{}BnH=7lz*tNeHFX8I)h?j{XRkMcxEpha)40}0K$j}4UwIYP!a1DU z8~@}oMk>{etMbjvNOgEf6G~&g9-=Kaa4%FKP2&;I3Rww-_i_-* z3=7HLD3V4Rx3=9Oqe__l_n33Bd^6&|D5_V@84};htL;wR@ExknyUkD-jj0d6tY?(t zw=sA;KRhehr%qM8tD(YMVukEb!zpf_r z2B(aEh68h9rBKCQ6!F5ZY~ADSCP`XD&Top?!znb;v&;kC2$b@@-S)wfqf+Ax#?B}!E?W5jCf>!jg4gN&h0eNVjM%;!5tX?!zE-DjQ-V|3+X_ow9Gb7%!)Rk7Zsa49do5Z$Q) zw#+z^5r;TFLtI2fKDKjSQ5x$E#i=F<$gyQ(g?0AKG?uP*cXhZX<~@wbZu2-c_&&_h zU9ZW?rC?wa@O_M>_jbIPq`3A_L5#D18INxGtD*+d+}snM7`__CD5?oQ@Q_9pQp%gM zoZIM9P}}mX+U9J+z_zpML{=~y+UWJB2)ihpk*Rd(2*-U~_j-(H7VBs(DePDci*|MM ztq}K;r@Md_ZBU79?r~ALe!rvOwn002);8Lw$q8;{8=KQUQWP#Sj#;@5thFU;>H zxyQy65<@p5Wze59w1T{p;POmkb4Hi&u-MUhIjim0P5r*lASPZeE? zAE};_km%cn1$WH&Q20`p6xB?Vo^rT`cJAy4`#LsgYh<-%mE#X~_x^bJ#-@0ArE-+- ztPd!r*2B?Gv@cWIChj&?PJ8$U4S8Z~|ET5IKYGYUQ|+~VZ$3cx;LcRzIZz4T8IA(` zDslqb@zU-@gX1F2CNd8@Azuuhh6`T=A6VCugZDbp!=LCPiA^FztIG24W1)HN$*$Ok zKSuRasM$ClTklGE+N4GhUA(AWRnbyldHK)AsPWUXA!&h2+A71_cFld12cq#=u_@&5 zi5^CFTJhG!t=^3+Dqz-v;qm_5;IV0*N*=>-lgxq=>ZSQZ9~e$dCo?0^7Yp#12?#7o zdQGA_Vkiqh+V$Nwofj9497;lC>I+^V@5G4eSO#$En>cN&XLVi<81}}9Xg@HI`f$~E zD8Gs$=aQq{E59SD>Vv{laxy8RKvsP?W`m_Yo=Yk_tpq;ZS-3BYR z3F*&&HocsI(4+J@=mHz3SJF2rR@XD`8juM zGhLNeyXOgwa`*EaIWe|Eue2I(_xp;D`#4Xjal*f=#cN-8%3w^5({d~3VDGZ({}Ipd z%iuX+qh)5?&GUWZ`X;ToT3W79Tcu^8dERw$xo_(5=kTgraP(V#FJ6VB@aVOj#BZn) zsW|$8x7`Ib8wTlX%3eJVkY$9dr`{{E`b)09L4s>DhhwIK4(PaBi|ow74`bq*hxKd@ zQF>K8yxoDEi23@04J~~JMVi{pBsI%ySjtx!v->JvV;;26Wz^rut_pFDwzSvr5K#6V zQqea`ak%g+j<~^Sf6}GygDyR4fuqmO=09m(kq+ZJBoBuIrg%|sHsC?>y?64&?0a~F zsFg%ARd`U)MT36cm~dxy9_IeNcC3c1n@O~Zkiirs-yOigXdp|eSY8<=$Lt_ox4{`B zn;zD?+U@yBLh=!q6hgq@D^7yhod*2y;){Nz@? zxE;7C7I|&1HmJ(OC_>h!dcs!^>^+dxj%8V|C;~$n5Ub(?p(uQd!ggYcPKp~X>l7BB z2Wv9z+jn143Y-Ru91!|KBX*lrp6mL|jB`H>#bPqZK-k3wbz0wnYVAQeLfGfxLY;~V z&$Vls^oim^PSyo_n7T^k{q1N(!G@cT{od^@c+E6|yyjURny3n<-%n$JL{;^57ZiU5 z&}6yxm?cDkvlA@CUHZ$r!)Yn0bP+|O?h|IlNK7lRcsL7V$UdyJK3#K?>KQee zW>hq8Sv1p7>tnGdY_oyW#q@*<7?O?gu2p%84!#5E)?C@$hcx6#oNUedS5Pg>vIfv| znhqViUhL^a1sl#x++&-DPXRDGo8lCx-{N6kiKwD7?Kr%=`?5CNsHaet&x3@IDBkdl z6%T$f<1P@SuNZ zp-R0=bs8w{y0q0Ow+C-SO`Dxh;3Q7JlvOpfn?T5r5^HF5OKph@BhD)V8aN z>kqX$tasOlhd3W7&Mg;)_rlY9@qQBOWdcDFRea1XX?CLkC%@qopT?rkwdg4Rw%a5- z#-!|(6%@_K=4e3`Wy+M(rAkmsoN8~LQFsQ*8JXqodSrYF(^Jbe@o z0OTG7dhk z0eo2b_1*}s5sj3b;6in-Pu+F2Bic{!@(?FKL+Si^4r``rDtq-*wfNecq&|+2XyN{F zzSV*^=Gz^2(y!3tU>_#PMb`j<{}$PFv8n-(T&iZR=fg_EMD;UXxZvkwT0SJQAALTwdN6X6tpaw5$Pt>0S&@Xx#+o2sEfVgS7==heQ+|{k2Uc6~E zw1D!!$;1{dl1gcrpsyP~r1zMGs{2)4y`&y=4UG5)^>uOi>2RTrtzyK&7Ot8xg;-?m#w_Y65t#y=v575*7TyJ6Y z)fQ@s*d8fZ6;?8<5MH&hxeTe0u)fJ^sb>-!B(H2TF(DZou5>dd&dID0g-JQh3)2Sh zsPUFw=lF@ zUlN1kk)0};5cyWUiGD;MSAIMyNwl8DBI=@f#Uvl)k2vL+JiF8cw<(9^WVL7x>zY*L z8O9dm*bDSp^ydiI4~KmaMX(pzf=u>~Nytw}xHdjw`)M3+xzVogW6ZzGsh^$Jv?tno zxmhWA(!G*rp_{@-cVzTd!22w~452;Z^_|&Cb;?tds$1z2M$7Gr0JU4}QRaF!cw64hRR z?&>9>5$zyMcjehIkMhV8^82`G^$7Xh4EI%d{%RW)v8d;`=t(nS*u*tuvE3L3j+HIw z2h_g&D&e1AX0Hog+4iL+l8llw;|jUo^hwb0YeZ*fBrsz$HLsN5Q$<$}g5D*IazKih z0MXzq!9zzWepUa7$q0IZ_};q|T3}y4or=LfZEbZdTm4ah6~8tuNYZ4}a4x9K?RpSUw+%jT@{$8clF?T0E%OWB}wxon0QmNT$h&oNcGH4^01Zr82j(!T3c; z?{UxP%27QVM1hrSbV8y;buu0gcaa5TDZzuhx0>mgY>_izf5O2w1j)E<#EJ`4Nmo4| zKzinIu~xrub~Lz1GVSF<3BlZ5jrs^@Y@U@=f|;emT(v|3a9vS<4RUyZDw+6)q4kAB zT+Y1kdSPo1lWABmcHj1kZ=aB@=7>O7sP;u4tzI5N9-Kn`qgu$PtJ7|F4S7T%F%`oh z&lp^!Kb^#1%5UwQ*(E|kUa=&K)12`l_h1PPacg|RRjbEu_Y&6&tvs51-n8w14=E24nD-J+V0gtiWv|hJ25=PEtdr=m^d#A;#%eleX9p0hx}wWZ7p0eZ)0xUt+)R7dwR~k=hJ4g zXbY?&stXdufLFhq*H?ct8{d#ZWQ*+BjvbsnyoV1NyeXd_egJG5H-9|-DR#6Af&DUY zSXR{JYnEk$kZmE~le(rEV8@{9(E44|yTqH5yZ8Z1X;b>lUQ@L;R#hV>k`>BhH{a3{ zoNwu7S$XvPLcp1fNO!Jm-FtJ`WR#V^I!z*mdRl+FsA`ls;juW5-G%3{x;I06ybbp7 z>N7)!RO%61?dz=19f^ABH$rL#g8s#k9qj_}(^{(7Yw&!0-@#4!=UebXgF;o7;2`d( zStMWbc~6^wAWw{|S|^|Yt!$5moBs7Ncq_;@Z+GTnaC4yT1E#t=>R=Cfw}LY`Ze5a# zjn;5TpPp-Jz>@Z0IwaV%IAU2GxN%U1v`nh(#+Z;xDhr3jgxUl>{&|pl5wvZsf~+26bH*I;-hu^Eq4B(t3xaJqI=6#Wm%{wcp*%|otGC0 zC$?~_1LCNs>sp&i2FCzA-y8hg(RWBPhA&y#6Cr!bJ&3Yl!oRpO?WVe!F_y>88;KLt zWY^G-{ufnVCUe&RKM9e4EGL!dRmon|P*Gf3Hr=0S$+25{k`M<0V6igE!4dO(f?<5C zUIPrRo%;eI*t`Num*fOk$}Px;iB>?8VvAYoHZX*Oyqi#eP@PN@4tIpWNC*k8x-vCX2J`3oF^7Y0P z*7g`aDSa;Lb0}k%9{&d&-jzP{7qUB6@ZVtnjhRM4HYh0w6MlyF7qfreKp|}mpz&u{ zD|c;m+`3P$`&zy*OT&79xMZ^kGy0b$c`|ttFa8>_tdO>vXsx=Mks+?PYw4+$FW!G9 z{hg()B484#y&S9T5)_g1;*YH% zUyKN6_WQ24IocF9UY&Z z!7qUZ?FGVmIs(blvleI}Po6Va@e)AI(P9G(;6M$eca#9hAsYVy_4WhJ2~Rk?4m5W| zgA5>F7KWVR#R0hR)xQwlSudbsC6CUK!E*xXXBi!3w!Lotg(07v0qOs+!27>j>;3=g z&`_lGSk51pi#n(g^^ZHX zBjX{^$;;f&5W^q&mpS!T+Kz=Wd(YFp_xmaM;g1Q~;Hf=H09hSnu!K0pgHI!NbvO6V zMQt8MHlF&c*ds|$MwTtsncYbLjU{8$euU*;-yK^3Q(Vgc0E$EPPHmU03rC?_W2gPo z#}wU8@g5nccaNV6j#U`VDOLIZ@O83XX08hnt-r+D(_mGZ{AYSrRZ}l{`sJew24j^T zbl&gig!0D{>gWZfoX%gr8U0wfWA=O2Rmi}Xo1~oc_Rgl0%KunZFj(2nA$9e6!+XN@ z;q;Bq<@%0I{ZbNLmRLIB5p9EHKNZLda_Sh>k9i`Lsn{=_S}l`3X#v01og}TMUfskN z>~aC*Xg!rTSMvO}<+Nb+RtTrjKnv?(-!I$4H%EgWhm9}x7n^=ad^A4bD6yU|Ksefj zJj)(_Jb7x4tozEUYIuy_&z^c@`OcS(O)^=aeob-uaI`zHQNZs(8A%XD0&k8|vwkMy zWpt0$W<MB_(fO+PuMI{&M)f%>TV#^zBQ{iaXt z_OGbS$}uS*_R>IIV6Z-$WMx6sP{i(zsa{IQz1LQP5?Bej7D$L=8N0vzEIZLd?zcLF64qE1u^jaG`3CKCc&;G<>16ve8?WmpB+?oB2LO*k^ zyK1M~F_9@OQhtO_hbWS_I&$aO!|<3)YVaAV+4EigGOEk&9aH-* zTFow`9fpvy<$R`!b^XaWvCY#i)859D`&4mYBrcs~+pwRxd8%9Lz4oWs(g6Sv$jjq| zIIV+6lPVY`ca0|U15vozXY8YrRwK`6TpG$!wL!02MYz^lro5KDOYmQNHRh|kBzJ$g z*kZOnd*%9z65yMnnR#AW!zguS&52jnp8tW>5f^{( zVaNHxJsVhcTHiz(ox2DDZYg#^lY~THk!gpsqD!mk!sB$nsguUia+Hu91Gq`MXplx` zl4;P%6*RmIOJ#VdEKkv<=F0>=6)r(t(4G1!wf8E0z1-4qQ3vE{E9VY9529!cKdixG z@4DNzMWF8;m`}Pt1ldk}iae%3;;-4xgh2(4D9cHm+87Q1>l%`Kvgf_wKQVRc)=NQJ zhdTep`yFVDL0z`JXX@RU)z6fJ z#`(^AsYGP3euhYgnbT@jCWk5$o7LA|wq*rnrmxG@>^Gi*I3rDjGdOw&;avv3MpvJb zR(IbHd&4vq8tQ}ayLlJyT`b@eK&mAjEu}Wtf9NdJHM4UVDA1nlj@`s`k=CU{6sKq| zzJAcqW2fj84H4PVpYpkFDHz|&tu~l-KSqb^!bha0gGB*7BP7Z#paAoMgC41-|D2A8IaxCyn{6$+V} ze2rcMcJ3;T$@D;p#|3D~m+KsDKI_C<;?0c{!7&fUjpT)C~ALy65@OTWLS2>Hp|_{Yc00Fxj*1h zfIU5~^maIEZA9dp$oY{5o-vU!)=elsQi>W?(fm@L-Pa8K$kpioVn3YscjIXXow&Y? z0gHs+Ef>r|i~~Jm@qxlw$jd&;SHrZ$ka$Kf+K-F1UEFex7eR|-Z}9I=v}&vCr%qLu z^b;o&KUFH_Bm3#e>{-PR9JNBWP)D`GeP?yv2 z^PBOEmv#KT&?Qv@pQM9d=8n4ZgML%tQA$`>4QOG3B5ZOseSSX;x&-;U0)+lB|8vFe z+%IS5pPC#_sI^sX^LyF1KwqHvY^P?U_uYlFw`EDJhMRK}z!o~ikz&j(kyKRnUN`LP z&YQ5iLqo;DibCW0_mFo;Gan0QJ{~%>AJ@D?Qyr1l_xA-@q`x%Gw4tD$TLJq-=B$`c(fAZ~Sv=$)I*Dv%FrTYur zh+nTJ{DEBUzH4rf{pL-=h;w0cv6%GK&g73#TSY6Uz4VgZ>wD*P?U_$%X-X%*^tv~Ov))F6HyIQ^fmG_@DquPaVvDw0! zxin}k$;fB**ZfsrZb0t#Y>GB+Hwu$D)l+W10RkC_HkcjftXLXtF#vn2b5Z+S^$!H7 z9}xrn8i3g33P>+JUUdFv=@B;F2j4k!yhe@kaDQjA>U%7@_=u@+s-LoCJFz=p(%=t+Az55~tIX z<+1fz#<#gB+dZoJQMtB_7h^lzB@_{|;Q`D}$Jp9M3&Wi^wy>4oymh`$sN@iisbgV< zqS%6$?@z9J$OakMr$DxVXkU9!&mnJjX`lX2q~k79`b6zXyE|&W#$lZE(XR2@==y;K z0=j;@1@^%a-=3PH4X05X&keUK_7Kz!I22geAK#%WW=BHpwlvfE(x1g@`oIKCULOkH zBs^^#tm1pA^s?yw)y+DwaKp_y-y525 zyLc$=W_L=5>TktYfEpYl`dn(K{KPKYPAwf)&>3eK%mA@~o@TpcPsRzAwms!Wm)805 zuYL6cc6_zd7OD;;&BCW@u@;7xX7yMu!yP~ZEfF`y4mrwLL0f!xy_rf`7{)eEbLU#y z%TD!d!JArTtfa{ZDcP?cQ=bbz3QmV+I=ra81G(DLdJUdO*!v#S)tm=Sml3xGuPYk+E*ksJsepcEryu{$J}&1k zMICv_B0qfln)$0WRSpRsI$eJ$fLuB519a~a1c6`bbd|q3eL5rURN;(pqf~I&dl@SP zBd$VFtdn3q%MAVNXS267q#mBT&FW#Y{_SLtBm1TT-WkdJG*15Ek^Tn*cAhJ~iF?tS zTrftb&%f?|vmn@WJvLen_EUDy{&3-`e*DA1Nl;FZfriyjo`!IEoq}3s z`pZuf;oe{A_Ph50!1bRS23Cz%8H+CYLMS)Rte+q5u5+!zZ= z_Lx7pg-E{~G1qi)W-P3RI(^%19}%!|F;Y6H^xR2?r9bQM6UuFX0(S!n#i+lPI&ZhV zs6F;u^Ja=%YvbCRTXz6`geJ_N9BwDJJC(_uh?gvCBL0mX9;9=Fu3BC=j&R}5h;;j& zR5sc^kO9QI=6+%51`EdBCh7_z^`Y^h$MLTL{{GVTW)9m#`TXm@AyBHxdu71bNqFUW z%{>$26dLbg?3iC2*VKA)Ccek|oV{hJ2D9{-OW66A$Vi=4SK9?1XU{j5jv zUvc89zLjPC8@&DRxNZZu`dM_dzbWZ4Fv`y^|4kNQfUN#sDdUdm|M_E*3EFx!OXKb` zP~o@KhV5{uoGsTuEY{R@@|rIoI|>4lN~g$d7W=q+F73c)NM1 z$-AQ719i|zRjY5PN?|D^1(|Y?|JJ8yQX0R zD&0Es!H^A!Z6o{%+ z@U+J_AU~E82lo`m-&slNL@{2z_N#2U>B_mMioNn9$q@Fn*nf;NH9h)c8YOPh6pNmz zIu|uLG^|v}OkCLp1l^nsEdf=B; zIKByyp5ap#t}EM)8K z?d_GNvBX#D{#2UmDHWuX`WP(M;~imIpl7m?3*@-!japb9^p?{1=-O|QNt{IQpj~gl z{C$K@=Bj^nO7KBRTAPV;c%gE^ciH>7eR4ifPDLHR(uiJ{gD6u%AGKfPS1A|CD0NuB z;AF#4(=(iaP45@mp290$pv>Y*jAc`lP6nB1mwX4201mXefpUHi6D#bs2JpKuz zk0i=3N2v_gVb!s~!V z)u<&$7G5cH@8J9>El3G}jlF@%BbZq4OLXOr82jsubG(2U<8g56bC0iGL+_2w$1kqu zXExg3k4JQ66#UL7w#v)f^i!a8rbyK7k0bqbu6$1hFZ}$|f7j%DL{j_FFtw~lSfCudEm*XXAutn^#8thLn4KONPOy#^dI>!MIOisxI#gs)--wB5&IKN-Np_M z^}jvmx$wQ+^UGW5{-dK0{~LGj8P!zRt_ugGDI!gZQdN4DjucU(NS6|jE^2@P0g-@o zP(+%5w9r911PC?sBA_4%y+i0oZ_+!Qh0pW6`#t-Nz0cU=JHNj9BV#0)YpuENwPv}l z`;?t;(-hO%o2@>aCfwqg@t+4R(PY;eMIfrrmZ)|!*Z+PenXMl^_`y?2b1Ho(sJ%1 zVVk!0cIL0P!Z&Sq%}nF0ZoC>6XxLGk9Ms)7b^Vo7x3jYovN_VclH;`4Hl_Vdrorp* z;(@rs`9%KYnaM?+po8N(LC4^&MMc+6%duQSSqs9>F!kG}y@BJUy)G-_pti@-fpvvxdU=Nwo2&zhoV*(yeP#mq*{OQM>e;}He?JMu>s6Lb*f zHWlPbLn#w4mh{y@qb`_sj7v+yESduf2|zauI7S+$-HSEW-fbLXAxI1033peq8)qe9 z$CT4S{AIW5z3}LhZRJQzdoL_~ECrE6d}Wf{G3K^h?~#ESk`ZF&SMl@|C(Vfp{|sF7Y+?Mq50ALYlgdg&ceY(>Stm{oBHblymwdyq!>U?~g~g2T5` zu}msvzM*R0ukLS8@wqOFFfsnNzU)Rkbe-oWXskeG=QcwHB-e!72lQvR^J{%S;Z>%>Q6twcQddp;Gpk~10 zV@npTV%V0mkPK9BMl#9gOAJuq!N^?s%FnG}J$3zsu&JZ!cAHMccit4Zr5gndxZ&q# z97CZKsqID!Sv%@s;p?{a-R+7e38}y^cem}Z;lB9J_lDLfk)5ammx=pyGVZ3WHt_*P zbhI1mzr!@vZ0itib`x6(OF_dBO!B5xe9lx!fAvnwK}TNx&(!YZsgsVW)VvFlr)t5a zNF}*g2q8EBPn+Yz!>m5x8qti?_35%i$kjL`!N7|uWKU5i`c!}5LeD5GG&}FW6Y8$C zpYMd0k16l`%u#EUL_a65!sMO5*0)yB%bw2$ zQjcMdW$)*owD)(6WKh5p^eZ-#G|m0z@r9HC?(L&*9uG+VL6-t~*xd~msbn)EiQjF( ziOzlNaH;wM$tG^&UsRNZ+H%#eCdP{i9GTVf1>ZjwnpZGg$z^sbaCb~Urwqv_egAlH z#np6?^^^ZmTRsUfxWp*~Gvv)o`|VB2R*mNFs+5q~aFDhl|9!KpDp%9%`S${LUU^4T zv_^Y{TE!DnhS~7Y*}SKlHP6lcF~YPoj0@`!a|VUU!v#hKCuKv<;mAhS9r-rTq@4QcC%YJoT3E4P>2%hyq&ei4KVbIt* zdhc0LMYHE&Fn+m)Z!WOB9c?V`uHfhOu`d-F!FIc1c|6_a>*}62MGR|ohsNF$2KExq zV)W+7r~%N-UI0v;_dafasIgKlR4a65&dGzHzR^bWJfl@X#Kz!(bS-9BhUW*u4O;cG z1EqhAd;x~lAreQem2?MJ^)@!|;Z*bgh4OH62HUMsGs4*ILr zxCj*LFb-;?T3k(el$-;jRs{x3Q6m{HHi|0)&aru`Cb^YNiNsD()nop)$`M*a_~BrJ z@QwA^;OQWvi6Enet)t6lw|J`v!W`~t6uQ%AG~Ir&GJ4bGaPcCb4ny?t6QU5 zE`-;=?Lf@EF*zOKmUhkKWs1Lac#yNQgl%ubC&8JWnjO;WxuDzY5VHtifS98}=eR1O z8h)Mysrf7%oxYa-3MUFpxJAZ+Lp&h?)Jo!>q& z5Ty@6J=SaP1?LXt*HsAOab92h+8+$=kCK8qJ71`|-7~B$^wX?0ziS~aj1e=5uzHig zm&1NnO7@bO`2cOZe%y%%&O2FmOzht2o{s8*lX)-po$!)EA;pk=5Q&CNd-l!9?j%}9pbZ%+|YFU+QuYz)X~4M#bX`?F87jh z6J{=YfF-(3J)RTreY}Z)dls#6xGn#NzEn_9v#WhG7vQr!&vM>w*!EZ1)q|q4ebqih#$uK=IiqW$lSs$PQy{O(24t0;|zi#&p>Vhk@lzt<|*X95>P62_y zn{bUXuv218`I%7;^=B^kkaxpoM?;w<-R*U0pt%ICl(Q2@S{)9NWE+`_Rq z%s3cQ)g-7}A!8rcXKF~*ClBbsU$?b_Q80-_;Z?fgTviVN*9T+UHTAfL}ssl-h#gLw93lBycCjIi) z4!PZ(V9`w=RGm>O-0z|{@2K-s(B!zMZ2qa*1H@ZK-j#X(lGuv@J2+KVD_Khk!5hP+ zQenq;&*WjBxjiPDrNPbbpw~)K;ox8`Pdo^*I^ zybzDd9$+zWi@b$%;*U#fis;oMZOIO-?PZ0>Aw^q=C||DClziz|Fh=EkKO8{N4XvWz z_kfGXi&L`z5SZgKkxX?pb3IpBEu@cLcbRRlZ~0=!x7TgbN5l{z>V~+PCDashC~kRdMWN*KemTo=1vLRjXS%%|E8qO_nUOMfr=CrvhUqE9 zBg`~g7s>2DKf`a(v=TqRS^PqwY19dZcj&f*IDT-22dyujjs}ijkbp5Q~aLYf+ zemaqD-I?cJrjA*aeC#M;Oc6Mmd;>o!(@{c5o~Mn)C7wTo&mSI55Y2K3 zP53eK{n1kh@p~)0;=21!qxd%1TS@YwecX%wPKMYUGC2<96t!f!LMcn!7RH1tgu}meeb-%xyWd`qWd8Vlf-_echv`&NpT7WUQy*u3eH@ouoHNN^#2DaF1-MnI zwt}D`;=g_=zkd0HdC4B$-{%8-dH;zK14iP1{ntqku#^8PA^^Jh*I)nF!)#3`-=)_X zomX9wNhULIa$*bL9I0Z;hgEiWYX9Namg-6^hVUqQN1tU+a7jeH>bq*2l2zA3_4bH2 zhT*W;Xkw`=6TlEq$cfk#t<~#hVXZ!s*ceKk(Uir!-cDgDjmWz4mYT|KG9=uB4`)Ob z-V2SSsS%Yo!feS{ZASx#=KWr6%5!;%kx|is5t5mQl4nD+)%<=qhX07khNkh}`v@6D zZF2XL^Qx7Z!~&1pZg^*nW)j#*^{2os0nV-5lFLDRTZaTBFQCDy`DN%qbhhGuOUHaw z&&Z4(TF$(me;51NO0Xdua{I^FwqH$Tc1I+eCd7-c*JvJUb~R~x!Qc;Z6O|~YNVV8Z zYU9@)#kbZb+u;4*6En$osWtv?py1^bviC(e7TNkSl$aea zV|=OQ?7O*cLkh6GEXMA4M5EHWG=PyHgji-1fB!OodZQuIS(b@ayn+pw z4Vvu?5NepcV`>5F$-9uWG|z6~`np^@t2gp7%ChUC_#RtYmv=Kv!-w0`yJaO@pphv; z_Vv?XE@FMmu!xT=(^m+=;Eq%;XEyiIYn9?rQe@sMr3$PWyV^>f(n^Be^Tgq^MlT{| z2H6LGr_e%gw3U?3*J@s0q;#Pe<5;J8Mbi(Cu*DYQ5cKMk8asV^d|v~l^RBO^5T=<- zmBbMI*UZ5m%j`B^lBG|wF{IQ4X$0^-HDOpdjcYn4vP05BHu{XGOH}vlwnoNI-;Qkx zaOv$AN^)~MLr(URK(l^@WJZdxocI7$T2&~`o z6z`jkeoQP8!gtftB<}Kvo8CuTm;bPsI^$iObirNDie7jqx;j+J6ID4=9<(pJklSV zRlth9_A<>(S2akHT+%O(=U$up?Wf%8(2l&8)kSF592t_=$@U*DfKGLVunMZ4U9s{N z(ncl`%1wGynzO>D`PmDZnf2E6R@+2M7GIVfE@c~3l!cMed^;1+5{whp;GI9QI>j^V zzgKzIBb<#XV^b~e`TpDuonQ=>(wd{Um`>WV{@Wp7P4%ZYZp%B=A#`Ngf;B8o#Bls* zfnb^CEQGnS5DZEj`oplVG|qh?UYkZSsC3P_ulEa=XtdG(wHMWE^O@cpgJgc(8jf@k zn$MhgOsxxdGGp^t83lDsF-vT!JoAEe!jJ@;`9ZViTF>u%ygw*^7pF%guO4ppl$%v{ zD!;~UW2{9AdMC3ubIt=I(XJMtMGb_vyDUbK%O&6BfFU2_wy&47V`Kzl=mcaeI{sP@w%C>4%7hThf_&TU zEYX=U@;;K`QSt&9$0BteoOZ>l5VPpm^X>ST8Bttf_Vi)wzF7I#n4ns}=nA3iHD^+X z`J}^W3O>VS%s1eDL}ZnZx8K6>sfIGRznZzVizAP9Sqd+*q8*m~la3CUqHeV@qXC{( zo$_6V-hr-5Hc#jmtidl;oHiIiO9P@$R;rgQ0;NTRY2(E~pS!l-VhAuL{#_^Qz{1Va zXe3Mu2wn*QQb`1T1g0dbE9BZ2onV~<=^=eZRDjy(HK}f1P~H_mc?|(1 zoZf%eZb~q!U^q0CvQH;W&Mf4?{Mxrn=(q;3YAQYRj@4a0^p0nn2#Y|h*J#G`6~08* zeN*etOuSJflk70DD*uFkqnwoqhuKblYclW7qf#S#CkrASmL+x_7;14Ux1EJ8xQCti z>#LLtdljJ=;T6kQQJ=)Rg%!s&c3*I9(E7eF>nM zh1YZ-+!9o<+O+?^$U2jTxmkSSFdSF=OsT^yXOh<~_>URxI4OrQ#yekE*kuGV)7xPY z{AFRYGOxFD^OJvc8+Zuys=V}^N^z$DGsue?2avN&RyocMJZTGOA^x(6eaKba?Z^mX z5HEHQU;5!+NFHe2$6xI^OqJDgNougO}5YZQT4}n#d~a-#-S990P%j{;4ta|J6STH0A$vE2cYjw?}bW zG2#DZm`>2e!>4d&z{9x2C?X;8ZuWbSq4M7jOOZFgH|N#3jyqCvTixQl4=nBN!!AAm zk@$d8E}ksu_!4l`3~5vJB41n{UuHic^{S&F9-`^cbAU$R;`!_DGRRwZmBLI zz>l*!T;#J5Z)M3u@>c_r!QC=D9YD3kg@wx>|EejwI#dX_c%EN_UF94ZVSJF`_xR5blc7p z`sA~DPihlDxNy|U@E|jguYRQDm1iCO-&ONclO~ESu1GUeBp(1qmkQWB8N`1f==SHj zz?GeD+b;Z1owLBP|EKZ)J|S?*|K!vC%=Esx zj^H=`m&FSa&VY4+93p^YTRzY#<&UL02C0(U4%-r!#{0Yb%v@*u%WnLYDJ-xn@_Y(! z6v#p@bLj(kVCn6B{ym7gqwlZ9e4!W}P&-hW| zOs;L+4-Y?i+>lGnz%HBb1f~Z+w{160E1HTFUX>JxC^#t`u-;A{6wwAQpd!KFhM=et2oQ+ijOF zoG<9{VgrHOH@zh`k6+Rtc*~yWe?w<~>f3KdUCk(1Pg#b9u!);Z7nI*iZq1^uqIzb* zyIttSs&=BrKW+<}uD{^i7~7J!!o|)~$4*O_U0Hi_2vd*%`9AQKZVYsldvj;r7HK%jKkU$&e&0QZ^0MM00Yq$SVa(DM7hjhzq+FK{ z5AT#vek&35+FF)qI6<0^1_ku{oc_AU*%p@SIo+^%Hc2h$b-`rdbM7TLwOi=L;y!JD zOq_zj1_+d$@2via3irBrV(xZZvU;&$)@zyE{N3uH`T0@jN#pkOog$yWC-cOcM~kw? zg6B&o7lW(J(!0M$aYC0}sTz=;q>DWF7U+`8T&h-b-<4i_G_(ywZ$`^< zuGxSfV<yNl#Bh;HjDDxlKyPN)jw+jcp zq1u?tN6=ha30;SMo$hKLg_&Hzne!|9_2}jm$bcPBc$bsl2d@gYJ#`ExL=1gyW0D31wpFct}ME zKS*H2wO0wm`F>7|2|wlZoNPt{MRIX|PixwG&(87Jp>_vsRjUDPQoh1)DejUIyrg;b z<1%MPUNII8Y6+?FA)v7qyI)erJ*&@&ffItVKhEZk`X|w(5@I;mEbD5J?hTmXvkH%2xP5&yw zN|T!4#Cvzs=_0JWD~}FXn)bsR7HNv2U*gn?m)Tb(G@{4DOfKcqn_lz3WA{&gN>{QR z)t}s8uHP6;^*$RKh7~X$W8}`;_hVVkLs&e?N5t1vFINW%-!4}NclaTO*nz%i2qZHX zZrH@k&fNVws`Vc!b(61EZ@oE=Z|<0R&Omyr$! zbynGkplWU|q`q17tD0{$B+yb&u$uChsJCisIbd|1D)PL@3tjcGTjcdVLBQ~dvy-ds zhr37DZ8ifgrlnKcj#+9Kw)dUsWQIhj*ZeqDCPbfHTM z8eTd0L}X=Q&yrU|%1jPwKvVO97w^$36{h5?9Vt{BbS6Bax~f_|v`r0!>HbSr-J($3 zK4!tIYK%GlziK=gkve?KuYp`!y4o7=GkWdC`0V-f;(=nl z_euQ(wd~yX<{Nir6h-(L2?aJzXVA(4xXG&bMaCrCO?#5jx|893ZJ-=*C;cnilL*) zl&V+Xb3#LS_kD=pO@=GqU{x)CCxV(FvisgC?dTIcp>?J0_lpk>cCEMo*tHuzbg96G z#{V}`YhFNec`d3!rcz9Z&ChI-FVU3RW%Zixu5sv32ee-m;kRvkX?xebx|c2n+*LU!wxb|Nv)W(XT|%fo~}pKn|_+ZK%h_mW8TTm`-T66{h%GO zX2yqkSBpxf=b%i6L+hdOy2+5$PNhD(s<3A+J!^0iZQFq>`E(QA2mC?gO`~)C1z3f0qSf{lVk;cfgiFGvsVu0ro5SOf=v^f-%+8U zH!xrG6B+xH`}-_yyq7<=q8ac&Z6&K3l807R?kr)~{!111)_cKO4fD3_Lf4bfs&yUE z$e#ZAmDASxs{R-K`X-xm!)CQo(*LNQUu*z?zAWs0V2ay;;&-e`QpEkZ2#{^17P%D<-dZBaDn_%`_8g;_ZsVB=BhAQt@nuL zE_YkoQWec{7);u%XKCJ2(j?zrZxf>$(Z0_a_d$9ZR*emPbTTx_cw0^f^p`PrrAkMJAvXK=H{;)Lgtmn>OEps9)p#88mmHy3u|=0YvV1YM_1s#0Q}oyz|c^kQ#%Up z>SJ;h@>og>V0K?hyJVFXKmS3rU6IJA{U)&x$fe78^|)eCPLI}TL>1+EMxPPVR~jR^ z7r%%PJxQaof{K8d_xdpVwhdmE`g}WVoz8~0?!>ISO)f!1At`Km6(2qo=_gr?8uTxT zwqG;7eJOWMd8Lu0JFhKB<1H&WsF|$;RBw@K1*^ z_7UN)lg73B%C;AdINfFA#a=wX=51os0thMZVP^Wa|?IxD$G#y zZu_U5R;*O_T&UF^G950}siaCYrLU;{(}KDEUVyrS!fLa>n%v=UUAZyOTVy-VgaTvA=gnw}>B0Bsso0ZzdA5S}HCuU3FMl(O4z0Y8>WMxIb3C z{j=oZ&dx=vjU|e0o4@F2aLv^1SMP++TIa!;zer`W+v9W2>3Z_gO8O6xRFl>BDi1bR z>)zQ){M1pMI-WN1{ZiU^tH~p7qzbwoU}%h%x_#APO=G~iXku#k%IIL|J&ydScV$;w)(Ddd^Xnjyk)f3~- z46+@yEGKe>N;lcHb1`4S`q27UzS6O0VZLXlNYb$qezLtMyhvhj-?0HtdE=~kg*ia}U8`id|jPGs1UV&wVRXZ413_?Sw35~-@Go|y0 zK8E$@o*X{Pr zH{Ntu(qAVk8@0QCk%klM-s;>jjLW%0xBf!>O3nI&A=cXptF%bjg)`6mUW|!O3n4!H z>9HkvcTG9$)-mPFvgA>_(W3$h&eaG(<#MTa#Uml3Z`Vwv&u*=Du6wE;JK?&S1vp&2 z?1=*wUPoYY)>hu@L<~-~fp6Rh$;h$H4=CF=?9n0}9P*G_;?nV3A<57+m;H@$-CNL)oFiL8LEqI8_wF;PL%3=CFi)Mnf<zW5*X7!NN|y8AZATMxXw&M zIv203DmqASqPz*-tyeunl)SVT1=QtW=}Hv9D9uO6AeDP%$5J6~_*3ef5XxCTEW3$V z{7|8xhrKzdNnE^h`#D@|f7mjvU_!U1hAUY!K-%bA<&Sc%dBAxIYJk<)Pa~JhZ8}Z(3PYuAU8FQqMYbhZaCZxmBkU2-|8(Av-gIbfs+_lkJ!bIN|O20W925_;= z??mE|LvBfs`ux#*@bRM5J+q(I)yNNpD-;kVy;!XQ`|$5aikYCV<=dg8&*bAJ9}joV z5?%u?o7@v5W&aacDx+LaXL(`=z{&!=T8k=U5PLW{w&v&cF5YVfdpW?c7F+z288z=o z7vE!cHYYq&X*e7N*12{zg5WL>tCq3n8I1g(-x2H$G=9FL+~8F4v8L z4LEnTRTWB=nnwihVaCrp$cOU9BrLs|ac_w<_gV+@r@rS+o)!!Xv)EgCc{rGaxC+2N z+g5m+4&7DuFY=FnD8&pR&<3Jap>^qp$sO3;pbw(%Zk`?l%BC@dm= zv$O+ewWez6y_HbyNps{Bf7F(GzDKv>gRi491LWpfOA3g-*5@!aDF;TrFnzw@uy|mv zdX-=kBs;YxTQg=7v!PLIENZ^K_NkXG#)9g#ffsMq^~t$yt(Oy-54R)2L9WBKlg@qb zDSAWSn{z0^)xKyW;-6`?&9#?vegKuYiNS*qA^^s!l3=KXIT8yb} z;s*5PJv&=GhvZ@HO2efQ#+WYyJL2G*(2*hhKKk{siq%21xOK*Gw|4s*PIFOM^lsPO zA#=>~!*8*w!K}`)4thpS4)KK0bX1@|54+b$uJ{`mgPy2hSqz7GI&8f(@d#t4sy1_$ zmi+|xyw^WH^ajkyGt%kd-HA>u&r$yuFt8y_TiZBZ04VUt^?<|bA1%PkI$;dT^9N{3 z&-p%KoKwufXtm^36Cv^ol>B+*0 z78xyjpa>(XfnecBw5@X<*1f}QZ`SDSo;aZIb{n*5L^(0HAkVyMX6krciVN)4w~Ilv z>wvbqchEWAd^iZA$lp_>GrmR_vhq{$B#xiDc|;iHjc{Y0Db0QhCanwh0c(d$UWe~h zU`6+5+y@m8j4he5@#X`RMZ>~HJpKmjEXd>WKu_(x>ZGI2747TCB61@-{<6lusysxd z!cU(muBXMD4+$4>I2!Zsyxg?pcWT@~A7Fq%DL;4&61uM~(pYfwq2&T`05G_{o3XKM zxJY;)y7R5Iqj5XS-0Az~?qGk3qegAj+;vz#@@H;FWl`M7QygDx9V}5iZsbwI)5^VV zA?~SxAq88BytGNVH!WxCcrV?atodyU#sMq+HekzTFTeZ}8v}cj;c5X|LmMotx)W*x z{)UE3NV0%fT?-zX#2Pyk$^TB{zcDt+F)uq^Pl4TBq*5M~L4p%?f~Vy{ABAe{jaYv4 zp=lHM%0WQX-Wq3%-bSLD$nFgHH5gs`!HeOculjR0XMgKjSe!isL?Ggap=LEAv+HT|fb-C%J8ySXY4U`{6kjyYG$9J72`BY(nl2n~SV& zkymtdTV}oSF~JVwAU5#^;1S#P0!wDwGXYN9S0*89hd{*q7-_To-iNcnChZ*uyf%VP zie~Q&PTS147E`c-p1&W$Q$2_m>*y$St|iZJH8o8RtYwW2pPafP!3WJ+jb%qXf{-hVH{^OYOu zn>}-RJlf=A87y&`mMAb3-4h@j-@qNLZ+Y~L6rM3MxlbvZLO)e;qf6rka-O5t$oj0# zctbePF@?#ns&1_(^9i-IEv|x_Ge$KaR>g_PSW~U2-?Lh;D4!=kYG$W)kXAl$+Q5M8 z5Ok+PSupeF7nOuV$Nulxy!Z{{RYBuWUI*1#Ncpv`m zc&@ls;`6(D(dFOMtg90=;wLAtn#P<(xr7;e(c4*suS+EwAdQzAZktOFH}7v4m^f-b z_P2p*^YI=$yG$V0)iK&B)*O&MVW(QRYo7!n+M5CLy>gj%1Q_8N$E%h55W7%n;5-@8SNev$|o(>+vVC8?xfLo ztK~i+u@|HJvbuW~o?cP;$oasIOY7_cNI<-|bW)>|Qz^Ihi9 zklq;k@Poubr?*lQiP;_jU3yzxnd75lhULcUCvsTVcSak`bO(hqKvI`lkmu497OCY& zflIC5#DKSkP@cdz2J*^_ViMynP>lvxx z;H0B9*%jVuXC)38sW>)$64D)6Y^r`DuuRU`>7lG64I#X%uvusJ6AT3>S@${^u^(BB zCShNZ(JY!GfXq8%H7gs4r>q*8rl#o2fY*kGx13qX32>qlAu8Or5^C&BN$yltAPWbK zm?d6clDt4H`O3z#bqAjloR-wJyEPWTe)J`MR=8A00x~ZfO^zr3G%JHI)u#<1UW4Lz z75mFDsT}fv`3X-{W_)WoUyd`PFoz^Y2+7q(ow$$}RLZBf2R9)g$(k!SR?=gK>LBJ1 zK{|KQfuXB;1CSkFaINt8`J)nKPD_v)Oo%<1tF|el-ahZ z)oOSfWAk>}?~4)Vkx+gIYQzQ0b3<4dX&fq{sjjWjijG!oi?FXH!eoCAbq4R&&d|HX)xBX3QYZ z#aTZe;4FpdIM%6qX-LFm(m%pAdD>jP9${@!$`=h567c&XrCN4$jS*GI`}3*VqZ~sc zu(OcBm5&1)SSsg^FM>Xle@GfsEr6!UH~|gK#8aZh?o_#D`oJX~ld3l}8?~Dcm$?9? zKW&r}`A#5trQpb06Rb?Qd0j~9Sm|}Bjerm9ABIX+P)l}iAYR0(V#9rZ)Cy0u6VhPM z7JL_!qgD1tVWh@F(Ho);^komGsTD&bb1m`(H5vkgL4B^X+0H)v3bJ75!oT^c?pty) zchc#o4MKAYdmIy%qi)P+wo7Vwp^YGSJRIewiu%_wpZq1txmy$AxO^p@p@Y9<_iv>( z(|h1j69rZOdd-dNV4lI5A_s9nD=6{zZNm=|mX_kEdALYiSnKVa$mi(n>LJ>~s zgI^`@Jbqf3AsH0?W-wDhN_}9Z_&mEJNl(+NMFFpZkq-9UL5ICXCNd;d-dK~DH=Qq> z2KAZ0t?*XXu^aiugB_S5Uz=-;$E6Sl#SUC(iKog&n5?}^rJ=T_2{7~3hEc#Xvf3>H z08z&TD^=F23Z4*rH|ATRrOwZ=avCUfPaeK+PsJToJ>2um+fq#HnIKP*J>lx<#UvdvP*i7g=G)3bL~h_T=J2M))=HC9!1}G zXaS+ggFHnxJHOXeH8aok25M|{LY^NFJs?kJ!-y9;nZ=2Q&@A|Hn}S$1A_3;0fgd|Y zc*i-@Yd&2Y>CtwThI3-7Y%our9a^(5p`RccYn@#gRXLb#k3ajno-Cb&Z`Z@j*?H2x5&IMMFt-@s!iWnL)x$RC1P6=mSQWdgpb$ewB* zw5H9(KY}bZc;oniKZNHgd&#~_A6Ri7o)8}W8wP|fs8d?01O>LKo6-#Db=Av>enF zN=Lju2QXY}<5gmANDHhAZ{N%_JX8v8%Xk}@j}*-m%bg>M1Rb7=WKR>3+% z<<*=X5a67WJizE1Vh$tEX8T3}x2FjV4O)@Y zY9*0=xkEz38Ke*TsCie+J~<$=f^nE9)TCc>)9-nQcH_OMI9Tj#VAz5|8yG!UQqeh& zvi8+{n_<_xPmb0gjp}3wkE}`-74KF0XenjC`-WcaQ5@T#(p3P|D2V~VT$_>fDobob z8W1K?0~qVw8cKhPk214QysH z0zV?wT}L0K%poe@FFKfws26v#NFIWOG0Hm`(fUp9pds$4t_v3^^u7HcQIs=_GV*Ci zpZj-tf~gA4+w9JfW-}z0V~!aCFgn!?F0=LaBP&`IPO@zjlax(`f*cr9=Xd(zpxXHs zMGLwl1h^_mhHM=mw!bP1Btsn!WjK&@TvPBQI{fX`!)z|3(c)@M5lTQcr++4l_BTu@ z*}x|lWh9Q>CU`RlHxgUuSs_9}>ov=Jn!m2eHn|ZP#Ka7bat$|&K}KB}F=?t%`;%R6 zESb^iSQ`|K_N`e0wP*h)J}e}56IH0PSC`=#%}Z$qj=p08Ri4mthCm{DZNr&vfS%T~ zldUC_zl|9}s0NKm(evv?e>atoPn?f$5-&7RA_YlpXrv#Q{ahmC(&Mx0c_jwOe zIV5>37YXwdN1-un_#l&hDF%+YYaogbrA;Q#Y!`lhzDyEv5IzO&;XO0|ULVibbB`SI zYh0K?9|G{@N%9vmqnqEI9aqGahj6{X|JpC3=M2_k#;`%ti$o^?W>(?Ph+hLK|M3`i zqWv@XB#bvLN9=Q0#^0R{8snpIT`Ttws8!+{8X;7CooRn2~_+ zNNky|x(&R_!AV_%4F*a!YR1U%vb7qY+`R9`J{7 zXx!zeG%=0=2X6nEqKCo32C!$^2VVkU$w{$8_@GSvG@BwgrE3;AcafB`x|N8A_nYj& z^b?Yh#%vVKlme6)*){g!Gf71W-fdrDDZS7TDnbHdgUL+o+z=%f^V7Uc3fh@x)f<)$ zT~cY}6WJ7(^M}Kq`6KLk1}*Ta22eDe465eUD{N?!?H}a7cFj#8Z2or~CipDK(mGMx zwP5}ez!Y@OQ>OR{sN7nNH)IHPz^32OMmQMq9SE10L6y@AZN4AvB;C35dTDXMk6c-t zD^pKcRaZ{nb|PHNpb636w<@|Am9C&5A$bc@ox4&)Psgmo&9n&J3{MP$i=o}mqXS{< zMky88_lSDM>F}KXf!M*uut@7NzMzn|!qCjV2yFq7 zud0s7CY=sv5CMf^Iv5ZdI=ZzKeGOd&Smb*d4SOLsJxZ0_e6YL%AvsGDH@=VT8}9{R60+jpXk^d$LRkEqBRhH z1^}*AEFucJj`s~W#M^0P01ON@qnDTqVUe#`vhd;lDA!dmQ#z_jXNfH$3i-B$z;__R zCPZafCR(NcRilOiyP-aK@YgqD0LPT-UAYfk!MFh70nFZ!je zGCcNmE|doUW03lw7gRMVE->dKLI(;gqxl+vhlt{<k@Pt~RRJOa8q75NlfIksIuP4zMa-$1 ztm_^~P#A~V#xEM|lW}u)OF%OE68CK`#}3L~u_Ku4FC1Y&zq#Xe-MF$D0_C6@kE(y9NqovZ(EVAJJ_ zQ6fbxR^Za{_ou+|552!Eaa`gMGRyn)CUve`ng8svpnkJ8`$IPe*rZYOVC!At5|Ie~ z^~0+?e>J&qdG<$0+2|TjJKgpJAnax6$kWY{C*cIPWc(?~zL)y{3Y@+4GFRYI!sRxO z%M;kwu2To0^;? z=i_~!{>^)%^u)gaV9NjE?Y-lgdfxrPV8cr2s0b(`ReF`G(gXwq=?GD(bdX-7qS8S? zK)QhRCN0z?AWaBGdhaa|AV7f7LiXgN-|xM@yT9Fiy>{>2`Gb(0Gjq<&GxN-Qo*LHl zQXsiI&1Wq?0kNB%tC2cyUGV2_@$@l!58(ZQ&ib#aK_vZ;L}u_*Qh#$+b5g)% z5`n({GrWpcHh&)JDC#-s(%u@zU|lixA5{Dbyxw%c@y36e6cbL!R7IdZ59yVptEW6B zRD%B6*>KDPv&9^c!gq)MGoS3m{v|6w54QjlRGnl$&Im>>#r}Dr;TC+Lg1F$R&tI22 zpjZDzOz?j<(*Hxt>;HF*Oehzq&zlx0t_ki!)@TyHOv%W!Qk zF;0%S#d5gyyw?(|RaYVFrjtNwv(lL~Ggve2mXu|KHRXae0|r!p03He)3myb=;wB>m zcCeaVO160v3&Ck-4S*+ITQ2bf!OZ=76KAB3`!TdEB%<$;#JFnCvC>>Inb#9Bzx7QF zQd@1!U8Q>Z2^T=OKj^e6bU2HsMo7TQe$!U{ieb2KZ%oJ6X?FOy%2QqD!)ApIYID9IRMQ-<$xOUxL%>bSVVn5XUGl8K{wMmVh+`f{ViiZg)&O#5;{e`9sMjw*Zb!^ti6T_9 zoV4p`KbKevY_}mfR)J$Tbmcgx!*nA@o(8LDEXkm%$`vQ%A~ea*ephg6s+0^c zMOj}|v8W7*@vDAK_H8`GQ&oRcq3_2Q$i6Hgw@i>-wLDctxOb+`7W%cV?K(|vGojPn zSYB$XJNDOC&YorC372>#kqx#e`fRxP+W`q*j-Kg|3jA!-$O62yJuAJ52E^QIN4%3m zce>@feKwt(@8yw;bY;--OjuCB)q&pL-qUFCd@=C?$eAqtq$U3B&sZ;P)SNrEmEh~; z`7!rx`iWoz*7aa4{U=5?viZhNds>^_CfVC|*&uS{WsoD1h2;e0wNmJ^uOWT?F8pj1 zlnf+`Ket&2c9OXQvPM!k)zL)Bd8=E81_x6d!=Xf}ZQ|(@eD4Sm&cQ6_)+m0mS{g7h zf;<5hjtqW&ekGtfy#v16^-)cN*j0M!;kCR@U^&h|$VD!JyYOiJP2!NgUwb_yI;9DF z;&@Sli0c;aojj(S*>0<6ImppRY{W4Tr*eB`_Rs(+-;$a?b%hXjJWt{(z%LS-NYG|# z3s?kwyqtrIahf?&{pvUie$VD@i#m>V2~B*iw_yL7i`|t^*a!M=57v*p_N3Mf>eNum zdc2{Tq|+QP)!g?aZs5fwG6=&)&!Z}*?|J-fSR7z?Q3s zMr4=bKQ+T$M40Z84@7q-X$h;Gf7vco?=HfXQ+B!AOJedtaKRMk7hkD3=7d;-pF_S` zsqpffxp^Kz=Ia`3`SgfsmI$Q<8^N1&MdzLkbY~m9Ub?+t>z=*@M-D}t16A_vnpn) z2Jqu6WFYQo9$U~>>-T=|=+zQ<>Z@BUUp9Wd2%_^T&B&FF0{dSm^`^vo%8^@KGWF$f zCX+i(_Ci70}^G9 zJ9&EJR*=9M=bK~3+dVuGV?dH&dc7J|KOncWgJP*3Dnz`25_#~wfMX7F`LrpBh><%O zBc8&rlRlHH^_yg-9{cGCd|avbCD3e_+NS@mw7~IuD81LE9)fH|C21$u{{X-k<7gdR zN&)4W#|1;+fbGiqew5SM=SG<@icX`9t~V9KmxqlWp80Gfm+&YAKSl|v_U{kNSulLp=wG>YSpJ6O_Zj4CC^Ixvs zMm8`2Vb#;f5Oa_G4!Xnq&)}!XKwdXBK~A-vWrVAn^)yi_n5rF-w2Pdw@1gw@`)#gX zYgf}~YoOah)1`zuU59nOC<47H&K*RV*7zJ!)!BdMAZsR+DBiS?-gGQ`d7G~R0|Y8* zn>kW%Aj;rE$32Nxr+vWzZvmjEARd}a?AR%p*Ca}aIF!Z#ZxCuRm3s(~|$Pqg%NQ~*b zUMH!Y!9eKH`fyV2^1 z<_y|Ghnp=6lNUi3VKxcf=PugJceRXhPDLmyR@CRxg~HA+Us<|<{n#KF3*&F~#}^Hi zIJ%H;%0;vzL2hz#b@28<1?3B}dx~rwPN)eXZ1J&piSxOR%Gj7i(sSZuQCv`m9AgeLc zKH`ix?-sAP;f8brz2~L4%(#*->6S^^WaR0!bOv;>#}^B)P!}ERR(>uxIS~TNsN7XI zCp+*d*MHn36;$$Uq*@I0S@k2#_6+5?nbX>G(4eo4P>KjM=<&elx2g|Mf^0Y`+N#KB zleY!B_t*Hu=2CDIf%UIao1~Z#Te z_FH>?0NmKYKQGSQ6eJrK#0+{HWE!E&+?Lxz))<@Ks!R{y?tjW_(jn|MPpZ`$$)~Kb z%rzEJIen0hEVK2i6veid=KNpnNME_wZ8gU5!nidtVRvIs(K$E|MS zfPgh>Unu-J&An!>PvpMd7T}%KehgdKl!V$&h-foBgEL5rcI8-XVus~jfOIE^E&e4N zDz)XhdCXMj`bN~%+cAXgk`j;D!{MoI)2kqtv{hRU6;F(v40L2f{sw}`e9{e`Jw--g z52dGx4qoW4Wiplq7VPou&6(CM{jCzuSziA0J+oLpMrJ>1=o)qb45jtt@FJcCZ4Hks zlo83H^SQ%_OesI+8Wvzf-u~KXM?b-GB7CrQ_5hfLU?PhbhJVhTYnph}x?kEVcSLG9 zS@zxWwDb%RO81{)e61&44%#nB*{2Jr;eSqfLYB1-R(NQeUsETvK7mXGppA|JFgO67 zkDYPrcljely0Vg=nNSw%S#^O1#5dOU`mS9vk^LTtaXOh$JRzdJ&Zk{zE`?Yv})k zhTYKu!3?G`R~l+-)1p^yYq3E<#!-&gvR2bHB><_zgl&GBRTSDe{<=C3^F|K0(5JCV_KhjZgS$#NALAoLyYyPb1>{#Irzrd z6G)Jpo)R4KSOoSLMqm=7lPZ?Hv}^Yq5Q)sR4x<4-MOU{uN5uZb|F`4zN{k9~VGcRf zGk(e_Jl=cvUbe=*2etlFj5!RiGazIkRUZGfJoHfcItm+YxiUc4@o~Ci z8U;}fbH562*y|1#yFUWVW!ORn#|251q9e;u5f*lAfE-lUnBfPNXX#mEU0pkW%Ew$Z zxgF9q5tw9wA1#US1!hrZa!PQ;)rS>`Du<%Tt{rAn%Q#)Jyx*Y5klX-fPN<{c z?p+S`utBk&?<0~%d*%vY}8&TKm&pa9`q6pUVEOhIm55Jd5Y5L0821%)68rTmd%Hky zGNG#fs;fWe_nfx)+0NxUi8b3gTO)%wtGm2`g{bnp#k_>qd+I}8?@;V2)ymZ4<*yZD zuB}8QxTm!<_O9tFuDT^fe7TA!4$b@wm~uGlYCT9@zI)Hlm|l@9XtjGMk&Ft6eF}(Y z1Hp28S=|YO$$*>3Z2wDvuiaL;7rwp%{5&d_kmSNI0LP84I$k&&zCixf+S2NnBf77z&M{I3&W=Q`Sz@52aTk9 z-IoQWsvKSfjVA&rHL*U6xv&U3vEJ-*c*f%mg_i@`nTFLE6xQ#AyIL?jOfe67+g%!+nyomK!=5Wyb~xfQG;D;cat zKSE7qLPe6XUFtDHi==Uwf9zLD;^qed)-07|8}Eiz1%wTBv>tOZwF0&rlu4^Ci3*jv zJTqTEXeNj?G8@E3Xh*!n6gU=L!JLmQ;R5VfKaaC1^QqI-Wl4V*Q%QO~hv{f(Eo23p zb3#9d58FNVRs62nZX?NmadbSZkmF<2^S=r+;Ch*9I#c;qW)NL zb(Pb^P88&$6#*pZUJG$0)4qK%c8OglcJCh3uoa(b{}WiN#fEK362;?I6@52eH)>2y zIfc)(|LejFTHn`DeEemi)H7mwt#`9p8n#`AY9zD(yN8E$v9=F^Y}|EG;18j!CJ8-f z`^9*lim|-;(z5ZrEF|DvXDk5R*k^F4Q*-3ueURH2Uze*b?&;W7kd^S&VZ4VF@BvJ) zs_Kcs)QOGr0yBuQH$d)zW0dr&{il+pL&HEzAXs)LliNEt-*?_H$22?RbrY5mR_mZ{ z7v49K!Y`+PTAdWSbkj3=baxr2?XhAtCW2XT9zT*;?$@vO_HyqydI46?)1&1CMO5-g z_RMJprkv`~9wvxO#sxltE*}fSg8e+Yy3Q&$WDZ*ez}Bk8%`w7q*jFBlUHr>1ah!46 zPq94858~v0F0Yx+3r(cc zM6Yk^Nj2NE>cD(&qf*O86WbiEnR)pNn16TGq`x9A7DlH_;Ya%@egDGZB&G*Ts}8X z>A)P&w15`kJ|pyLRHkp`jK!hzm-g44nG`PZVCBSpU3i&LF`(lsQr+>B=*;@96GR;i z*j;eMlIn4WJvR9w=4O2ab74o}<~Rp33X*v5!ql%gd=ZG=SI2-^oV~trGi`)_Dtlcx z%z_Avw@S>pFzf%hoX14B&E8P;8z1InIq>E^oWdFTw*14TR!}~-ls1(5v~O8K!$nwu z{1^Mx`5zn663Ad$&=-fZ+Y%Yi>}2uv{|l_>JB1Rb^QNI~LsHO6F2)&F})t z?y@yi@5^YKcC$9Ug`4;H9b0A!eNA99P(_DGyR4bk@a_v3oG^=@+-d7%k97ciW~p_w z|J2etz@w#$Usl*)SE_M5TqY8yHroz>||&d^n_JPbTgU|h{|>ygJ$wRYz$zvl|z$;;aNwNsa2yDtY8V<;xAFk_BjGGtga8q!f6Y_k} z+r@XY)yKqIPOu@bYvWB0ih5X)u*@AIiS;aIb>GHr#N{Ttry9^o@|9Pjk|y`op6uRD z*QfO4bxNLSYMU&vvNw0^OP&be-8u=qdLhii3hn_L6w1;U_RUK@Jp$q8dKMrSR%Nfr zQ0vQr3r5`rP61Y!1*5_|lLjm2npF7(qb_cGeJDKj&=SHpGGEvABfw<&#tct{f*7qN4xa^qk$Cq7KhWFRvLj~`M>za)QxHtzwQuql{9C+PoF>gqa7vm{ z6<78}zH<++XJK;owCrdBV{&)aKaQANL35Jo4*uyOxQ z6~Ve!-?9$f`die^#?-&oh{`=1IY5Q0N=v1cs*5<)|vG5stp6I z=Yr@!^2Cx+NyT2ROjT!wlL8`TTo(x!wH;|2f@Hy1piuSEQ`IVCtAMlw=&3NWVLov0 zhB17+)MJ^YhDXEnztsoCi(>#TutxAtW?J{WN6j8H(8fxy9`QN>!%RtKa)ZQ0{&;|Z z^wo3RX}U3LEB|K~kgxWWK4aa1k$FCo@hLRy{klU(qXt*}Ib&CoGhbWAgR35jhi*6^ zN7wHAPe@9LKgf$uR z?PH>An0hsRrWi)~_#CqCaTEE)G8vyc*>9irw?rkgiz)p&O}r@U>}|xFS^U5E0`YwE zKjsCpEFJ>v_IF5wbRJQ>_Okz!`czfL0GZa~Hhi9rKk`iv9!YTLx$_+_`%-K3#+eW3 zk@hoFJG#b%(RH=8yWvTJ_BLQ8A&$@NDN5(Q zD^HJBwm-COOH<(w>kxbi8GVS53e0$Uk@Kg=Ga4`JE5e_67_CaW&G}%02`-cxpMu_8 z8kX!bl1Nl!zZSppvSYa@yeddc(0^PzN^>A;Ty95pN4DUocw%Uepr9P&DBE+m)L}bR z3?XdtXU9AmVV8)(-FV4r&6KbJxTFLO*6y)dH)wb7X|(N63->(eJ7533uFgZKBbUiC zurRb->`@P0h=!zB*%;3o+57dZ5QgkI6~w{sid5%2ckg69xk_Gj$#%uFyD46tX#?HNn=(&O^_FymA($M687tCKpj{-qc<~ zq`dL#PBzf~a69_2dP1ge*~CJz&(VViiH;Nl-5-S&8)P|05;W)?Xq)%#n4PYoBBtgq zfZm=*YhC%lB0yinhZ;Yk41T-?Vm! zKLhRBSYXyV`Ml$=AJ!GUS2cU|%D&Qi$F}al*G#!vA2pil1snlc+e+2r;cPk=Rzu&{ z(Th`#)@)JtLL{8ZE1QOWY(}Q;MN{fWyTyr-JW=v_qmu0Hez;Hh30`WFH;HTeNcmp4 zuKb0n2HRalB|DM<)v)g8zU+G;K-PIuDS4ikcYYN7^{JF$v7Jnj-^G^e3__hdeZsKs zF>%bhCNb77W}W3T9DF)4RxzI}9~70_i_x%up;jDY85Pmg{bKn@zd3*^_|lEsd@}{M z7!fX1`OIBDU23k}gi6+ztlE#jN5eD;+zxb_olgW8;+JexTJpI&AC=BOR#Hi-dt-1- zEiIQpcp+}VCH|8(*hf}?&oTzvZ|TvQbkF3L*LQ^hz!OC$gPaiRdS~HDQqJ6m;Mkf$ z#ZuOcd5IWU5{sC@!h6@|fT5crShk<>1hW@lt^H$#@8x?s+4($$kB}GNx!lRJYPH^6 z-c>R-m%q65Z79GXXXR|YZHtP5cY2a&?HPWQL54PsA>wD5<)D+zP z-?c|5XelTJQDpdlXOG?wlY^X7FHNI6QC>)E8-LOD2qU_4xjd*34FQ0rWcb`Wt1?nM zKuNfUlqP$N2^k<{yb{t7u3dYk7twCBwpWRI3_Gx>!h}4n&<*-?E9gf;#vWNCa|OQ< z%b#BwDSjV;W|ckykNbDw`J}-6?=Ov^ceJkj``Zt*CxidI|3AMJU8k=4_a)H(Q?+&1 z9P~9iBrd>spL~3yyk0!d2;skq+MDr<_jxdTNLDA%?-+4FiW7te-PKDqZUO*YJjis^cV4XJL?+KCwUr& z=s_^!!gaIr$90@nY5exh0=dGH?ipH(co{CKPi`EB^>>J1E;BduDQazLFuS~|`>m1V ztuNrznaSzR5vR|wdhPG%OFhn{e>w9qrJ~g@nahVX@2V}4)HaK2kl6_)_ig4 zzgi>j<(g8)HlXm~kwZhM5;-u4Kl*%fmLcc$<18Iy0AH1UI=Wsd;nl8(Z0$pNk6R0F zN-zWqUw+i$&1z!f;Crfa!|vsa0tpq`q2@>FIpr_T%Lo*Q+Bn(r%$HExi+Djs_%zN%bNM8#Oes*ty4>XI2ni11 zjL}_{rmC*G6BohvK-Ek*&w;g6+gs@GfUKVjF#=t`f)KlddZS^1dSqxg#J$A^AsJdb z_9Sy_m~`YQ36GB7v*DPV{vtn>LibebdfKu{2j}P;gZQFYu4PQQn>1!I=6oTlKlVlN z&-jlNTEiXZu9R?xKCaMU<-cy~>ge#q>V3!)TL(W{tm&4myVmoRVr5nX6I(_DY0q=| zHseNTSR|ZO^%Hf%GGYrulVUXV1uvHXnV~J{ml82H@t(^^cLi=#W31uUekleyFWu=f zp9=cVDHJ~o4PK_e^+esh$4aezFn*poT;WSgOGt1{id7w4FtS5|(tG7>x|r1L;~$-p zMjroKfzEVuugpLK_u3Tmw0CAFyX1d9zHOEj^Mctjb(LdPac~gw=ANQ!ym%E|U)~Z| ztO{FE*}L}@8hHlgR)t3UWdU~fpS1>x<3c3jo38s(%~5FyR9SLUE3m~~%|Sej88B26 z3T=^Sn~I{)v*Vffs&0P#O-uQb8EX&Uq=`q%NZn;_W)_F4eeQ?@OZ~D>lxOTxd#-2A z%1}`yXbn1AybKK}9bwd=wa&LtxZ^d<*I*Uiy(f0z54GOfkI zRYLBWD~QBq(74 zE)xy@K&#E29WUKz7hM=uZe=4|QsZvqo0!Akx^~ZyxmMMc>V0Y98v|_)U437@zZ0xR z;txCMi1pyTSKktz zaQqzIh8r!%f=u7{RomNb!LAyP5t8USMB zVfLqKfpkhXV_M+1%g+e|Dc>Qlvwz!{g!OOL-okIz=tS#8n+3kw=uZ`H{V!Nf)h38? z@|#o4le<=Q$0_Z)u>&jEo=gd_xelfV`GiU z-q5M&+blf3I2}K}^ZJX!9;^iLw7TP!SJ+=e`Pf-WKVGdP#Nzv}*AD&{n}+Hx3)NM! z3I6Ti3ksdVGmVO`xYbPzuN^5~48L+Cd&8*fp0f2LOeO?iv^qBQF0AufNpb5b%gE9yq+wyWPaf`?k@>_x?OkvOn{GDq zJ~z8XO3Tk4eBQO;n%T)8kg5yaQXv;}&b)151{^f{k<1NK9A{=Q73Xdr-Y*3DT}aj%CS{rwU2ZSP+I1O#d<`0qkI|7sod4^Z;$ zJ;bf(GyPq#MW6uYb2aaPNBw9oDlSfy@^l>1`ct9ta*CAaW+NG!E6DNbLG`tNZkQ9# z`*Zbppw@K-R$Z;nbRP7AhQx&g$tiAKyi+83`Oggo?*?);S){$3f%Y0j-zo3{cn=2i z*qUv!PWSg1diVDsu4`k~=`vnhv*Y9A1!u|L+MNk8dJA7^e0SE)&JM79HgdoH=gzHL z58T%V$H#Bp96U4GNKp(#Z>+hco;x1Pp~?K<&EjlRAUT-1#S=)&0+hFmT|h5;f$p51 z98#VM*gRV^N9v=#sCyL&9lI`HW#<3G_nqZ!V=96um>##EeapxUAW2_?fztTk-)hW% zo(p=ZDc>#q_kVya|7}3@-%S>cX6ORAH^Ir*_xR)xr!aW(7f^|RJxfj@0xc~rE|#PR z)xb!+wUDIf-kf~UHu?^@$SghThx^lyw~R_S*BO8)8tJkT>Bk^PFA`By)GuuS=>YVZ zF8+Dk3m$+<63(-YKv=~a=Do7V>%cfg7oM+qH%w|KGHeS)QyVCEGA`r~UKDA7_AXkFlMa~26zVw;}J{#hIQ z94FEjozFJQIsLg0`1_VlK9m06-(I!?ZT~+FQo%ZKU^x zr|i+s{qz&0?Bm^0Un1%X5*JI{ZlSPF^*ZuJ;QE~Wu^p(f`q|PO{B=eIw^d$A`CKz)@0tD)fGd4XKk~>IZsH~RW4@xIMh+Dt-Pde3{d{4HWPCh;_C1x;ujFR3< zj~tM}L#L3wsF6vRH6-Cgig?uFgv4vkoY1+Qgvk-Yh*+%O@@@ipOJ-&#vzVohkY*K( zl-oN^C#?4py(L(q90-dhr44atY&szoFMC*`tVy`W;1 ziK02qEv-M?!K~P(%T|h!=B4V3W*BInR)rqw9ih^{&)KaWr1vN9-b)Es_ozGWr>TlM zp`wGlKw!3WSEtQ^=tk)|S)SBX{%cX_?(`gSc=YLd@h*l)BeV0$r~SEtg63-r5>h6N#nps;%0hXkZcC= zC_&a8QAIqt%W|?_>Nn>tw+{pIB4k0_7kkr>#G_8|R_zJ4_-%$z=ppZVnC8g_@{nL1 z+#`4T2$yw)MXq+^PdC9+CxB79$%OTX$5_$ikF5D?GA#0(Q?W!bW4-(MQdxtxckO&WZDX^UbU}^Q}CXljvg>hy|1U5FM$yW z`Wg_wXDj?G%dL=oqE}be-b|oXDXk5-)-8TnzP`sXtXn!lN7x%VE+g#MO3_!AygJ(L zwysx)&EAD2^e?1;&kaVq)E!&f{TZifRh)+1V$dVdkm*zJqhj!HI7?>hizAxeev@iyDw%EX zI;>qtaO9*4nBeL&5Jq}3J2%3^Ey>unbSA@si&IE$Q&Mk&xIj& zA+n4_rzlaV>HdJGOs#XHI3c#wmw+dGiu4tw|(9G3<&I`}&0;!r0 z-W`Y|7I>6>7q3(!@cS%&xQ&iiN9zoazdc7jPnY$d&?n6DL`~yD84#x*S7c9rg~Uds zAGF-?Ig~rBgYK6zG;I4uPT^8s+y`jhm=*-=@3QTHFXgEHUcp0)W8?FXnYAaZN)}-m zs0-O81p#K_Bfg)4{c5K-j*B2So--tu8##&5Xysku6?Y$k%FCix-mjc}?r_gF_R#EE zhls9_x~|N9Blz{lqMQe9KRDfMdr;k`iVESJU*`neJ~eA&(h915EN`<)p*^k)GYz&4 z2J-RLO=VCE@6Y%7bP?ZivX8WK4Fq2Kd;flzvTD);1kc>A7q?TPqvS5 z`}#&0mORyMV&&N!p-$tZ+xFG=_p4T=(z;Kc*3;1{^-1z>i>O^=4zxHlCg8;52pzP# z!GTUChNk(D!*Lou$EAJByuOB(dJ;I*q?+!|#r}%R`Pj1Tid_rF(On`rzd-k(5 z#GS53xV;#dV96CHs2w>f%@F)|^ToT|xUVD9nZ1Ii@gnerJ3g~_j-uOFN1(9*2ZD9S z%sx}8S>|b8;mi-zzztu6(bEak$G7|!C8+wgk(tU!>2VVK%rM?O9ET7>68wG~;=XzH zot#z~AKYQhTw!VWh!&~hu?1((;Pz%XdQa1l+l!ur10t?`6LPv^tz`Bnx+t2d!g?%j`@?HTWb!CJ9N)hmm8NHekASLla!#BTb{AARQT3?UB+FA;qMq(2{9z!R~%EZ%bp0w}ICu5khfQQ+F#7b1DP7*d zQ%sle?rA%hujrJgFjb#_X^wts_RP_(zLk85M_@$KrXMqtuOJ+k{v0;YY#F>G-rroB z?Td_KKBe+GcvOegI0|IR@SG7JKkftfr84eDG3Q_d?!s5~SToJ}vx7`k<+gFVqKvy> z*@vEp_-VIoxH}nWc1_}ERL+dg1W&RYK42-^UD~H_#NNRl1@$MQ0H>gU4VhfTF7#x0 zCiO+YWQMWhy}FIj($fia)C?108wnj_L2fAW`=8#)+~7Z|09?&ZrDU~9c*j|5OF)pc zX)!ADJBV8@Ed=>*yV=2?)Ek?@(FLoH4{YYR&vC5WGF*6KQQ-fU!^z0%KAKpl*|5Mza zf zgihYf!XtbdqWYMCWKMk*yR?00t0Ssea-oi0K)K+c8aiAApN%o+7 zwzJ^a(+hRJBQMhqcaIGGcx0+}eJ}MCZ2c)Pz)cRXE zENkU=hoO(;{EoX|u~8}ZNR_D6Z+Fu($b&WH(b6@znlAWja#jRG|mOt^32>?>z1gkeEm`95u*#p<${P3u{)V$iyb+Cot z1PYF?RgN#B8H-$sjpi-&sR4;>M;05lEw#pk-tU@p)Mkx)j)!|lGP*~ew2dZBpMm6X zq-97sCO??nl;1sI2x$u$U*JHG7Q5~0Nz$W3td)4F(!5em7b)SZjPdF&;nT-=cVYOx zWajVnA+kFGk_5?4Cw5k!8U8+VK{y^U++-bp)%hs11UP_{B$@LcP0g61Rn**Ol95YS zfEzQN;47|3ndzgL`cs_Kj3Qt}L~L`4LKxoi?+803)GsT2-Mr`X2zwTfPa>X#CYut? z`!LqPaF2LS;?j}kCu>rROAylOt?~37AfP%{a&~P1D=2vU5d7AoS27=QP1nIuZ-Jc* zhSrY^_7mj{SEW{w?x2C}2C-8Dz~?ND zF$tkqg(tb0sLmVjlC0`q*U?DW$={}qi7n>VRDez1ys6Q1$S$y)f)jS2jFC3CEQOG> z?&vx_3%Ng0)+zpFip_-I{x<7Q)ppBLt7SoHM)Pi)FZz{wl#IOFCuZJNO-FzSZg-K* z_=9Cmlz008E}ve2W|30>Lle;yosK#9xZRe7)q@rE21}0I1Hj&Tk<$P4TxtrcI_+EU zjDjcNJh||06YgDkh?rQN3tAnKQ*KYs(3DZa9^RUQf~ool{5kE+*+H6=R7}KMaHj?m zU?-J$(y0-7(MLGMTA||;yZ9tJ$S;d0Bse!2Q$9Iqt6jy~p;MNoz-?MOw5+F3MWyLW z002apw$?Dt23fssPIO69^bl65ZC9L*luaOpS3K2eHfwM9+jzK02%tYQ{H`;3 zqr71Tb&hIVLd|A01ZT=y9^r0S+OC!2IJm9kB(q>Cjv1m|NgTwe)M-_ep{sbe8Jj6Z6eY9_?Dk-DkHNhx zTfb9x_k++q*VazBWDb;kX1x`HR@;&3n^GFY1IU0lGCh)gx5pQ7g~Dt5%;V84{%^;@ z&`X*$Q$TFUv#T!z6)4PH;?`8D}{TTLFnjX^xaYeAR7;i?97bKiZ|0fhbe|l-arufT~cOyHfV=9l*!`_mlp< z5M)m(MB7BBx-7XhTH7VbDQ^G$gt2{DVlL@eQ4C%am|ox2reJCA;Vi4)YTpPLi+c;_U_G&o=?1yMFn%83dDw$4F^(T z)jG*j>NF0ty1I^*h#3=p8LH@2&ygwv79Cwt^pElhCJEM^*5PMGVDe^DTiDT_M9#Dx zol{Yvo$nGbP&a|Y{A^N%CG^|Wx)=oP`|%P=vxZ%Gb8%ehAIXZd`uSd5Kwt0O;c45j zSH*S4-fR*H9m_uRp|7e?`}8vR@-J@k+ILU^2|s@6corPKs;zC3I5h^{85uCH}uuPdJZ3kyWClj z(9IlM3b$01S&W$ZOYb`Cu>#e9-W&0b>(f13B}{5{8F)Oh2gX; zgP*jGrGU64O)nIG4zhn{%XI0Je;FRU?ws{y>RLq_{?@hyd?do@bNVg5<{;VKJFGsW zI23CSK`)&i0(=E>G3VAAWws&rgREKmY*gkqjhySz#Z_7fp8+%bVKHrO)8@D7YT;ok zYpGdD?9LXF~hwQUKNSbvN-;W_rEvhWb0R z1?BfeurMHj!f=%}8~RMVGBtpk?{`RMHKDkB&1Ioxh7~|SyhqdM%=_LP&$~tx4RI11 z7Sa+~C_p7B#3BU(QGOI&A$DhRK+8^j$EQt=jqge#VK5jR?*B3ZX5;G+@!EF>KQ5Lk z_X7r-vbWiAU_vTQtc(l~C&8~MmqbJY#%$yDQO< zL4s4_mAo!Ngew--DIchh>^Dw!R=3|!lwj!f^2H$T8+wnMW~J(_o9P8epNbkBqS3^I zHL25tW^ynjj(Eund)7%*o-!rY)3JS{81Ft^pD6w-#ns8rnU8Xz0iO6C|J`iqQykVp ziSdSeB99n(X1xszwy4)<9@sDuh9VfhI8MViSOdcMM$`8sh&EevFR?Al;y#U#%JaD! zRAc3kuMcyqEyp`(BMLb~E?BUb>wD!p``&`0WYx>Ceq9HX+k6&4 z8U-A4?@)G0IACQ`Zf;lb`_@v3THN;#3&hQT2aAB!a zHqAP?diSraieYwd1J;{S^6Iy{O*6G%j$iB;v8nJ9m!8PwQo9t_N%or&0xZPcI#@f6 zSaZ12w4>s5iWq}jL|ew{(Qk{S{iCOMo%ozwmXrh4+gPHcp^)(f{>g;eU)9wsdI16c zuZ!V38+44y2so;fW@C#^v|?vsz}uKdtbQl!2F&xt;r2L8FQCI+)UJG@QxlQtVR%^8 z8kc@p?aaj^C%4npInX6TSnMDkcX=G-LhyuyZOs63+!TEDJPbk~Z*Py@(-3>fI)UDP z1MeLUDl?u5tqTp#42Ib%qo!}21vN^OB(CTAyz-uIv@9-QUWgs4CoY$<)Og9nDm2k% zTdE*RB)p$rRpV6}tA8005v!pukeO8i`A|7FhsmrI!nmEk`;VE0c-*2}KD?{G_GQyr z=G9k*<-CK3My2#Clr&{Jozs4jS4 zby@|`eoqf}i!AOs=h=yl+zSu3|4+sSMlF*sw1&Iyxw;z;r_m38E)+>c?8rJ#e<`_~ z9sC->>#VDHr?dVrZ&6QQD5|2!UEKI<_{3htv@o-J9k+qWN6zO@Are~k($p*M(bQCz zxcoNGTw>Qrc_thYY^1VEd8RC9=k7C%@})LIfu4bTyBmOHkw289msd|&8vsVC#=JUI zkv*A~Lmy+;o+K5s+UqlvsN;MFBo!ndE`)E`6BgDR>;@LB+tUa2nZL46E^})q^h}fh z+)oZsm*~Qt*zRKH^k$RojctV7sc!^YT7B39#usOVi*%id?J~DJQCW5=->IN$8ZE&?3eocd|;(^J9}c?5gKdS#@qh~YwsP< z)YdKfviDX|5wRf(Vxf1GDt)UI>C#DPB25TYAQ-9*>97FMgq5r|=U8)%@*88unLQ0~Pg?|Jr#A6h3Ldp8TB~rM z24tq)X3$i9U|b5%oR1}f{X7d4UfdH_u|_!97_9r*2kf^w9nHAV^!?A>W=t<+Jp3nlO_QQWPGTyQT!e`~O-vK0J|IawIPN>TWjpe;s5rlkzZu;1cOE(A=GSfPt z;(xg%T=j}5^t7ih)!~M$LUkl-Me`*Xr8YXHOQ%@@dJ&#;Pwk2(y7`h!B{euX|290jXhA0=c*A?OCf`BJW2qH3oewk@0LkaWSUd zwI-5PvRNs_ay=ktb5o+Mr<9k$Nv`?g^+BKXe*bBa&%wUe}n@HPRk4(?Qmuy|tl<$HIk zOG@VNur``va_)vC0#T3fAEXsU;8iDMlss0fl*{)>L)UlKG6;e<_Hwpz5?)vH-prC5 z3YB7it$zLQY-qEaVcuhH2?q!XdFFY5Wxd>4x0KTKqQ{&^gxWQqI#gnFD6pDnucq#b zt1l0R`1lhB-?Av8L7+df7+$$}V!d@m6Lpxl)pF>jiwj%GNbt|?* zht(y!kq15}q1G^$GDT6FT})g1z{@#evyCK^uztmQ6?bOo!IsaQ%}#AuL5W&(`;g;7 ztmMbB@W&D9Sp6b%FAII}WD3Ax2+s&5r& z?k1k`sn*BSt9s-GN7<;jQ)NY+md_%D~6mhq3oel+X>$4l^ufX17A1a!23aZfETh{5-n@ z)T6!mDil{XWx`w};dNtU{X*i;xW=3nMqfa#p<{Fm(ws^MNLp36i;_)Uul1`8^{&hz z+Wk{Q#~)XV%I;U~KABX-U>qtYO^vv+xRgspzm=i{#zxBKD&K*@4ArW~v14JK$Go|b zg}XmOINJ)-y(%2C9;NxFoHaJuSmJDUo;FY>-h9>CeXH_O7a2W!!${Zel{rZJ#ht`O zO7>mF4uu2*Z`O{My2)Y@02fQ+*x0bYESZ#BI}@L(BT3t zu~DT8?Z%|0{tl_k$|Ya01;PiljFhp9KB+=@)X4bERdWM)_4z1vRhX7My_uK(d~o&7 z$A=XQ%Lbl+^QzrVmNS@$4I0ANxea+SG+@k7EfX&`CSAB=Wy14<2J~UWH2fyg9lcl+ zy&hZ!_yL|5Q1w_CmB(7I7*zI6jg61AKhx&YBJQW2;pR%1uvT$(h0ZYdn2arMqKENc z{2?8=7O5{b41Hy?-DO3?`BUjZ`v6(39Mg@et?68=8s7izN1egN2!7}rIPrUOw{I|rr?cgo$xLO+ZT>vCI|nGK5< zsciXzPQt&w`7-jnXLiWP`>tBZ{j=$B{A|Z>zX^44vqp*`!!NguJ&NL?K^3@RBAQ=e ziFts9r;T3`i+EQZnOjonb?1bKwi|5l!2k9Lt}5r$H(V*`gKFSb{~K9gl{~R00F$Ms z`tr@)<1V$tPX1bnkJ-Zw{IQ}xK}SP^|~E2^Zu-C5?cL(@Fo}YHATY5>t#3M zDx^O6_pZP85=;zC7OOBC7FnEW;O0$nul344tMtPxjp@2ZzZ5gSYTDO4F@M%GtS#}C zLkH!J>}+WqTP4Gv05-a6nT_lp4iNfWaAd)M0?ed^mAz#PWXITnHTZR8Jtf*&tD zKHb0C|33)J=SL+#gz&hbPI^?Lklx364OjT%pXMl46${y=vqUo$&D>8qXzzCEasjk> zMJxTeFLh6Si(;6*I;7>sS6?+d29e6mI0GpbQ@G#YoJvT>y{h8T^qU8f)2mk6!9s8I zTt~$Z=hfnCosXfehF~(~*^ZMBt;A+M#jKvI&g>t$IcUIY`>cB7m0JfR6-%s(kba&+ zN7z6`W`CaTc5#`qPmNPoCm1aDgVtU5Bb9AZ{KR@d{kE*c{v@$>W-|3~-Y&$xo_*@} z9}Hq0ivtBLHx;XWhyb?~(JF`CKmG7Dm%Jy#t%_6Cb;l2sjVDf}wCTVUE?VL{8WUcj zXsH4k1HfNrwf?T0k`H*OZxJQm@NqGZrh4eZ$>&zrNgu9$(=0~4WQKlx-kC)IQe2yA zSvXi&1kNr@Y#F%NJbj5x0&HDY*~F?V%f$LkQ^agb0g%AzhgHS3?eaMuf9e!U*)>aY zy)c<`=#Q}XzMX84RQ(#2ruB6$eJCZVjVG>J&;@_Zu)H}!fDA+>OGkR|UlJ%j_EcR1 zxTU94|7h}?RKT#>JzwsLp)v>4Wig3baz^_Cb!QjJPusK43Ux^~)($xsn5HYv+jl$+ zKsZ`8z8z3Acs^P4^wj1HKK2SQDXLno5un2@J=dsRdTe8DQna)4x}Yn(v|R5qL{8N6 zN;#U=)o8Ky1YdLs{7*)PR|`oMv28x{c-G{|%kfyTaP#gPqVUr|Brn-F>3Ps)$G`m*EBF3Ia=DL>##2d?%h08hONdh1nJ>hwmGk(W2eIAh@L7Y*3eUBFXuDGc{Q`Gi=a zlu26aV&h#NO3|?obywgGm=;%=+tUW3L1$))XYJkJCjVb$xjmm^UCve8y(o z0s35e6}%={lh?$_I*8XOE5+`#Q@`9C+{#Doza1?(JZEv|oc*>m`T$qQH{^dX{$UNO zX*`A(E&?R^9+00kovrIY0J_J^|@lQEU&N z1}=(|9-fY}`f$ze&btmp%BAue8`bw6yO?9bwt@MWZlmk*Mci0whhAEQxS6MEq_tj# z8SRVEPIH+thpF+(BUAL71xCicf%xa5TNkr80Q>nwd`SyyN}W*2g}I5e%ey!MAKvxR zb~7{9yhLXwD~YxIYq7;DAS2OK2iU`3hB&5oqG71HLsg>f3LKNi$y}EF4=X;>AwSj)duUY*Vcob z&%^Isy}I@8!pEn{uwmZQ-bEn$Uz2ivuPpo1sk>@rNR?jQ5Ex&g>WCHzY<8t!;7_fK zF>66whF>fcl3Td>xQBQS0cXOu#2oJHI@Iuox7p36ehN@#*Xd`{?mbYAxZ%KJ_bvJ} zdDKLaUH`>A{b1G}^2kt=>ylPNdK&gGkMv}N0dEu4&$tGb_l+)YYBzd_t2ix=tNJRt zf`iWqxwDOODu~B-UD$tG-R_c;jW5$~N}N3lZhHUSNhsr*o$aL(@PRYVSM8w|+IK<( zBZ=AciSRW9q8Aev4F5@WyF!UU0e-%d-riHPNH3k!!3ZZS5wU7NGVRCY{=^s$Fy0Ky zkB)QJWUu-U>h;5P_=lG2i1XeFhf^vWA7QjR!y$y&=Y$s9ms`L3;>`~O%if^tV> z^V5_!E z5u|)OlP}XapyOUG^S~;U-=+p90J*v_>MB|as6wFv9m!H1guhGK6b$a-vhUY_VSN_e z6DTFUQy$phVD{BM!7tJ46|Uzh>EfS&m^J2zIN^#TgQp{y?vMiqb<&F#Gk^>#p*FiV z)c4XeHY_c-Mn5Y((s7vhKq~g}l7)XYPN_!!Fne9CgBJFe03u01l!+0D!`!w>X4PIB zyb30DR}&#Wm7f>^<@NbdBYK(|&2sBh_xwAR=xWQ>$E^aFni>Pe;P+PO8?Sr1!fwmO zqW0YD$T}0yLVs}d+#5;qjiPxGAnV}U?An^}*=~70jnJNM z^Gi$wAPw%R__UB_vhr|3wGPyEB^c%u6C1v^`8?R}gI~*08(@wnScM(S-3Ycf2Y6|m zo(z*yRh@-c8Ken_C90ZRjjf(RpA*8M@SW2Zhpkj_bC5RRZBR78&^v&b7EScJVkdRP4#+8cx~ga3C)fN@iNu zzG*WxJM1cgGQ6eNnJZHqohW0Nja zb;AP(u^uD@E7P%9urt{hlF z91aTrp&chiOzYRT>z1;mHraX_3C8oGUa~6ZJJ$lmHPH{_Io?@|FIF0npyLgG>%jph z_)ovRzlbrEv%aw79`AmWcm?L%jwV@8+3jztKac!kyo6cbd_)-QtcsVnz0&M8?%oC^ z^lrD?cSxeKh}P@wog|M-<41KuXiN8MK@;n|2|XM*aNNx)ffI0XQQu(%G!xwM3Djy^ zw+lsr6+GTnJ)e!;z15&km`d(1Rtj}M3FTFD!N)o_!>ba9XI@@jYIKh`mfH3SJ6k6r6#&5Gjq{&^y zSV<5tX@E_@7v%FN8cYD7W&{T8C3)}O=j!VFD}NvVihj`F;*H*5yYM`{9 z&xXQKhH{Yq(IFk!8)yea_Tyhd5ln1iBI4pU)t-&lntFm7P#VAyi}rjm+P|72P~z|S zXL-RdptMPXk0Jw9=@O>l{4mW-IKq1E_&Wv$1$OE6-=7%nI{urlf7%Injec+bKkMkx zNiRl*?~4{75YXFiau27D5Aw>&u_r@`UeC|R*UZWEsw4{NJtM1XPg@vl6kVzmu%)A? zw{=ARl`1hJy{ED*DlVQ0DnSh?{b9qgL0+|LO zyML`rXm8&9x%{Q<*{k%h-@<3MBLuMsT|ISP(+c|8#cT<$t>iLnYNj_;=78t~Z=7-3T$m)D$*4i13d z=yaM}a5zV?ulQ3o<0!`Cz$4Q zzgmB^B(Ahd&8(i=!EY@NZqs%*k7$1Nur0AMqCJ6mN3t23I^qf`~Ioy#hlwQq4hsp2YI-t~l^%%HOUeH9<*4I*HN zm#+gBegoWu7rASF??p0e=N4L|11+`Ktr6+u1RiY#B(#Cm6Y9Juu7^0xG-6r^^8*GlF53fgO3@uH+V9fqcNxy5E$BJ$E{#T4tit8p>4~H4i`{t+*y)oMZQ*UA zEmdwL=ne7bGLK4HJBGYSbT7i~;vW7vaQSK=_gTdrrX|VrPFHrMmgARCeWJ#`6l+4o zReS8p5Ggd*zBz8*e`3`C%BcrWQWxjV9GU-M7`Yhy^H%-k+Wiu`ia34&KQ;NHp|9S1 zCq_JAQsHpr^#Wv9JmW_9to41o&x0r@{lI}VrJd}lUDuyA!~^lSN-MhSM?XSoFNsV| zfQNtsP!z*2JQ^1F`?uYw}UHqy}OVr4W7miJ1?rZyM__T!K{`vZaEvFgxrs-;~h4F zz#x^=w_^I-dhi=w+?@W62M0{^Y)36-^`^ngvWJwG!=aD>a!<6PJI$Nr@c^R=JuB53 zt)C!;q3uqYe=x1HNl)H#o(@U#Pq_%(pQRHpX%o3g*Jq7II}#ieC{xBI(j~F@BvxNb z3ul9N!LuE!!!meOPPPD#MFJIGXNyys1 z{sHB!O5onq5tjD2r+#?#_(A{sSRK<;?;rj$#B71oX06&9?S4fGlU!%!clPrZOowzk zPU^^NCh%y2`|^amqe@>q5{F>>yR0A@)wK=4o2{Hq#gZ1UlLPtf44w22YJ+*dbCOiSrb26rzi)nkr-JH$z~Iu-gY18kA+=5BrTU6ee*kbJ3W|R zcbqzUEi@9$zztUugwe%246DfPUP_6xd2fO_A@RvC{xmDjXq0Nc28UViVf=f0dai=p zXSLBch=S{1TCqGukL8O@O7;^Dnk^yCOM?(^Zv)`P!qbre(jH5id!lM$ZM?){btTX8 zs9&ubxc5Lt088yn%Bbu|()K>{Pw{HvsPiqiz!Q7@$p?Lp#N&AgKfPH9<@MC&Z zG}evP*;Gqtz*IQit-bwmu(9_*cc9@lBL`~DDAc8NNRS76PsOgK!aE|~wZ%75N|d)q z3#Tb3<LMcn;?dUBz|DyWk=B?yqp3IqJO5Os7x36jiFUdUW|iLh)oE5wQrx zAq6r$oJ#It<_DfaoAEfa^9*=qVC2oLE@BllYrHJrz7~Y>XyxhR&g%3Mjx)g7Jxvhp zC~N=~=fUhD>Oav5=_UrXR84rg^N9pZY;cjncqT>$9UUJLtI|K+BYGz4S!#mHEKW#z z$2*a#%u*r1a&D>GYSv>J!fmUTiEQc#kz1BCUEO=c=O-DS5=RtPfGLyPB!dTJkOYr+ z7>(sP5x=nE2Z!0hP(iEfVIR4~IeRc-L9=>K0}LiGsJ?H;AZ((s^J_`Mk;s)VnHjxwK!<1bJL^ z<9S1AH7h3PcED)Y1u(%|q@wd>hS6;3D(dh2;aM+nX=~#$Ia3oJFL6DN&4#4nQeuZ_ zOI@^J6+YFWpUSFd48c_i4%}big?bD0fgLnWK`(sHW%C7o_&F{~2eE-8e06`9eC{s>l#1ix7mcHxjWVZ&O0>ki zdv~lp0yZuPG%A_Jiqz40EDQC{AE|=^%?_$0)X0)jPupHL(L}9SaK+yVe~`gQ>{d z`PEk$#F!$$-*Hc_hK>tdZJ)N`vuE^Yiy z|{Yj6=85b4NeCV?j0-LTS5An&`gv*jsbg@ik<4-SK4X=5PJb*l;;1%~AtGzC+7 zcZjmMCfzmplpN+kzN=b-sj1XwFOQIx_DDXy%Ir-7U6<~?AHd68<#Dq{rMr2RxRxs{ zycDN+AM#9;McCQ*#^zs(H2A4e+yj)GFRhsZ?izJo_eIX%Fw1Q`ol3T;N5O~A>^GfL zx4s@(8#0!^BYjPSLdn56zb85=tsTy}S?xAHg^iRQgHQI2pz24lc6akh#(-O1xKxNL zWH?($PJBv3Y^P@MOc4W3T?SL_7#aXK%WKLT>Ih#~ML%eN2S?pnPjxqDDb~B*IcV87 zOQtC1XShKp9StPXnr8*10n#-dnR5p3- z+(eXQ!3v4g^?Xxh%%)`MkqHQUd0Xd6$&eddvmkYT@_i50B0&(%RXp@p$x++kAjudB z$|qF&|292dkT;QsiZuRXz&_J2$0U^*DII zx?H0}o_*ab<$cUZkqfkflwmi=e}Ju)o*a}2m!qB2HsQqgFyd9nfG=|LO95t$QbUf& zQmCHbtVx60SRo9`3cLL59p*?*hX-2p%vaPAQ;-Q{D%pXT1hqHI!Lnq!DC7%tfe)cS zC%e2v5XB$H8|RA+>t%=$^EkR+o{5AcBqhC7hSP4l#KVmwW0I#Ga*2tdq3P=@$~!92 zB|{TWg%*&a5=F#I48lOx$kA)Yg<}~DA6W^>K-weW1f^Eqf+j#$J|jTq=1K5Nid8iz(-jH$z@~ zC{#JPVlzE_EH#G0P09%mW_qAE@Di!T%I}%*@{DG8BP+9z)?00vH=+r-o^QT$(O$*Gmp+iRCY4Fc$a0J*e-P1(!CrkB z&jV-MO&&0_#s3}s`Mn-iL_5Ej2ZW+|P#bL{d6471SYD9(6`0AmGX{)=V#{-chnCqG zpsDrrY$d=&%dun_=E5@6(_3W2SyNQ!3q-u_trnilYo{(fk~L;?x%=kS^}n6(ldDYg z3NIqi5%8=tm1uCi+-Wx@5I&2`cazo!)On`JW57SE!MAXxiTKa*hE4G-LgD+|abQw} zE(BlqEOprQ5d@EWMxW+4HP!fd4Q#=Tq)i+poVe7kUs~r_1x#Sw3O9Dg+RIAI4s?5VAvnYF2H}dpP0j*98Cq3x+NV-# z8!Dn)k#lYtbInYOqZzdM#j%#$Vi8frkS2?JiQU-t=D@D_QjF|Zc+qZN}iEIh@iO6@Cxi7Jv+Vo@iuZXaX7@ zTdxrbNKMPAP|kO=$d^(_EB5GC)UN%7?__dA6%2j1HH0~vF8JjIw=F;=&%0quvsj=p z=UnLWq7X{Km(J#GM3tMc>`*>v&&gd*y5}s3WuG6P0huX1Exwjk7Gq|wz_#xbV)B(R z@)8rJLhLy6CzbwpQ&rY!fB1~Q z8r9P`Yq+`oxz*B`eM!w&L=0)gc|iaiz3RGCIF*RyX^-q;bn6DA z-b=cq5BPn*yC=iMUzCd^=cWS^afBzeRbW?d|zfV6i@gtNVva8i7slE^*? zAzc+K)BT~wEG`hm)l*TOc;A3LgRnwytu(_qhKS(JBu&F=TQY($R3*|OF|@}i0a!0F z;geT1#l@(F))wjY?G<0nr_gUy%$=x9{RX{-mRyF1bMP2MPtzwe~BgcAE%Uy$&+9EdX;>TODkj(1=twrJ6}Ot=|~+qK^I zfK1?^l3Tt?L>`d4R*i)=Z%20J#87QkQE{tm)eQ*}Pb_yt3bsfz|0pwmwosv%F zTH)g>B!*pTaIpRWU&Ju-4mN*84|F$a^9Ev@pPGH!>&O?;CY`C;8hRW6s3%KokcD0r za#1JXDJUnYQZbH}9H0jk6AvsUA{RrqE7OSI{X>e{fwUjILLnF0uz<AfugAZAoZpH*M1V}66F88R8b19OWSE7K zPn8=Y*aKwbsOj!%GBnq-qa#8Im(7t<6MOy%M<+cx^SgM4Xt2Ar< z{X@N(FJ~SS*>8@Wp>#RgeY#ow;*GzGHY8Hy`lzp;Di2y|N_2onJm#LsMgCCgc8Jo` zND<*v%#n%rD-L?GvX&{-`;cu={`k)CqQOu3bAX5Jrn&oIp@U|U&p`V%P!hE|7^(cS zIf+u^-0z%hOSt+Db^}X7bBag1pm1K}x)4l^;KP>nH0$w|D+Wqy2jDIqXY7O|`iAT^ zMN-?DoV&4Ps5W5K)S2;d7vB_Z+1BZIW`nyDqu%5C026v{u22z~c6-g=s{lR**z5+z z%@dUzq|F~M90TW{m=Jxa+HC87D^vlwK(FqrqU_nTzD-28S6Sgcd6kF`v`Q z4Yt$Et+j_yUwoU!9DtP?k6Xqj-&HoY7S6w36+CrzUTo&@xZ!uj4AB;l-sv_-FW7j! zXN3$W6d7%v$56=%lUqpFKkL ztN#EyUlMj17W3uJ@0__Wm^P*}d)`AQp;915rbS6ABq5A;X z1^^JZ&k>?(f2Wo=z=4Q+wR`n+{w$ELhh9dW`O56-#F*!7;)qyPENEpSTfNpnwgIpH z+CppZ(&EQ6)@d?JuSJV=a9+*|$%$V|*QBJgurSV!T#|5F;joXJQNGu>_VWCbY)pGX zDcYy)TN&C~9D5^x{4GUs#lh4Za_&^zfWOKLGk-IJ_1$Jlj*z~;m#eHxhKU2g+MUR* zOH2?$ptf}O<=cwN;7Zq)&9}WYGl(hdMuD2MNDK3U+od9t zoBYneGm5s7J-z>z`{hYK2kX|Dhb=zn+O>nmKTL+HR}OW$NC)YF7pglFKw;jwk7`r> z?Hy^gZ!7wvP z{ECu&q@iBLc77%5zxOO~rT#7#kRb>}zfBQB9lA(UY#!IjSLbPF)|bUv3ak2$`Zh3N3YPCq|8B$Jpd=!ML+V$k{P7g%bCdTOO#N%)B+zDHMrs> zelHkl;4Na5Pp}sM%$^m{_H%E+@ju(K;x9!Ltn)v~kQD%#%)M(`zId7xm7T>~_{)+N zC6d^2?Ul)bu2M?c`8s6l1iW;;wXj8m$&@ABrKZu7?rH+sa(M>Ie{4u5Dhk*5WDl<@ zYNnnqX?Kw})O=xC(5}VZl0Xi7?W+0W;Q4(wS9>{nU?qeXW>A9?HF6B~Vw8jadZIn0 zSMT0qprj3P851$OAfPVf$9oCV0&N&N*p8mK5j!O=hfyLfW|{y!yfghtWPmvc5NmFu zt9j(j5@}J{4fUXmIzZ0%=oXEx672!&N^vTF)*sbYq0J-Lk^K*4zuP66 zO=awgrs4euo}Zhd`#$Bo6X7qk!r_37Kn}!du)lau|iM z7AEmJHx0C;-S)(L;nQnSWLO<`%d_~R69R2`5er`3TdcBb6B7nen3-Hj!}*M9ZN~7= zVgBqRRSsHqVw~10`jGwU+i>7j-W}BjoxQUrE?J1hKc|}-8RNUp|1JD&%#<%pZ5$tTb7XBDS7pPz27Hk%7 zFb69_FIOgDbWN>WeX&Z9#*_o(N;5T?moOJ~y<|Vr!A)h6wE~eME4l8-n^u(`uLNh@ zEo=Jr(ghQ9J)4>(wWGmF4hR>ecsC*L4)%)#!31l|F#|%CdkL3Gr@Vru7NO|zmxoY|(e@H8Fa+mf4???H%#Ar3nG7YV z7CDeiO;DL2lV-HK;$Lo25*=EyrDZC%oxk-#lw)GoZBd0Ef82KKbTqX93vS3d0o@`y zS1sNRRhrri>{;@ukQh@oa*eD`{rJiHmKH-Y%)m6Ccr&*$=AJ`AdKg0~aQtyVifr zh}Fw*1@ z%y~QdGcZl(^b&sbNhk6UpqAHxcU9p$DZy0*&~n_d(bSg@z9+8^3z_nlhH75ICJsr8 z7pSAa zdbw&K>j5RE_a3Z_lkKf80V)OxCR{&d@I0YlyS!J}df^1ztR*T8H5LJ7( z`D=O>^a35#NZD*k;)NhHMvhgg*EH8ocQ!9PR$g@+0RFtcthq>E<|ja1MM#^B6!D_Y zgh6N^5WUToFzI*Zf22KCxUXkk#tV@h12n5|dk4RGvRA_P|20JDrSX{wQA!d6Y*VuU z8Nrzm)p*&JUrAQcL?A_^AU|JyMLy*4U@or0Pkbn#p@Io09b<`Y&#f3MZn&apDu-Gx z?dsQ3bH8zO@i-HM0U!v~06g!BK>#ltlx7d)jiN88>rY469)EfF=D*dZ|0v2#1c0jl z9XkBK0<4Zta*Q69uUi6y+xVU1{~4NJWir0DI^No(qoV^bdvOZCYAZKf13uG6A4xEd zqyu%Ce?fx>r&y17LFxYoo)-J(Z&c70Kn3lhum1~F1WNKO&(6+P=KwK3po$;#V?;y* zz#mFFbw?}wN>FN$_MPuU0Gog>ycqxdO(iM!N3)i{$$-BJsHcpS?Gm`EIPCbQ7+!J4r2PK$e@(I-AFlZ~A-RDpEd_e} zf#K1wO2Q3d6;3@Fhf~L(W$ej@A+=*HQp~ctytdXAsBoYZz{k(Of3*6m*zz(vJ#!zp zDF9IHI7TRCf1#My7!)1om&Q2Ut8K@B>GgnO3-s)BOaoM5p(nHc;E(Nip6$HE-={p z+q7H%8r_ll5EZp!u{>r(fy-_AUFG}fIpA~Av66oba#oQlQjCC7#2&k+wkC28h-2U6Cn>g7YQyGIA zlb;k+pdcsawBA$~|B0nc1&);c$~r&TYh>4EfC^Um;ZcuM#y>ZQtfYhPPHt@=o#<<; z%udLdkX@PR-BSd5&|be?vfq4OHs{umJ_RNFquXzZ!ckwRvyjnSuaxhdT9q+4uCR&g zh@FlE4c=1DO-zRU%rjk_eUu!qE$+M_^asOFk>7~pHT!j2LRyE#boE~~xRw3fy1QC77PyM2^|5!MI7J8A%@&`4+p_O{%1&-Q16y3G+R& z?V32eqNnmAYz6!ekLXf-*yh*D*c%^mGwH9~s~?XiUR zE$Lz-e`}bOKdHO$R#*wy=3ivOB<}wjHZEz~h2^R5TVdbUYVqU^+7|ZTtsfW_CB+@y z+UtjuE$b8?A#i?=wf%dXe7E%%)gbsJpQ!>Lnl-&3r1WhN0c)9uVb4f(x3jm>WGOVxiV*mE#y`8sv2MN}z?k5NO zbM|Iid~w9Y!tSM!}!oHbF3Va%X)?Ngp@yp zDc{{2+8JWpY`FEK%^SCPmA(;B={fb@WUnvL{}WQ<=0Hu5d|+xYu855`KxT27XTq(! z`OiL>@K3GVV4c{c_|xw=Ke!cGpjY>hr8bO5oZ)*4u!z(vVL(4!Cp_Dzt7Ic#Qc``M zdFK^KYX;O=SO$(%Dok}19};61@Wb%)Gg|(A$i4Rc%>o)Rv(89XoAxs5aD%=#uuYMi zX3h997}6I*A6(w{U21reWV$!(rL;rO^0-W(loTI50^~H?#=l7UfF!@AQs#LV5YBl_ z{ilFK#7f;GqyEMwoGp1$aIue!qq+zEJfAL)(x<x@m3=eo zYV!`td=!5eP=p`x6_O1FmNPAz2WKAO+h3rq)O&Q})A56Zwv4#E7+seu@H{0a@9fA|=@0l#R%wM&s8mYB)3?k+AaR5RUq5ZDFJ6>{t9-UfgV@=46-E}|g`y;}&9cGPiF#?R?( zAMIL#5$}@+#})&tZ5=QBT~b);GIbJeBEBkD%~+(dl{RUEfAw{I(vCAr*imw zzp1#mHu+rza*6swJSN0OJRaS}N+ zre^oON1nHBj?p#Y4Kl)7_6G{1d>7;dtxNo0)qQ7JQ%khA=Xey4f*cE?BBBB!QUxJW z0tb;2s(^F|BF)gHw*(OdDT0LFrFW2)Kp;UuK%`0NC4lr0NGJ&<5b|x#x%c_*{c(TY zKi@Zh_B?sA_spI>Yt6f6)_Rw1f%9lpZ%?mkB`OoX{@g|eF_x0)wr4CvIJkjbkqH&s zr3`;P8KBqVN)+>+aU*^0^j>X7ET~a)f{RCQe#|$BC1RO6%wGqxPQ;OPnx+(kR>C$9 zUjosbl8z`%r|YYoa5PT9i=MwL$m)Q(`ZMMfWAKSDq0s7p{JbrhsU_BENC~RuBOI9x zGi#5ALEZ;54XvYWCmJNqa}VUG)K8dUAd<8TEH@Azp#CPZ#k^KOZ3OBl=eg68@nsUaV70~K^H**XL+RRfS9pgaEDOv|o-aZA<+x7#2fYM3To zZDJT)c2^(lVKF0DRB}m{U&T$&{fSyK^?M$x%#`EhWB?HrJ+{`VIIaAB5#R@n?d9 zq~kQFCh~mqV@Y4DW)cRR^?Viyt@hL|yXY z(iwnB2lfRCL|JfbYUPc4&AAe)g$-e1*)aa?ESG|ArNO$I3M^YGRKPpWw7>pN)$n>^ zy=sfx8-BxA0xrH7NmQf`IL%3Ivs|nzKNC7c+__3v-oIccN;Xpiz_CfLWQIM;iP?G zybricYdP&hs!V9dRlmE`0hMi-mugxV$SZakYv!!@Y%$Q)NJEVd49BDbcplL;~d;)~8W!2;8FmAa5IfY8zJjdUQ z4zAV1xKr?Wghe13>*{U`0jVi2Jc-C}{fl0Br%zKZ2BAl)>qHe)9} zE1Bdr|8Ea0|FAoR@954+v3ZBL4=rV6hv_HG6U>bR5=Bwci zd_{QvJD26mw41DQ;EDnERiIMf2~1zF<|usK`L+0bVrX7$s_Z9DR+T2=@FV%wb1|$j z7M`xwD~1Bxp9|v69)TJ@Dwj#(=!*e+)|n=AJfZnDR zNwr85{b}yj@$X zy~TtT{=21Yx{K0HrfoooLlVzBs8E;CK5t$b()GnC#=9T6@Q0+g;tI7U)zpEyW@rbv zF!Fb8UC+#a;&VVEhd7-cuY(r+f`0Gl}P9ly4A?9m$Y^GOt+3eV5X zGKMnRez1_8G?ItR&ocwSkV0u^)=%Z^;kj#uIG%8l1!XG5Nop?b=T6P*ngotFs z0}&uYsBktlU~2g}n+3cOZ|%B751=v8zRZDF3A@-%MQN1Otn79CbM45e(|r~4e>@$1 z+EZ8pmwq}UT;yYN-p91h+ldh6lrVe7OyGWdO2g~MA(iDHH-heXu0vBBk_&u(oE{Jd zD?AEZtFuh_o$I%*F*J+nc@f?wnP~O5ok?GJV}~g`NAK!MJe1K*9z3~cTVAs1G%1gQ z;AszEvjeT;Z&NTYh9-Yv~R5BYrtBH!4Czo&Lf`s@UIRM9NV?yt+Ta-!;0nB z>iQ42tgp4J<$Yn-a`@a-?qYuS_uy}~);Y8gulsT8d>11yNv*T5^Y);(y04vy)}EAR z+`8fB(@vu3?y>XDm0(!1zArROPQ!Njp0Loym;=c-2PyJ}Gn19UyvftCOjVlWM1%Kzy{Hc9`WV zuXbe&$&e6~7;V7xXFKuqPCshpi&wi7b%_+PPJ2M_Iyw+t_cIi88!*SO$q1a!uLR&l z!Kg3s3rm1^)JtkebHN9TyPj{k$q`vY-SX z;!7(Nk_yC3Gt{tqrO;{oC~o%qZ!TlGS;RE&Cvv#aU5DMKyphdH>82lOVq)+pb^N;G z0Jj}x$+B!39ldZ|p`M7~z4fCSvSHiivdWXrbIu z`q6Hgtb}%Lq2}O2!i~U1BsFU%nl_i!(QMp1xQNe+-l`&v3(zv75#Aa8ooPDQ9~69G zThzPGW*eK4**=`n>Oc%-wH_Qm9^MqVU+GisJFmU{upaI9cIhs-sff8UXIVPXoEm`E zmV+ANw8d~`soNcnt>T(!ixTt^l&N3|8r(f`;;AFBZTcujy}fwrQS5brm7%nLTdcSV zm+#0GwBz8v~qFKzQ>TJ{zPO`+TI8(kZ4XGxGVs`i9)Y{1s z`{f8{rysJLFbA!wk;4zrCRMRR-`D`|CKN%#p2^tD|RB}BF->XR-a?`|5 z9we3ybXQK7Uc1(;upKSI;`O}Ll^~!(M?=?VK5-#2j3WAS88x&_wHHxsbjV(@Vu*ew zV`vUqt?7@IJ$=GG$#3~JAG``O1F+h#yYl$|-7AkhDBoi7+NTgnUGZ06p>CFshD94O z)d$5!E!i*{0i^c<0jE9DC#zQF+TXjBie^{1jgU%BF(iAqL+l8ys0mkC{VpDOLRQ1wW73k+;|4LZ(n;nBRV`E3}&@{<_!wad#t4=dRZ1X*Gh2t$nLRQ0- zlofn;ILBoxS+{bzo%#o=4XfuFn+o7%$2KTIO4qIaL+0{MKd7TgFN#`ma-vx?n7Fz{ zqCF24Cewb`0qRU1<+l@`sAn91ix%8CSz-da z{zr&bcaoi)Uf;RPVt3EglPZX>a(#X%8^L9NeJ$i=I^tz#Y;v<|9IISb zd6WPNUqQ6(;9jXWzsR#vp-vQoEj@$KgAcqLn5kst(u|z+d5laY^&Ey_2&PXWsZA|a z+8|<1(a`KFOH|k?vXIdVDBEy%U8Dc!5f8CTlQtxUq*koc{qx1JB9QRd)wVo3-VCt2 z^~0(Cajsa6L?ogld1jOiokYg}cA{vhYppo^=HO4Jt8SCU3?^`F8y#ehrUqj%iep&s z@21B5_11p*2g5YF<7OENa1Or45838=*pkCc4{^cS!`hmUSPp#aC*=Fi^q+?ZrCnjZ zyTWeS#$2^(dpdePOUhjY^Q;>O_Q$@4K1sS9XkZ1yo8NvpdVMVBLIB;94 zQL)MJCV`~r!D%z}$!c6yWz|u8@pz}%zM=0*qnv7tX}!AGdEw_hz=;t;F+9b)Hmlix zPuZroDL{RNDbP=y`9P|-p1l}`8|-Mbu_?F4Tv3$X`!OA``<>UAooUy$;+vPhM~Gw( zHfC?qtfkUAskC23Js4%N^DQ3+i=0ZmF2UJPoUr>FpiQ$8s*_$;hA5$i*=)KRZp$S` z`lkv^{jI;?kY0mF{xuR`qh2ubj@}Oi6-I-Sl*rV(m>hB633}n5e8EwQQNvli~TypX?58Owdf$N?G7{+2HJ(@@FFh*D*bTk9A1ESvyT2Ww-vYkfnubUO{ z9OCXPRB;*+`!v1x;8OBs4m)KBCtV1YqY71ad1bDE(?_2nIKU+~tEYI^15TrR0J$vP zZ{I0CC4N^fRYKA`A|>AW{6$^)I24~LF)8WU5HeCmr&-h{-#o2zZ!B0oHFDqSQWxDh z_zYJ=YDTI??25-6Kt!(FVYPm8X*HxtbkG2TO)~5~ ziJ7ZPcu)uX^?A;8{w&x#K5M2jSoYz8cmcOa}_Q;94 zvjYSj>8#z*Ly3-D_pPPveCzv%XylP0fj}@QiosS>fDD?thzvmkAcIAQB|!W1?{UU! z3Eux(1>y|dV+8+$h2+b4B*00s9|^>O{#))zzWM*Jda2LX+|2XqMrZ5eLK%_@f5t2M zOHIhyM2a+esm-J6U#bskx<2{TiFnKFZ#V7bYoJUm~&_5 zoN%Y~-IoRY_Oe}dE(t3yfaghZ%64J$Jqzul4>L$teiyLJUI_#*Hi@+5I!Go}YMK-s zwW=U)CyVVV-D|6Pv1uE$R}z=(;+P5ouprRyl~wsbu#2OE{Ou{k==P-4arDj{ARvYU(dEjt4F9 z4GMOjVPvFxZ+jCPU;-r4CFRbp()$nA^^MbX& zHh1UqdaN?h4~h z-svKjxl##8`NZB5zLzV?lExXE=JO!A zHlFA=cKsVgcU*tL&T>XQpW>_a4k|srRF3=ja0{%MRa}N2|uerw<(O^TG%>tD@8M zG5^apubefzqTuq5oZY|iJ8Fz;GD`9zKMvmO~1tl=j6g{JGsUr1Bk?6&pTcq<;QHFyJ|$*@rHKvu`QP5dK;U&f<8U1_+LQvK|{ z7ePyNx(*;faq09?4TeI);!7#IC$&4PrygI`D700#7na5-|NZpdTc10wv{O#61hO}S z#tV7g`jqc|w(jJv{mA2K>NAtlj@0N+c9Dz8+-?@XBH7sov*TYSQ>-f2O$s^RBCp+a zQ0*2pLZ>Ccp2S)9(1a)TMb9io-wTq;e34kB)h*HX^4m?DkYyF4=BgePGaw#ikZ~7! zqq|1C?VK{~39Za{gH6B%Qo}9xgX4|g+#{ZGN>eTlIoM#Zp_37DUR+DXUibp&!}tEC zac5CO(3XNm&5L#q!@+715YOA|Xokwd@>Wi!`VN?9!2_3BT)U8ci0lA4yXP^7uluX_ zq368Yvi>}`uNQkfIxG8<2B{}bT-G?w5rB3cuNh!>C1C2QLch5q&p3vhduuJAPm8#> z!T7Svcr2YPY12h5riyM%1>HWw3yFN3dQvb27Q->8qIcDQd?|+~7~T5xtg_9p?GiRM?T~Z+Mp)Z}ghv`qiwD#hGth>Ky;wX^$gyM36clWmAGN zVS=h{@4S2kR(wz@wf<4)0qY0cSV*Fut29r$K?zR!FYDXO=@F8bKgQe$Dpw8ykyLdY zpM-~gdip+NVZWx}ICGbKhYJDY{lrL9<6Oq#@moh)kH^@_J>XEJTfHA|Qvi{h#w{-E z4s$qN$2xCAF272v_9A@$@icfrY2Nm=%{%8X183I1e1oiZiuojkZELeqHAct7B8?yH zjLjFsEM5$Ne!BJ2rH@VC)(@vq&6^HOjZ?h;NI~XUWJ6N5*#9W8zHKai33k9L>9 zood(a#FxZzdmYf#GeYc+RmXBgj;V;Ci2Dw(_KW_JW9+;n!#7eRY=C$qk3DyIt7eX& zcIs=fE+zKGRr{-9arAC*o_g4WIt3$S>wwD%dXm&wz%ACi5c|Czcd!IIl5O82_zwBF%(AZHmCx5XnPmaopM=v6-+jBFQH%-&XpoP~o1(`a{q zMSf6hA&S@E->pX}|GU`bNQWzDai@3N0YTdGfG$Kk-zs}?hwF4s{=It(&EKD0%{q+E zPU7$^|MYR27jpKjzu0w8P?g@{8%~+(6gyV&^%NWWDX|3WH2am^dXJ5B2rEC=aTJmB ztkR4Y;*n2T*uB+2k=&3f+b2rX-shCA3Z9h*Enm|qZ}<_yNw)n%2L1fo#Rrm+r&&NY zL^>UV__I?Sn5{1~xGsB~EaSm`dpUNo^QMLVWxrEZDS_dqKs+)L4FwlE)g#~BdMM&? zjN4}UJ_3Vt7H~m57Xqv{`tK}mCwGPG@wJJI$R%m)u2 zq%kDC{E4Nk{rU9{A-2mT)ez%*W7T8tS4VG+rU_E9SCpJIb!y^if}YJ^mBq5p&BxeW zi=&*<`^(nsbXDNzUe&9rve^69$5M2nPd`2&Am9h~^ogQRXf(Q}re>-v>{ zS0NUPGcH}}M6bhVT&xNDA3Z}&wh{tw9pCtf{~uJY!@XUSrxu%a_{gB}DV8O}65s@! z6}|-EFrJCcbKnBD?bIc$>h8yvfwjwe+`;#c>wY=bnFP&$7(`~*0AD}%7@2+{$50E+ z4oLv~aOC|4TWM@;rf@b1f4dYa7k8|Z8(st6ml;)n9`fw*zxl7TYz(l3J+iv+pG0?m zrw-%+eq8^QAJL!ww^e72Mo!>a?)E{dUx?Rdwx~?rKr894<+i`2^gOjwW=|7K!$H>`Nw4R|wlETT9b7}c_zSVN0o>^QDG%~;b6@;O z(C2(}0~|oz_|?XP*!xv&)2THmytxMS)8Q{AfW73=a>f4Fy438{tz{-nD`tOB@&1|& zMjB#t?uPEwu)#v@gQ;Z&WAeL|*{`8kEY=nvUzwd!IrfcqcMaU^jk>FgTj?pRLjsiI z0g8<)Jkv`&nqL|e=qq_ZC$iZDEAqQVo1C{6x)O2#P36g{7nep_v~G+Q0=R`CTV{|)~k4`4Cw14?LYo$c&GOHmQ;53$GEBk zHZ1o>EdO&0U<6Bwi|_5;KW+Hz8DI}41s*$77DZ%(0l(13b;ZED@H9eOjLi}&_u=r{ zq1E4)!*vz*^QleY+?@%61`0=e^Z%Gbvk(8vy!fB1uWMwyR(|RR(+7dO9>Ifm9D;ZH zipJb0&ISJm0uJrrc`0zt6UP}<7*NKwqMsCV@ zF7dgVqg)*vN|x3q7+!6e5l8-$8q$<6hb}^Mp5&kaH<*2tk91FCzRS#PNa zGKG3!X&y114~&d}&lMLu;EF1G14m&|a_^NI4O)`S7X0ePGHpbcVKgr=H~yTw&Xq-m z)*(oAvs$NROh-pYkz{{W2Kd)DnK0!()3?eps@U?WW2~j9Wb;>8;VPx7G;_oM;GNv4 z%i70MO(`qiv5LmB8m=qGH1P8j`-nnljcpe|o@rcZF+MQxY6iZmon=9nc<=6DgQ>j=t0@Whxpm|9YW)~_vBIh5E@_7KQAwi36D{akFS~C8 zq>*5izdO%&WgD7(VF#UW(wPNBD48JSS$ zO+z-;4+Ev%Gr4@4+Ecops2BXfd8oSLm8pb+gO2>G$;iSuJA&n;F&~c`W#UzT#H<@)S<4 z)beRfYT>%Glb!4lUZagPO+QNLYE;?%X$w)b*{|R_}FI|i-y~~9bEg%#>Qxn^m&T9Uj0I9 z(G_Du(P^qGd0=0^bCAtxqRd#4354Y>NDH%qK+BkGCv`KwLFZW9>TaP!MfFk8WDj9gDXBwhi~?06iut&c%4tS#$|WwwDQ zcb|!xHQJ(XmpGEKQBJ$-K_xa*ZE`-iA;TuG;i>zOjy3*n&Z~hFI~l$z^^De?B6PiR zpVd<#7`b$<#o0_|G{Lsd@*&!hg42OG&NK?71@CT|DN8I@njvgvQZrq|7!CHs(tShZ zGpXQ3$1D3H7ziQ!o2Zqoz@U?=nB)D35kL@W!^X)q#i(*wUuo%blG)SN z=4Z&IN;<*^=CGXQ5(}EPah||<%>~XoQkDt{Qe|0U-fjJjWm&!lEtOCl!tfv&M1BcS zg?r`fi6J(l+=Ax0I5;?t#iAe?YF6W~7BOKIR9!7*B6YP%U^KWbGxh(Rmglc=(+ucKnoHJurdRuXpRTsV>0~LUB=#6>%k%5b`gYWQK$c<)u*AF`oedFcnm%8&AWn3&Qm6hvA(f86V0QqX`Kl&J% zCSmsj(3_*<9D zJ@4vUT`It3_2tyNW3sbPtfggTPL7VpE$q$&w@f*uJ`F?@1{-RZ;wng*sdzF0HHYdz zg_J^+ks2FBT3@CM_knMfCjZ5VCwB%m^nE*%>>}*a1#o{Vjp6(WOJB9CTKnJx!(smW zETR1ovt8|g=V9%ZbV`2ytHVCYsI1g?$e8}H!bo%VB54HY*cxmbV zrXBV}%ZbtImOYi>wei&{(B5=R>vm&2;8b*v<9J*3>CJ4$nuAUC7zIMfEft*; zR1+ux|0^`0Mq8Jm&ThogqF;BAVJT-W2U2LpVue_ncE}cJ|JVn< zO6zZR&9Olw_TUx!s4OdRRgFE-U|hRf;7ED9aDaf@{1uM&^?GM*e1Hl^dku*~A8n2b zK^)!Om`yDQP3EqR1pv>dH5tn;I)nBuQsP8>s3gA*=$J5M)ArnUG@`y1-XN#~Vf zB-U0|5-y|M0)vW5hX}*`B|eA>A2)!34TvV9exaN!Fk**!RPjB5{-&8t&B=ak?ttZ;Ha<08zBUX|iSk zvtvErOxvm;mtLgmtWOZ?>}||Dag}WxQuXJO6dEuMsn7nbuZjQ}y1)^3u(Wg^va{K) z*JSLjxAC*lKC2)~X!mkV3wFf+-s;QA2LH~!otVD1((=+&OF6eqd_k0B?~wx*PC_R*2%EP%4W{E!}rFXJjeZU8uHp zZ-Z`f8xDS8-12V<<1H;sZSkUof>1NQ{Y#aNvGyZk zkD3E5=jUCxb4(?fWJ3_S*`af4ZM(ay;4^%Cp38TQydrDi-~J)W2t0wVy$awy;y;`F z%c%H29d##*&N+r`Psv;vLO5DZCh_}<9{w8P=Ntk~B~Sd_^&gpgI)-qf*Q|ED0`HYL z?G8@vj`THsF0BVtrr^I&@Jq^4rCCeOYA3tEvITg#uds4Iw$cjP3jgg`Xzl++{s5*H z%=01AD>S>Y0K#%t0#e!l*yOnUYhGkbDu5VcU#)b$e`i6IbR*Ws-mN@ixJaihS>=p6+AkvKHEt@chdR9i(vyb_4Z;*s-N1FV2W=zBhHdA|){q5Kz=O996* zMN>{d>w9%0n8BEv>02zLtC*ReJ=cm9`$DW%dznvn1UeNAHaAyKsr60$9 Date: Thu, 22 Aug 2024 14:53:49 +0300 Subject: [PATCH 14/22] Remove the undocumented login-with-token page (#8336) There are several problems with this feature: 1. To use it, you have to put the user's token in the URL. This token lasts forever (unless the user explicitly logs out), so it is nearly as sensitive as the user's password. Embedding such sensitive information in the URL is problematic, because URLs are saved in the browser history, dumped to server logs and displayed on the screen, none of which are secure locations. A user could also accidentally share a URL with an embedded token. 2. If an attacker can get a user to follow a malicious link, they could forcibly log that user into the attacker's account (AKA "login CSRF"). This by itself is just a nuisance, but the attacker could potentially use this to trick the victim into, for example, uploading confidential data to the attacker's account. 3. By design, it requires the use of token authentication, whose drawbacks I have explained in #8289. In fairness, when originally implemented, this feature set the session cookie rather than the token, but this cannot work if the user is already logged in, as the `sessionid` cookie is marked `HTTPOnly` and cannot be overridden by JavaScript. So the only way for this feature to work in all circumstances is to set the token. Generally, the use cases of this feature are better served by single sign-on protocols, which don't suffer from these drawbacks. --- ...240822_134319_roman_rm_login_with_token.md | 4 +++ cvat-ui/package.json | 2 +- cvat-ui/src/components/cvat-app.tsx | 11 -------- .../login-with-token/login-with-token.tsx | 25 ------------------- .../actions_users/issue_1810_login_logout.js | 18 ------------- 5 files changed, 5 insertions(+), 55 deletions(-) create mode 100644 changelog.d/20240822_134319_roman_rm_login_with_token.md delete mode 100644 cvat-ui/src/components/login-with-token/login-with-token.tsx diff --git a/changelog.d/20240822_134319_roman_rm_login_with_token.md b/changelog.d/20240822_134319_roman_rm_login_with_token.md new file mode 100644 index 00000000000..45d1631c893 --- /dev/null +++ b/changelog.d/20240822_134319_roman_rm_login_with_token.md @@ -0,0 +1,4 @@ +### Removed + +- Removed the `/auth/login-with-token` page + () diff --git a/cvat-ui/package.json b/cvat-ui/package.json index 89baa2e1dfa..4f46788fb2a 100644 --- a/cvat-ui/package.json +++ b/cvat-ui/package.json @@ -1,6 +1,6 @@ { "name": "cvat-ui", - "version": "1.64.6", + "version": "1.65.0", "description": "CVAT single-page application", "main": "src/index.tsx", "scripts": { diff --git a/cvat-ui/src/components/cvat-app.tsx b/cvat-ui/src/components/cvat-app.tsx index c9ebea4d12e..1282ac21bca 100644 --- a/cvat-ui/src/components/cvat-app.tsx +++ b/cvat-ui/src/components/cvat-app.tsx @@ -17,7 +17,6 @@ import Text from 'antd/lib/typography/Text'; import LogoutComponent from 'components/logout-component'; import LoginPageContainer from 'containers/login-page/login-page'; -import LoginWithTokenComponent from 'components/login-with-token/login-with-token'; import RegisterPageContainer from 'containers/register-page/register-page'; import ResetPasswordPageConfirmComponent from 'components/reset-password-confirm-page/reset-password-confirm-page'; import ResetPasswordPageComponent from 'components/reset-password-page/reset-password-page'; @@ -501,11 +500,6 @@ class CVATApplication extends React.PureComponent - @@ -590,11 +584,6 @@ class CVATApplication extends React.PureComponent - {isPasswordResetEnabled && ( )} diff --git a/cvat-ui/src/components/login-with-token/login-with-token.tsx b/cvat-ui/src/components/login-with-token/login-with-token.tsx deleted file mode 100644 index bdfbdbf5244..00000000000 --- a/cvat-ui/src/components/login-with-token/login-with-token.tsx +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright (C) 2020-2022 Intel Corporation -// Copyright (C) 2022-2023 CVAT.ai Corporation -// -// SPDX-License-Identifier: MIT - -import React, { useEffect } from 'react'; -import LoadingSpinner from 'components/common/loading-spinner'; -import { useParams, useLocation } from 'react-router'; - -export default function LoginWithTokenComponent(): JSX.Element { - const location = useLocation(); - const { token } = useParams<{ token: string }>(); - const search = new URLSearchParams(location.search); - - useEffect( - () => { - localStorage.setItem('token', token); - const next = search.get('next') ?? '/'; - (window as Window).location = next; - }, - [token], - ); - - return (); -} diff --git a/tests/cypress/e2e/actions_users/issue_1810_login_logout.js b/tests/cypress/e2e/actions_users/issue_1810_login_logout.js index 391c04b1a1f..8f7b7e36867 100644 --- a/tests/cypress/e2e/actions_users/issue_1810_login_logout.js +++ b/tests/cypress/e2e/actions_users/issue_1810_login_logout.js @@ -58,24 +58,6 @@ context('When clicking on the Logout button, get the user session closed.', () = cy.contains('.cvat-task-details-task-name', `${taskName}`).should('be.visible'); }); - it('Logout and login to task via token', () => { - cy.logout(); - // get token and login to task - cy.request({ - method: 'POST', - url: '/api/auth/login', - body: { - username: Cypress.env('user'), - email: Cypress.env('email'), - password: Cypress.env('password'), - }, - }).then(async (response) => { - const token = response.body.key; - cy.visit(`/auth/login-with-token/${token}?next=/tasks/${taskId}`); - cy.contains('.cvat-task-details-task-name', `${taskName}`).should('be.visible'); - }); - }); - it('Login via email', () => { cy.logout(); login(Cypress.env('email'), Cypress.env('password')); From c5ff8bcf6a3e057107e57e7ba43258d3d77ff830 Mon Sep 17 00:00:00 2001 From: Roman Donchenko Date: Thu, 22 Aug 2024 16:56:57 +0300 Subject: [PATCH 15/22] Stop using token authentication in the UI (#8289) Currently, the UI authenticates with the server using two parallel methods: * A cookie set by the `/api/auth/login` endpoint. * A token returned by the same endpoint. This is redundant and confusing, and also causes several usability & security issues: * If a user creates 2 or more concurrent sessions (e.g. logs in on two computers), and then logs out of one of them, it will effectively log them out of all other sessions too. This happens because: 1. The same token is shared between all sessions. 2. Logging out destroys the token in the DB. 3. The server tries to authenticate the browser using the token first, so if a browser presents a token that's no longer present in the DB, the server responds with a 401 (even if the cookie is still valid). * When a user changes their password, Django invalidates all of that user's other sessions... except that doesn't work, because the user's token remains valid. This is bad, because if an attacker steals a user's password and logs in, the most obvious recourse (changing the password) will not work - the attacker will stay logged in. * Sessions effectively last forever, because, while Django's session data expires after `SESSION_COOKIE_AGE`, the token never expires. * The token is stored in local storage, so it could be stolen in an XSS attack. The session cookie is not susceptible to this, because it's marked `HttpOnly`. The common theme in all these problems is the token, so by ceasing to use it we can fix them all. Note that this patch does not remove the server-side token creation & authentication logic, or remove the token from the output of the `/api/auth/login` endpoint. This is because that would break the `Client.login` method in the SDK. However, I believe that in the future we should get rid of the whole "generate token on login" logic, and let users create API tokens explicitly if (and only if) they wish to use the SDK. --- .../20240815_143525_roman_no_token_in_ui_2.md | 15 ++++++++ cvat-core/src/api-implementation.ts | 4 --- cvat-core/src/api.ts | 4 --- cvat-core/src/index.ts | 1 - cvat-core/src/server-proxy.ts | 31 +++-------------- cvat/apps/iam/authentication.py | 17 ---------- cvat/settings/base.py | 2 +- .../pr_5331_missing_authentication_data.js | 34 ------------------- 8 files changed, 20 insertions(+), 88 deletions(-) create mode 100644 changelog.d/20240815_143525_roman_no_token_in_ui_2.md delete mode 100644 tests/cypress/e2e/issues_prs/pr_5331_missing_authentication_data.js diff --git a/changelog.d/20240815_143525_roman_no_token_in_ui_2.md b/changelog.d/20240815_143525_roman_no_token_in_ui_2.md new file mode 100644 index 00000000000..2c54d4b4c9e --- /dev/null +++ b/changelog.d/20240815_143525_roman_no_token_in_ui_2.md @@ -0,0 +1,15 @@ +### Fixed + +- Logging out of one session will no longer log the user out of all their + other sessions + () + +### Changed + +- User sessions now expire after two weeks of inactivity + () + +- A user changing their password will now invalidate all of their sessions + except for the current one + () + diff --git a/cvat-core/src/api-implementation.ts b/cvat-core/src/api-implementation.ts index 1e7bfb5164c..d274027bdc1 100644 --- a/cvat-core/src/api-implementation.ts +++ b/cvat-core/src/api-implementation.ts @@ -137,10 +137,6 @@ export default function implementAPI(cvat: CVATCore): CVATCore { const result = await serverProxy.server.setAuthData(response); return result; }); - implementationMixin(cvat.server.removeAuthData, async () => { - const result = await serverProxy.server.removeAuthData(); - return result; - }); implementationMixin(cvat.server.installedApps, async () => { const result = await serverProxy.server.installedApps(); return result; diff --git a/cvat-core/src/api.ts b/cvat-core/src/api.ts index 00a65ac3a84..ba7191bbc05 100644 --- a/cvat-core/src/api.ts +++ b/cvat-core/src/api.ts @@ -122,10 +122,6 @@ function build(): CVATCore { const result = await PluginRegistry.apiWrapper(cvat.server.setAuthData, response); return result; }, - async removeAuthData() { - const result = await PluginRegistry.apiWrapper(cvat.server.removeAuthData); - return result; - }, async installedApps() { const result = await PluginRegistry.apiWrapper(cvat.server.installedApps); return result; diff --git a/cvat-core/src/index.ts b/cvat-core/src/index.ts index efd069e919d..89f5d98f4b9 100644 --- a/cvat-core/src/index.ts +++ b/cvat-core/src/index.ts @@ -71,7 +71,6 @@ export default interface CVATCore { healthCheck: any; request: any; setAuthData: any; - removeAuthData: any; installedApps: any; apiSchema: typeof serverProxy.server.apiSchema; }; diff --git a/cvat-core/src/server-proxy.ts b/cvat-core/src/server-proxy.ts index 1400b1e3db3..91dc52a7182 100644 --- a/cvat-core/src/server-proxy.ts +++ b/cvat-core/src/server-proxy.ts @@ -358,10 +358,10 @@ Axios.interceptors.response.use((response) => { return response; }); -let token = store.get('token'); -if (token) { - Axios.defaults.headers.common.Authorization = `Token ${token}`; -} +// Previously, we used to store an additional authentication token in local storage. +// Now we don't, and if the user still has one stored, we'll remove it to prevent +// unnecessary credential exposure. +store.remove('token'); function setAuthData(response: AxiosResponse): void { if (response.headers['set-cookie']) { @@ -370,18 +370,6 @@ function setAuthData(response: AxiosResponse): void { const cookies = response.headers['set-cookie'].join(';'); Axios.defaults.headers.common.Cookie = cookies; } - - if (response.data.key) { - token = response.data.key; - store.set('token', token); - Axios.defaults.headers.common.Authorization = `Token ${token}`; - } -} - -function removeAuthData(): void { - Axios.defaults.headers.common.Authorization = ''; - store.remove('token'); - token = null; } async function about(): Promise { @@ -474,7 +462,6 @@ async function register( } async function login(credential: string, password: string): Promise { - removeAuthData(); let authenticationResponse = null; try { authenticationResponse = await Axios.post(`${config.backendAPI}/auth/login`, { @@ -491,7 +478,6 @@ async function login(credential: string, password: string): Promise { async function logout(): Promise { try { await Axios.post(`${config.backendAPI}/auth/logout`); - removeAuthData(); } catch (errorData) { throw generateError(errorData); } @@ -570,17 +556,9 @@ async function getSelf(): Promise { async function authenticated(): Promise { try { - // In CVAT app we use two types of authentication - // At first we check if authentication token is present - // Request in getSelf will provide correct authentication cookies - if (!store.get('token')) { - removeAuthData(); - return false; - } await getSelf(); } catch (serverError) { if (serverError.code === 401) { - removeAuthData(); return false; } @@ -2345,7 +2323,6 @@ async function calculateAnalyticsReport( export default Object.freeze({ server: Object.freeze({ setAuthData, - removeAuthData, about, share, formats, diff --git a/cvat/apps/iam/authentication.py b/cvat/apps/iam/authentication.py index 6bffc8366b6..41280638038 100644 --- a/cvat/apps/iam/authentication.py +++ b/cvat/apps/iam/authentication.py @@ -5,8 +5,6 @@ from django.core import signing from rest_framework import exceptions from rest_framework.authentication import BaseAuthentication -from rest_framework.authentication import TokenAuthentication -from django.contrib.auth import login from django.contrib.auth import get_user_model from furl import furl import hashlib @@ -52,21 +50,6 @@ def unsign(self, signature, url): except User.DoesNotExist: raise signing.BadSignature() -# Even with token authentication it is very important to have a valid session id -# in cookies because in some cases we cannot use token authentication (e.g. when -# we redirect to the server in UI using just URL). To overkill that we override -# the class to call `login` method which restores the session id in cookies. -class TokenAuthenticationEx(TokenAuthentication): - def authenticate(self, request): - auth = super().authenticate(request) - # drf_spectacular uses mock requests without session field - session = getattr(request, 'session', None) - if (auth is not None and - session is not None and - (session.session_key is None or (not session.modified and not session.load()))): - login(request, auth[0], 'django.contrib.auth.backends.ModelBackend') - return auth - class SignatureAuthentication(BaseAuthentication): """ Authentication backend for signed URLs. diff --git a/cvat/settings/base.py b/cvat/settings/base.py index 106048d8601..1cd454564ff 100644 --- a/cvat/settings/base.py +++ b/cvat/settings/base.py @@ -134,7 +134,7 @@ def generate_secret_key(): 'cvat.apps.iam.permissions.PolicyEnforcer', ], 'DEFAULT_AUTHENTICATION_CLASSES': [ - 'cvat.apps.iam.authentication.TokenAuthenticationEx', + 'rest_framework.authentication.TokenAuthentication', 'cvat.apps.iam.authentication.SignatureAuthentication', 'rest_framework.authentication.SessionAuthentication', 'rest_framework.authentication.BasicAuthentication' diff --git a/tests/cypress/e2e/issues_prs/pr_5331_missing_authentication_data.js b/tests/cypress/e2e/issues_prs/pr_5331_missing_authentication_data.js deleted file mode 100644 index 6821234cba8..00000000000 --- a/tests/cypress/e2e/issues_prs/pr_5331_missing_authentication_data.js +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright (C) 2023 CVAT.ai Corporation -// -// SPDX-License-Identifier: MIT - -/// - -context('Check behavior in case of missing authentification data', () => { - const prId = '5331'; - - before(() => { - cy.visit('auth/login'); - }); - - describe(`Testing pr "${prId}"`, () => { - it('Auto logout if authentication token is missing', () => { - cy.login(); - cy.clearLocalStorage('token'); - cy.reload(); - cy.get('.cvat-login-form-wrapper').should('exist'); - }); - - it('Cookies are set correctly if only token is present', () => { - cy.login(); - cy.get('.cvat-tasks-page').should('exist'); - cy.clearCookies(); - cy.getCookies() - .should('have.length', 0) - .then(() => { - cy.reload(); - cy.get('.cvat-tasks-page').should('exist'); - }); - }); - }); -}); From 3f57f2ab884b2c21c850576e44a68a893049825f Mon Sep 17 00:00:00 2001 From: Roman Donchenko Date: Thu, 22 Aug 2024 19:22:06 +0300 Subject: [PATCH 16/22] CI: run Django unit tests in verbose mode (#8338) It's easier to read the logs this way, plus we already do that in the full/nightly workflows. --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 395d3fb5129..822934ee9ba 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -248,7 +248,7 @@ jobs: while [[ $(curl -s -o /dev/null -w "%{http_code}" localhost:8181/health?bundles) != "200" && max_tries -gt 0 ]]; do (( max_tries-- )); sleep 5; done docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash \ - -c 'coverage run -a manage.py test cvat/apps && coverage json && mv coverage.json ${CONTAINER_COVERAGE_DATA_DIR}/unit_tests_coverage.json' + -c 'coverage run -a manage.py test -v 2 cvat/apps && coverage json && mv coverage.json ${CONTAINER_COVERAGE_DATA_DIR}/unit_tests_coverage.json' docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash \ -c 'DISABLE_HUSKY=1 yarn --frozen-lockfile && yarn workspace cvat-core run test && mv cvat-core/reports/coverage/coverage-final.json ${CONTAINER_COVERAGE_DATA_DIR}' From d9a525b0902ea0cf5cd9de629ee7a90168a92580 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Fri, 23 Aug 2024 13:48:44 +0300 Subject: [PATCH 17/22] Job validations (public part) (#8321) ### Motivation and context - Added `max_validations_per_job`, `target_metric`, `target_metric_threshold` fields in quality settings - Added `assignee_last_updated` in quality reports - Added separate `accuracy`, `precision`, and `recall` fields in quality summary - Refactored some IAM code ### How has this been tested? ### Checklist - [ ] I submit my changes into the `develop` branch - [ ] I have created a changelog fragment - [ ] I have updated the documentation accordingly - [ ] I have added tests to cover my changes - [ ] I have linked related issues (see [GitHub docs]( https://help.github.com/en/github/managing-your-work-on-github/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword)) - [ ] I have increased versions of npm packages if it is necessary ([cvat-canvas](https://github.com/cvat-ai/cvat/tree/develop/cvat-canvas#versioning), [cvat-core](https://github.com/cvat-ai/cvat/tree/develop/cvat-core#versioning), [cvat-data](https://github.com/cvat-ai/cvat/tree/develop/cvat-data#versioning) and [cvat-ui](https://github.com/cvat-ai/cvat/tree/develop/cvat-ui#versioning)) ### License - [ ] I submit _my code changes_ under the same [MIT License]( https://github.com/cvat-ai/cvat/blob/develop/LICENSE) that covers the project. Feel free to contact the maintainers if that's a concern. ## Summary by CodeRabbit - **New Features** - Enhanced permission handling for jobs, tasks, and projects, streamlining the creation process. - Introduced new quality control metrics and settings, enabling more granular tracking and reporting. - **Bug Fixes** - Improved validation logic to enforce constraints on job validation attempts. - **Documentation** - Added descriptive help texts for new fields in quality report serializers, clarifying their purpose. - **Chores** - Updated method signatures to improve flexibility and maintainability across permission handling and reporting functionalities. --- ...20240819_210200_mzhiltso_validation_api.md | 4 + cvat/apps/analytics_report/permissions.py | 16 +-- cvat/apps/engine/permissions.py | 36 ++++-- ...tyreport_assignee_last_updated_and_more.py | 43 +++++++ cvat/apps/quality_control/models.py | 24 ++++ cvat/apps/quality_control/quality_reports.py | 4 + cvat/apps/quality_control/serializers.py | 18 ++- cvat/apps/webhooks/permissions.py | 2 +- cvat/schema.yml | 66 +++++++++++ tests/python/shared/assets/cvat_db/data.json | 112 ++++++++++++++---- .../python/shared/assets/quality_reports.json | 36 ++++++ .../shared/assets/quality_settings.json | 60 ++++++++++ 12 files changed, 374 insertions(+), 47 deletions(-) create mode 100644 changelog.d/20240819_210200_mzhiltso_validation_api.md create mode 100644 cvat/apps/quality_control/migrations/0003_qualityreport_assignee_last_updated_and_more.py diff --git a/changelog.d/20240819_210200_mzhiltso_validation_api.md b/changelog.d/20240819_210200_mzhiltso_validation_api.md new file mode 100644 index 00000000000..2031da1456a --- /dev/null +++ b/changelog.d/20240819_210200_mzhiltso_validation_api.md @@ -0,0 +1,4 @@ +### Added + +- Last assignee update date in quality reports, new options in quality settings + () diff --git a/cvat/apps/analytics_report/permissions.py b/cvat/apps/analytics_report/permissions.py index 9735fce22f1..e6177b1d4df 100644 --- a/cvat/apps/analytics_report/permissions.py +++ b/cvat/apps/analytics_report/permissions.py @@ -7,9 +7,9 @@ from django.core.exceptions import ObjectDoesNotExist from rest_framework.exceptions import ValidationError -from cvat.apps.engine.models import Job, Project, Task +from cvat.apps.engine.models import Job from cvat.apps.engine.permissions import JobPermission, ProjectPermission, TaskPermission -from cvat.apps.iam.permissions import OpenPolicyAgentPermission, StrEnum, get_iam_context +from cvat.apps.iam.permissions import OpenPolicyAgentPermission, StrEnum class AnalyticsReportPermission(OpenPolicyAgentPermission): @@ -34,9 +34,7 @@ def create(cls, request, view, obj, iam_context): project_id = request.query_params.get("project_id", None) if job_id: - job = Job.objects.get(id=job_id) - iam_context = get_iam_context(request, job) - perm = JobPermission.create_scope_view(iam_context, int(job_id)) + perm = JobPermission.create_scope_view(request, int(job_id)) permissions.append(perm) else: job_id = request.data.get("job_id", None) @@ -48,15 +46,11 @@ def create(cls, request, view, obj, iam_context): task_id = job.segment.task.id if task_id: - task = Task.objects.get(id=task_id) - iam_context = get_iam_context(request, task) - perm = TaskPermission.create_scope_view(request, int(task_id), iam_context) + perm = TaskPermission.create_scope_view(request, int(task_id)) permissions.append(perm) if project_id: - project = Project.objects.get(id=project_id) - iam_context = get_iam_context(request, project) - perm = ProjectPermission.create_scope_view(iam_context, int(project_id)) + perm = ProjectPermission.create_scope_view(request, int(project_id)) permissions.append(perm) except ObjectDoesNotExist as ex: raise ValidationError( diff --git a/cvat/apps/engine/permissions.py b/cvat/apps/engine/permissions.py index 414d6488263..efa7d3f7b99 100644 --- a/cvat/apps/engine/permissions.py +++ b/cvat/apps/engine/permissions.py @@ -305,12 +305,17 @@ def get_scopes(request, view, obj): return scopes @classmethod - def create_scope_view(cls, iam_context, project_id): - try: - obj = Project.objects.get(id=project_id) - except Project.DoesNotExist as ex: - raise ValidationError(str(ex)) - return cls(**iam_context, obj=obj, scope=__class__.Scopes.VIEW) + def create_scope_view(cls, request, project: Union[int, Project], iam_context=None): + if isinstance(project, int): + try: + project = Project.objects.get(id=project) + except Project.DoesNotExist as ex: + raise ValidationError(str(ex)) + + if not iam_context and request: + iam_context = get_iam_context(request, project) + + return cls(**iam_context, obj=project, scope=__class__.Scopes.VIEW) @classmethod def create_scope_create(cls, request, org_id): @@ -422,7 +427,7 @@ def create(cls, request, view, obj, iam_context): permissions.append(perm) if project_id: - perm = ProjectPermission.create_scope_view(iam_context, project_id) + perm = ProjectPermission.create_scope_view(request, int(project_id), iam_context) permissions.append(perm) for field_source, field in [ @@ -684,12 +689,17 @@ def create_scope_view_data(cls, iam_context, job_id): return cls(**iam_context, obj=obj, scope='view:data') @classmethod - def create_scope_view(cls, iam_context, job_id): - try: - obj = Job.objects.get(id=job_id) - except Job.DoesNotExist as ex: - raise ValidationError(str(ex)) - return cls(**iam_context, obj=obj, scope=__class__.Scopes.VIEW) + def create_scope_view(cls, request, job: Union[int, Job], iam_context=None): + if isinstance(job, int): + try: + job = Job.objects.get(id=job) + except Job.DoesNotExist as ex: + raise ValidationError(str(ex)) + + if not iam_context and request: + iam_context = get_iam_context(request, job) + + return cls(**iam_context, obj=job, scope=__class__.Scopes.VIEW) def __init__(self, **kwargs): self.task_id = kwargs.pop('task_id', None) diff --git a/cvat/apps/quality_control/migrations/0003_qualityreport_assignee_last_updated_and_more.py b/cvat/apps/quality_control/migrations/0003_qualityreport_assignee_last_updated_and_more.py new file mode 100644 index 00000000000..0b9baee4454 --- /dev/null +++ b/cvat/apps/quality_control/migrations/0003_qualityreport_assignee_last_updated_and_more.py @@ -0,0 +1,43 @@ +# Generated by Django 4.2.15 on 2024-08-21 13:56 + +from django.db import migrations, models + +import cvat.apps.quality_control.models + + +class Migration(migrations.Migration): + + dependencies = [ + ("quality_control", "0002_qualityreport_assignee"), + ] + + operations = [ + migrations.AddField( + model_name="qualityreport", + name="assignee_last_updated", + field=models.DateTimeField(null=True), + ), + migrations.AddField( + model_name="qualitysettings", + name="max_validations_per_job", + field=models.PositiveIntegerField(default=0), + ), + migrations.AddField( + model_name="qualitysettings", + name="target_metric", + field=models.CharField( + choices=[ + ("accuracy", "ACCURACY"), + ("precision", "PRECISION"), + ("recall", "RECALL"), + ], + default=cvat.apps.quality_control.models.QualityTargetMetricType["ACCURACY"], + max_length=32, + ), + ), + migrations.AddField( + model_name="qualitysettings", + name="target_metric_threshold", + field=models.FloatField(default=0.7), + ), + ] diff --git a/cvat/apps/quality_control/models.py b/cvat/apps/quality_control/models.py index c92e677479e..37f0f1f9612 100644 --- a/cvat/apps/quality_control/models.py +++ b/cvat/apps/quality_control/models.py @@ -69,6 +69,19 @@ def choices(cls): return tuple((x.value, x.name) for x in cls) +class QualityTargetMetricType(str, Enum): + ACCURACY = "accuracy" + PRECISION = "precision" + RECALL = "recall" + + def __str__(self) -> str: + return self.value + + @classmethod + def choices(cls): + return tuple((x.value, x.name) for x in cls) + + class QualityReport(models.Model): job = models.ForeignKey( Job, on_delete=models.CASCADE, related_name="quality_reports", null=True, blank=True @@ -89,6 +102,7 @@ class QualityReport(models.Model): assignee = models.ForeignKey( User, on_delete=models.SET_NULL, related_name="quality_reports", null=True, blank=True ) + assignee_last_updated = models.DateTimeField(null=True) data = models.JSONField() @@ -204,6 +218,16 @@ class QualitySettings(models.Model): compare_attributes = models.BooleanField() + target_metric = models.CharField( + max_length=32, + choices=QualityTargetMetricType.choices(), + default=QualityTargetMetricType.ACCURACY, + ) + + target_metric_threshold = models.FloatField(default=0.7) + + max_validations_per_job = models.PositiveIntegerField(default=0) + def __init__(self, *args: Any, **kwargs: Any) -> None: defaults = deepcopy(self.get_defaults()) for field in self._meta.fields: diff --git a/cvat/apps/quality_control/quality_reports.py b/cvat/apps/quality_control/quality_reports.py index edc4264d380..a62e46f52cc 100644 --- a/cvat/apps/quality_control/quality_reports.py +++ b/cvat/apps/quality_control/quality_reports.py @@ -2302,6 +2302,7 @@ def _compute_reports(self, task_id: int) -> int: target_last_updated=job.updated_date, gt_last_updated=gt_job.updated_date, assignee_id=job.assignee_id, + assignee_last_updated=job.assignee_updated_date, data=job_comparison_report.to_json(), conflicts=[c.to_dict() for c in job_comparison_report.conflicts], ) @@ -2314,6 +2315,7 @@ def _compute_reports(self, task_id: int) -> int: target_last_updated=task.updated_date, gt_last_updated=gt_job.updated_date, assignee_id=task.assignee_id, + assignee_last_updated=task.assignee_updated_date, data=task_comparison_report.to_json(), conflicts=[], # the task doesn't have own conflicts ), @@ -2420,6 +2422,7 @@ def _save_reports(self, *, task_report: Dict, job_reports: List[Dict]) -> models target_last_updated=task_report["target_last_updated"], gt_last_updated=task_report["gt_last_updated"], assignee_id=task_report["assignee_id"], + assignee_last_updated=task_report["assignee_last_updated"], data=task_report["data"], ) db_task_report.save() @@ -2432,6 +2435,7 @@ def _save_reports(self, *, task_report: Dict, job_reports: List[Dict]) -> models target_last_updated=job_report["target_last_updated"], gt_last_updated=job_report["gt_last_updated"], assignee_id=job_report["assignee_id"], + assignee_last_updated=job_report["assignee_last_updated"], data=job_report["data"], ) db_job_reports.append(db_job_report) diff --git a/cvat/apps/quality_control/serializers.py b/cvat/apps/quality_control/serializers.py index 2ebe8f5a8c7..0a669962c8c 100644 --- a/cvat/apps/quality_control/serializers.py +++ b/cvat/apps/quality_control/serializers.py @@ -34,13 +34,15 @@ class QualityReportSummarySerializer(serializers.Serializer): error_count = serializers.IntegerField() conflicts_by_type = serializers.DictField(child=serializers.IntegerField()) - # This set is enough for basic characteristics, such as - # DS_unmatched, GT_unmatched, accuracy, precision and recall valid_count = serializers.IntegerField(source="annotations.valid_count") ds_count = serializers.IntegerField(source="annotations.ds_count") gt_count = serializers.IntegerField(source="annotations.gt_count") total_count = serializers.IntegerField(source="annotations.total_count") + accuracy = serializers.FloatField(source="annotations.accuracy") + precision = serializers.FloatField(source="annotations.precision") + recall = serializers.FloatField(source="annotations.recall") + class QualityReportSerializer(serializers.ModelSerializer): target = serializers.ChoiceField(models.QualityReportTarget.choices()) @@ -74,6 +76,9 @@ class Meta: fields = ( "id", "task_id", + "target_metric", + "target_metric_threshold", + "max_validations_per_job", "iou_threshold", "oks_sigma", "line_thickness", @@ -95,6 +100,15 @@ class Meta: extra_kwargs = {k: {"required": False} for k in fields} for field_name, help_text in { + "target_metric": "The primary metric used for quality estimation", + "target_metric_threshold": """ + Defines the minimal quality requirements in terms of the selected target metric. + """, + "max_validations_per_job": """ + The maximum number of job validation attempts for the job assignee. + The job can be automatically accepted if the job quality is above the required + threshold, defined by the target threshold parameter. + """, "iou_threshold": "Used for distinction between matched / unmatched shapes", "low_overlap_threshold": """ Used for distinction between strong / weak (low_overlap) matches diff --git a/cvat/apps/webhooks/permissions.py b/cvat/apps/webhooks/permissions.py index 40b4d3ebfd2..d2ffd87b395 100644 --- a/cvat/apps/webhooks/permissions.py +++ b/cvat/apps/webhooks/permissions.py @@ -39,7 +39,7 @@ def create(cls, request, view, obj, iam_context): permissions.append(perm) if project_id: - perm = ProjectPermission.create_scope_view(iam_context, project_id) + perm = ProjectPermission.create_scope_view(request, project_id, iam_context) permissions.append(perm) return permissions diff --git a/cvat/schema.yml b/cvat/schema.yml index 4c6a14a51b6..9135c153fc0 100644 --- a/cvat/schema.yml +++ b/cvat/schema.yml @@ -9329,6 +9329,28 @@ components: PatchedQualitySettingsRequest: type: object properties: + target_metric: + allOf: + - $ref: '#/components/schemas/TargetMetricEnum' + description: |- + The primary metric used for quality estimation + + * `accuracy` - ACCURACY + * `precision` - PRECISION + * `recall` - RECALL + target_metric_threshold: + type: number + format: double + description: | + Defines the minimal quality requirements in terms of the selected target metric. + max_validations_per_job: + type: integer + maximum: 2147483647 + minimum: 0 + description: | + The maximum number of job validation attempts for the job assignee. + The job can be automatically accepted if the job quality is above the required + threshold, defined by the target threshold parameter. iou_threshold: type: number format: double @@ -9728,7 +9750,17 @@ components: type: integer total_count: type: integer + accuracy: + type: number + format: double + precision: + type: number + format: double + recall: + type: number + format: double required: + - accuracy - conflict_count - conflicts_by_type - ds_count @@ -9736,6 +9768,8 @@ components: - frame_count - frame_share - gt_count + - precision + - recall - total_count - valid_count - warning_count @@ -9756,6 +9790,28 @@ components: task_id: type: integer readOnly: true + target_metric: + allOf: + - $ref: '#/components/schemas/TargetMetricEnum' + description: |- + The primary metric used for quality estimation + + * `accuracy` - ACCURACY + * `precision` - PRECISION + * `recall` - RECALL + target_metric_threshold: + type: number + format: double + description: | + Defines the minimal quality requirements in terms of the selected target metric. + max_validations_per_job: + type: integer + maximum: 2147483647 + minimum: 0 + description: | + The maximum number of job validation attempts for the job assignee. + The job can be automatically accepted if the job quality is above the required + threshold, defined by the target threshold parameter. iou_threshold: type: number format: double @@ -10334,6 +10390,16 @@ components: type: boolean required: - name + TargetMetricEnum: + enum: + - accuracy + - precision + - recall + type: string + description: |- + * `accuracy` - ACCURACY + * `precision` - PRECISION + * `recall` - RECALL TaskAnnotationsUpdateRequest: oneOf: - $ref: '#/components/schemas/LabeledDataRequest' diff --git a/tests/python/shared/assets/cvat_db/data.json b/tests/python/shared/assets/cvat_db/data.json index 1f3b9cfc0a1..5bfe4eec632 100644 --- a/tests/python/shared/assets/cvat_db/data.json +++ b/tests/python/shared/assets/cvat_db/data.json @@ -11864,6 +11864,7 @@ "target_last_updated": "2023-05-26T16:17:02.635Z", "gt_last_updated": "2023-05-26T16:16:50.630Z", "assignee": null, + "assignee_last_updated": null, "data": "{\"parameters\":{\"included_annotation_types\":[\"bbox\",\"points\",\"mask\",\"polygon\",\"polyline\",\"skeleton\"],\"compare_attributes\":true,\"ignored_attributes\":[],\"iou_threshold\":0.4,\"low_overlap_threshold\":0.8,\"oks_sigma\":0.09,\"line_thickness\":0.01,\"compare_line_orientation\":true,\"line_orientation_threshold\":0.1,\"compare_groups\":true,\"group_match_threshold\":0.5,\"check_covered_annotations\":true,\"object_visibility_threshold\":0.05,\"panoptic_comparison\":true},\"comparison_summary\":{\"frame_share\":0.2727272727272727,\"frames\":[0,1,2],\"conflict_count\":37,\"warning_count\":15,\"error_count\":22,\"conflicts_by_type\":{\"low_overlap\":6,\"missing_annotation\":9,\"extra_annotation\":10,\"mismatching_label\":3,\"mismatching_direction\":1,\"covered_annotation\":1,\"mismatching_attributes\":1,\"mismatching_groups\":6},\"annotations\":{\"valid_count\":21,\"missing_count\":9,\"extra_count\":10,\"total_count\":43,\"ds_count\":34,\"gt_count\":33,\"confusion_matrix\":{\"labels\":[\"dog\",\"cat\",\"unmatched\"],\"rows\":[[0,2,3],[1,21,7],[1,8,0]],\"precision\":[0.0,0.7241379310344828,0.0],\"recall\":[0.0,0.6774193548387096,0.0],\"accuracy\":[0.0,0.5384615384615384,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.4883720930232558,\"precision\":0.6176470588235294,\"recall\":0.6363636363636364},\"annotation_components\":{\"shape\":{\"valid_count\":24,\"missing_count\":9,\"extra_count\":10,\"total_count\":43,\"ds_count\":34,\"gt_count\":33,\"mean_iou\":0.19532955159399454,\"accuracy\":0.5581395348837209},\"label\":{\"valid_count\":21,\"invalid_count\":3,\"total_count\":24,\"accuracy\":0.875}},\"frame_count\":3,\"mean_conflict_count\":12.333333333333334},\"frame_results\":{\"0\":{\"conflicts\":[{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":65,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":101,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":91,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":118,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":88,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"},{\"obj_id\":102,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":68,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polygon\"},{\"obj_id\":98,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":67,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"},{\"obj_id\":99,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":92,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"},{\"obj_id\":119,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":107,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":121,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":122,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":70,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":76,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":93,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":74,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":95,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":73,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":96,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"ellipse\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":64,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":66,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":94,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"mismatching_label\",\"annotation_ids\":[{\"obj_id\":89,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"},{\"obj_id\":123,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"mismatching_label\",\"annotation_ids\":[{\"obj_id\":92,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"},{\"obj_id\":119,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"mismatching_label\",\"annotation_ids\":[{\"obj_id\":69,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"mask\"},{\"obj_id\":97,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"mismatching_direction\",\"annotation_ids\":[{\"obj_id\":67,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"},{\"obj_id\":99,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"covered_annotation\",\"annotation_ids\":[{\"obj_id\":69,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"mask\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_attributes\",\"annotation_ids\":[{\"obj_id\":65,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":101,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":82,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":103,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":81,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":104,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":83,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":105,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":87,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":106,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":80,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":111,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":77,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":114,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"}],\"annotations\":{\"valid_count\":21,\"missing_count\":3,\"extra_count\":10,\"total_count\":37,\"ds_count\":34,\"gt_count\":27,\"confusion_matrix\":{\"labels\":[\"dog\",\"cat\",\"unmatched\"],\"rows\":[[0,2,3],[1,21,7],[1,2,0]],\"precision\":[0.0,0.7241379310344828,0.0],\"recall\":[0.0,0.84,0.0],\"accuracy\":[0.0,0.6363636363636364,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.5675675675675675,\"precision\":0.6176470588235294,\"recall\":0.7777777777777778},\"annotation_components\":{\"shape\":{\"valid_count\":24,\"missing_count\":3,\"extra_count\":10,\"total_count\":37,\"ds_count\":34,\"gt_count\":27,\"mean_iou\":0.5859886547819836,\"accuracy\":0.6486486486486487},\"label\":{\"valid_count\":21,\"invalid_count\":3,\"total_count\":24,\"accuracy\":0.875}},\"conflict_count\":31,\"warning_count\":15,\"error_count\":16,\"conflicts_by_type\":{\"low_overlap\":6,\"missing_annotation\":3,\"extra_annotation\":10,\"mismatching_label\":3,\"mismatching_direction\":1,\"covered_annotation\":1,\"mismatching_attributes\":1,\"mismatching_groups\":6}},\"1\":{\"conflicts\":[{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":130,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":128,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":124,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":125,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"ellipse\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":127,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"mask\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":129,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"error\"}],\"annotations\":{\"valid_count\":0,\"missing_count\":6,\"extra_count\":0,\"total_count\":6,\"ds_count\":0,\"gt_count\":6,\"confusion_matrix\":{\"labels\":[\"dog\",\"cat\",\"unmatched\"],\"rows\":[[0,0,0],[0,0,0],[0,6,0]],\"precision\":[0.0,0.0,0.0],\"recall\":[0.0,0.0,0.0],\"accuracy\":[0.0,0.0,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.0,\"precision\":0.0,\"recall\":0.0},\"annotation_components\":{\"shape\":{\"valid_count\":0,\"missing_count\":6,\"extra_count\":0,\"total_count\":6,\"ds_count\":0,\"gt_count\":6,\"mean_iou\":0.0,\"accuracy\":0.0},\"label\":{\"valid_count\":0,\"invalid_count\":0,\"total_count\":0,\"accuracy\":0.0}},\"conflict_count\":6,\"warning_count\":0,\"error_count\":6,\"conflicts_by_type\":{\"missing_annotation\":6}},\"2\":{\"conflicts\":[],\"annotations\":{\"valid_count\":0,\"missing_count\":0,\"extra_count\":0,\"total_count\":0,\"ds_count\":0,\"gt_count\":0,\"confusion_matrix\":{\"labels\":[\"dog\",\"cat\",\"unmatched\"],\"rows\":[[0,0,0],[0,0,0],[0,0,0]],\"precision\":[0.0,0.0,0.0],\"recall\":[0.0,0.0,0.0],\"accuracy\":[0.0,0.0,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.0,\"precision\":0.0,\"recall\":0.0},\"annotation_components\":{\"shape\":{\"valid_count\":0,\"missing_count\":0,\"extra_count\":0,\"total_count\":0,\"ds_count\":0,\"gt_count\":0,\"mean_iou\":0,\"accuracy\":0.0},\"label\":{\"valid_count\":0,\"invalid_count\":0,\"total_count\":0,\"accuracy\":0.0}},\"conflict_count\":0,\"warning_count\":0,\"error_count\":0,\"conflicts_by_type\":{}}}}" } }, @@ -11878,6 +11879,7 @@ "target_last_updated": "2023-05-26T16:11:24.294Z", "gt_last_updated": "2023-05-26T16:16:50.630Z", "assignee": null, + "assignee_last_updated": null, "data": "{\"parameters\":{\"included_annotation_types\":[\"bbox\",\"points\",\"mask\",\"polygon\",\"polyline\",\"skeleton\"],\"compare_attributes\":true,\"ignored_attributes\":[],\"iou_threshold\":0.4,\"low_overlap_threshold\":0.8,\"oks_sigma\":0.09,\"line_thickness\":0.01,\"compare_line_orientation\":true,\"line_orientation_threshold\":0.1,\"compare_groups\":true,\"group_match_threshold\":0.5,\"check_covered_annotations\":true,\"object_visibility_threshold\":0.05,\"panoptic_comparison\":true},\"comparison_summary\":{\"frame_share\":0.2727272727272727,\"frames\":[0,1,2],\"conflict_count\":37,\"warning_count\":15,\"error_count\":22,\"conflicts_by_type\":{\"low_overlap\":6,\"missing_annotation\":9,\"extra_annotation\":10,\"mismatching_label\":3,\"mismatching_direction\":1,\"covered_annotation\":1,\"mismatching_attributes\":1,\"mismatching_groups\":6},\"annotations\":{\"valid_count\":21,\"missing_count\":9,\"extra_count\":10,\"total_count\":43,\"ds_count\":34,\"gt_count\":33,\"confusion_matrix\":{\"labels\":[\"dog\",\"cat\",\"unmatched\"],\"rows\":[[0,2,3],[1,21,7],[1,8,0]],\"precision\":[0.0,0.7241379310344828,0.0],\"recall\":[0.0,0.6774193548387096,0.0],\"accuracy\":[0.0,0.5384615384615384,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.4883720930232558,\"precision\":0.6176470588235294,\"recall\":0.6363636363636364},\"annotation_components\":{\"shape\":{\"valid_count\":24,\"missing_count\":9,\"extra_count\":10,\"total_count\":43,\"ds_count\":34,\"gt_count\":33,\"mean_iou\":0.19532955159399454,\"accuracy\":0.5581395348837209},\"label\":{\"valid_count\":21,\"invalid_count\":3,\"total_count\":24,\"accuracy\":0.875}},\"frame_count\":3,\"mean_conflict_count\":12.333333333333334},\"frame_results\":{\"0\":{\"conflicts\":[{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":65,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":101,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":91,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":118,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":88,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"},{\"obj_id\":102,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":68,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polygon\"},{\"obj_id\":98,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":67,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"},{\"obj_id\":99,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":92,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"},{\"obj_id\":119,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":107,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":121,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":122,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":70,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":76,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":93,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":74,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":95,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":73,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":96,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"ellipse\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":64,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":66,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":94,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"mismatching_label\",\"annotation_ids\":[{\"obj_id\":89,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"},{\"obj_id\":123,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"mismatching_label\",\"annotation_ids\":[{\"obj_id\":92,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"},{\"obj_id\":119,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"mismatching_label\",\"annotation_ids\":[{\"obj_id\":69,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"mask\"},{\"obj_id\":97,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"mismatching_direction\",\"annotation_ids\":[{\"obj_id\":67,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"},{\"obj_id\":99,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"covered_annotation\",\"annotation_ids\":[{\"obj_id\":69,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"mask\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_attributes\",\"annotation_ids\":[{\"obj_id\":65,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":101,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":82,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":103,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":81,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":104,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":83,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":105,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":87,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":106,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":80,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":111,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":77,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":114,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"}],\"annotations\":{\"valid_count\":21,\"missing_count\":3,\"extra_count\":10,\"total_count\":37,\"ds_count\":34,\"gt_count\":27,\"confusion_matrix\":{\"labels\":[\"dog\",\"cat\",\"unmatched\"],\"rows\":[[0,2,3],[1,21,7],[1,2,0]],\"precision\":[0.0,0.7241379310344828,0.0],\"recall\":[0.0,0.84,0.0],\"accuracy\":[0.0,0.6363636363636364,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.5675675675675675,\"precision\":0.6176470588235294,\"recall\":0.7777777777777778},\"annotation_components\":{\"shape\":{\"valid_count\":24,\"missing_count\":3,\"extra_count\":10,\"total_count\":37,\"ds_count\":34,\"gt_count\":27,\"mean_iou\":0.5859886547819836,\"accuracy\":0.6486486486486487},\"label\":{\"valid_count\":21,\"invalid_count\":3,\"total_count\":24,\"accuracy\":0.875}},\"conflict_count\":31,\"warning_count\":15,\"error_count\":16,\"conflicts_by_type\":{\"low_overlap\":6,\"missing_annotation\":3,\"extra_annotation\":10,\"mismatching_label\":3,\"mismatching_direction\":1,\"covered_annotation\":1,\"mismatching_attributes\":1,\"mismatching_groups\":6}},\"1\":{\"conflicts\":[{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":130,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":128,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":124,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":125,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"ellipse\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":127,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"mask\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":129,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"error\"}],\"annotations\":{\"valid_count\":0,\"missing_count\":6,\"extra_count\":0,\"total_count\":6,\"ds_count\":0,\"gt_count\":6,\"confusion_matrix\":{\"labels\":[\"dog\",\"cat\",\"unmatched\"],\"rows\":[[0,0,0],[0,0,0],[0,6,0]],\"precision\":[0.0,0.0,0.0],\"recall\":[0.0,0.0,0.0],\"accuracy\":[0.0,0.0,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.0,\"precision\":0.0,\"recall\":0.0},\"annotation_components\":{\"shape\":{\"valid_count\":0,\"missing_count\":6,\"extra_count\":0,\"total_count\":6,\"ds_count\":0,\"gt_count\":6,\"mean_iou\":0.0,\"accuracy\":0.0},\"label\":{\"valid_count\":0,\"invalid_count\":0,\"total_count\":0,\"accuracy\":0.0}},\"conflict_count\":6,\"warning_count\":0,\"error_count\":6,\"conflicts_by_type\":{\"missing_annotation\":6}},\"2\":{\"conflicts\":[],\"annotations\":{\"valid_count\":0,\"missing_count\":0,\"extra_count\":0,\"total_count\":0,\"ds_count\":0,\"gt_count\":0,\"confusion_matrix\":{\"labels\":[\"dog\",\"cat\",\"unmatched\"],\"rows\":[[0,0,0],[0,0,0],[0,0,0]],\"precision\":[0.0,0.0,0.0],\"recall\":[0.0,0.0,0.0],\"accuracy\":[0.0,0.0,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.0,\"precision\":0.0,\"recall\":0.0},\"annotation_components\":{\"shape\":{\"valid_count\":0,\"missing_count\":0,\"extra_count\":0,\"total_count\":0,\"ds_count\":0,\"gt_count\":0,\"mean_iou\":0,\"accuracy\":0.0},\"label\":{\"valid_count\":0,\"invalid_count\":0,\"total_count\":0,\"accuracy\":0.0}},\"conflict_count\":0,\"warning_count\":0,\"error_count\":0,\"conflicts_by_type\":{}}}}" } }, @@ -11892,6 +11894,7 @@ "target_last_updated": "2023-11-24T15:23:30.045Z", "gt_last_updated": "2023-11-24T15:18:55.216Z", "assignee": null, + "assignee_last_updated": null, "data": "{\"parameters\":{\"included_annotation_types\":[\"bbox\",\"points\",\"mask\",\"polygon\",\"polyline\",\"skeleton\"],\"compare_attributes\":true,\"ignored_attributes\":[],\"iou_threshold\":0.4,\"low_overlap_threshold\":0.8,\"oks_sigma\":0.09,\"line_thickness\":0.01,\"compare_line_orientation\":true,\"line_orientation_threshold\":0.1,\"compare_groups\":true,\"group_match_threshold\":0.5,\"check_covered_annotations\":true,\"object_visibility_threshold\":0.05,\"panoptic_comparison\":true},\"comparison_summary\":{\"frame_share\":0.2727272727272727,\"frames\":[0,1,2],\"conflict_count\":40,\"warning_count\":16,\"error_count\":24,\"conflicts_by_type\":{\"low_overlap\":7,\"missing_annotation\":10,\"extra_annotation\":11,\"mismatching_label\":3,\"mismatching_direction\":1,\"covered_annotation\":1,\"mismatching_attributes\":1,\"mismatching_groups\":6},\"annotations\":{\"valid_count\":22,\"missing_count\":10,\"extra_count\":11,\"total_count\":46,\"ds_count\":36,\"gt_count\":35,\"confusion_matrix\":{\"labels\":[\"dog\",\"cat\",\"skele\",\"unmatched\"],\"rows\":[[0,2,0,3],[1,21,0,7],[0,0,1,1],[1,8,1,0]],\"precision\":[0.0,0.7241379310344828,0.5,0.0],\"recall\":[0.0,0.6774193548387096,0.5,0.0],\"accuracy\":[0.0,0.5384615384615384,0.3333333333333333,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.4782608695652174,\"precision\":0.6111111111111112,\"recall\":0.6285714285714286},\"annotation_components\":{\"shape\":{\"valid_count\":25,\"missing_count\":10,\"extra_count\":11,\"total_count\":46,\"ds_count\":36,\"gt_count\":35,\"mean_iou\":0.18567508032031,\"accuracy\":0.5434782608695652},\"label\":{\"valid_count\":22,\"invalid_count\":3,\"total_count\":25,\"accuracy\":0.88}},\"frame_count\":3,\"mean_conflict_count\":13.333333333333334},\"frame_results\":{\"0\":{\"conflicts\":[{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":65,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":101,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":91,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":118,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":88,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"},{\"obj_id\":102,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":68,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polygon\"},{\"obj_id\":98,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":67,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"},{\"obj_id\":99,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":141,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"skeleton\"},{\"obj_id\":132,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"skeleton\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":92,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"},{\"obj_id\":119,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":107,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":121,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":122,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":131,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"skeleton\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":70,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":76,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":93,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":74,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":95,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":73,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":96,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"ellipse\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":64,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":66,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":94,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":142,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"skeleton\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"mismatching_label\",\"annotation_ids\":[{\"obj_id\":92,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"},{\"obj_id\":119,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"mismatching_label\",\"annotation_ids\":[{\"obj_id\":89,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"},{\"obj_id\":123,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"mismatching_label\",\"annotation_ids\":[{\"obj_id\":69,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"mask\"},{\"obj_id\":97,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"mismatching_direction\",\"annotation_ids\":[{\"obj_id\":67,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"},{\"obj_id\":99,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"covered_annotation\",\"annotation_ids\":[{\"obj_id\":69,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"mask\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_attributes\",\"annotation_ids\":[{\"obj_id\":65,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":101,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":82,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":103,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":81,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":104,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":83,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":105,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":87,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":106,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":80,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":111,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":77,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":114,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"}],\"annotations\":{\"valid_count\":22,\"missing_count\":4,\"extra_count\":11,\"total_count\":40,\"ds_count\":36,\"gt_count\":29,\"confusion_matrix\":{\"labels\":[\"dog\",\"cat\",\"skele\",\"unmatched\"],\"rows\":[[0,2,0,3],[1,21,0,7],[0,0,1,1],[1,2,1,0]],\"precision\":[0.0,0.7241379310344828,0.5,0.0],\"recall\":[0.0,0.84,0.5,0.0],\"accuracy\":[0.0,0.6363636363636364,0.3333333333333333,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.55,\"precision\":0.6111111111111112,\"recall\":0.7586206896551724},\"annotation_components\":{\"shape\":{\"valid_count\":25,\"missing_count\":4,\"extra_count\":11,\"total_count\":40,\"ds_count\":36,\"gt_count\":29,\"mean_iou\":0.55702524096093,\"accuracy\":0.625},\"label\":{\"valid_count\":22,\"invalid_count\":3,\"total_count\":25,\"accuracy\":0.88}},\"conflict_count\":34,\"warning_count\":16,\"error_count\":18,\"conflicts_by_type\":{\"low_overlap\":7,\"missing_annotation\":4,\"extra_annotation\":11,\"mismatching_label\":3,\"mismatching_direction\":1,\"covered_annotation\":1,\"mismatching_attributes\":1,\"mismatching_groups\":6}},\"1\":{\"conflicts\":[{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":130,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":128,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":124,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":125,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"ellipse\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":127,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"mask\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":129,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"error\"}],\"annotations\":{\"valid_count\":0,\"missing_count\":6,\"extra_count\":0,\"total_count\":6,\"ds_count\":0,\"gt_count\":6,\"confusion_matrix\":{\"labels\":[\"dog\",\"cat\",\"skele\",\"unmatched\"],\"rows\":[[0,0,0,0],[0,0,0,0],[0,0,0,0],[0,6,0,0]],\"precision\":[0.0,0.0,0.0,0.0],\"recall\":[0.0,0.0,0.0,0.0],\"accuracy\":[0.0,0.0,0.0,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.0,\"precision\":0.0,\"recall\":0.0},\"annotation_components\":{\"shape\":{\"valid_count\":0,\"missing_count\":6,\"extra_count\":0,\"total_count\":6,\"ds_count\":0,\"gt_count\":6,\"mean_iou\":0.0,\"accuracy\":0.0},\"label\":{\"valid_count\":0,\"invalid_count\":0,\"total_count\":0,\"accuracy\":0.0}},\"conflict_count\":6,\"warning_count\":0,\"error_count\":6,\"conflicts_by_type\":{\"missing_annotation\":6}},\"2\":{\"conflicts\":[],\"annotations\":{\"valid_count\":0,\"missing_count\":0,\"extra_count\":0,\"total_count\":0,\"ds_count\":0,\"gt_count\":0,\"confusion_matrix\":{\"labels\":[\"dog\",\"cat\",\"skele\",\"unmatched\"],\"rows\":[[0,0,0,0],[0,0,0,0],[0,0,0,0],[0,0,0,0]],\"precision\":[0.0,0.0,0.0,0.0],\"recall\":[0.0,0.0,0.0,0.0],\"accuracy\":[0.0,0.0,0.0,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.0,\"precision\":0.0,\"recall\":0.0},\"annotation_components\":{\"shape\":{\"valid_count\":0,\"missing_count\":0,\"extra_count\":0,\"total_count\":0,\"ds_count\":0,\"gt_count\":0,\"mean_iou\":0,\"accuracy\":0.0},\"label\":{\"valid_count\":0,\"invalid_count\":0,\"total_count\":0,\"accuracy\":0.0}},\"conflict_count\":0,\"warning_count\":0,\"error_count\":0,\"conflicts_by_type\":{}}}}" } }, @@ -11906,6 +11909,7 @@ "target_last_updated": "2023-11-24T15:23:30.269Z", "gt_last_updated": "2023-11-24T15:18:55.216Z", "assignee": null, + "assignee_last_updated": null, "data": "{\"parameters\":{\"included_annotation_types\":[\"bbox\",\"points\",\"mask\",\"polygon\",\"polyline\",\"skeleton\"],\"compare_attributes\":true,\"ignored_attributes\":[],\"iou_threshold\":0.4,\"low_overlap_threshold\":0.8,\"oks_sigma\":0.09,\"line_thickness\":0.01,\"compare_line_orientation\":true,\"line_orientation_threshold\":0.1,\"compare_groups\":true,\"group_match_threshold\":0.5,\"check_covered_annotations\":true,\"object_visibility_threshold\":0.05,\"panoptic_comparison\":true},\"comparison_summary\":{\"frame_share\":0.2727272727272727,\"frames\":[0,1,2],\"conflict_count\":40,\"warning_count\":16,\"error_count\":24,\"conflicts_by_type\":{\"low_overlap\":7,\"missing_annotation\":10,\"extra_annotation\":11,\"mismatching_label\":3,\"mismatching_direction\":1,\"covered_annotation\":1,\"mismatching_attributes\":1,\"mismatching_groups\":6},\"annotations\":{\"valid_count\":22,\"missing_count\":10,\"extra_count\":11,\"total_count\":46,\"ds_count\":36,\"gt_count\":35,\"confusion_matrix\":{\"labels\":[\"dog\",\"cat\",\"skele\",\"unmatched\"],\"rows\":[[0,2,0,3],[1,21,0,7],[0,0,1,1],[1,8,1,0]],\"precision\":[0.0,0.7241379310344828,0.5,0.0],\"recall\":[0.0,0.6774193548387096,0.5,0.0],\"accuracy\":[0.0,0.5384615384615384,0.3333333333333333,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.4782608695652174,\"precision\":0.6111111111111112,\"recall\":0.6285714285714286},\"annotation_components\":{\"shape\":{\"valid_count\":25,\"missing_count\":10,\"extra_count\":11,\"total_count\":46,\"ds_count\":36,\"gt_count\":35,\"mean_iou\":0.18567508032031,\"accuracy\":0.5434782608695652},\"label\":{\"valid_count\":22,\"invalid_count\":3,\"total_count\":25,\"accuracy\":0.88}},\"frame_count\":3,\"mean_conflict_count\":13.333333333333334},\"frame_results\":{\"0\":{\"conflicts\":[{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":65,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":101,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":91,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":118,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":88,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"},{\"obj_id\":102,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":68,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polygon\"},{\"obj_id\":98,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":67,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"},{\"obj_id\":99,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":141,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"skeleton\"},{\"obj_id\":132,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"skeleton\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":92,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"},{\"obj_id\":119,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":107,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":121,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":122,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":131,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"skeleton\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":70,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":76,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":93,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":74,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":95,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":73,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":96,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"ellipse\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":64,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":66,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":94,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":142,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"skeleton\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"mismatching_label\",\"annotation_ids\":[{\"obj_id\":92,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"},{\"obj_id\":119,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"mismatching_label\",\"annotation_ids\":[{\"obj_id\":89,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"},{\"obj_id\":123,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"mismatching_label\",\"annotation_ids\":[{\"obj_id\":69,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"mask\"},{\"obj_id\":97,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"mismatching_direction\",\"annotation_ids\":[{\"obj_id\":67,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"},{\"obj_id\":99,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"covered_annotation\",\"annotation_ids\":[{\"obj_id\":69,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"mask\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_attributes\",\"annotation_ids\":[{\"obj_id\":65,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":101,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":82,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":103,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":81,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":104,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":83,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":105,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":87,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":106,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":80,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":111,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":77,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":114,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"}],\"annotations\":{\"valid_count\":22,\"missing_count\":4,\"extra_count\":11,\"total_count\":40,\"ds_count\":36,\"gt_count\":29,\"confusion_matrix\":{\"labels\":[\"dog\",\"cat\",\"skele\",\"unmatched\"],\"rows\":[[0,2,0,3],[1,21,0,7],[0,0,1,1],[1,2,1,0]],\"precision\":[0.0,0.7241379310344828,0.5,0.0],\"recall\":[0.0,0.84,0.5,0.0],\"accuracy\":[0.0,0.6363636363636364,0.3333333333333333,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.55,\"precision\":0.6111111111111112,\"recall\":0.7586206896551724},\"annotation_components\":{\"shape\":{\"valid_count\":25,\"missing_count\":4,\"extra_count\":11,\"total_count\":40,\"ds_count\":36,\"gt_count\":29,\"mean_iou\":0.55702524096093,\"accuracy\":0.625},\"label\":{\"valid_count\":22,\"invalid_count\":3,\"total_count\":25,\"accuracy\":0.88}},\"conflict_count\":34,\"warning_count\":16,\"error_count\":18,\"conflicts_by_type\":{\"low_overlap\":7,\"missing_annotation\":4,\"extra_annotation\":11,\"mismatching_label\":3,\"mismatching_direction\":1,\"covered_annotation\":1,\"mismatching_attributes\":1,\"mismatching_groups\":6}},\"1\":{\"conflicts\":[{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":130,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":128,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":124,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":125,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"ellipse\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":127,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"mask\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":129,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"error\"}],\"annotations\":{\"valid_count\":0,\"missing_count\":6,\"extra_count\":0,\"total_count\":6,\"ds_count\":0,\"gt_count\":6,\"confusion_matrix\":{\"labels\":[\"dog\",\"cat\",\"skele\",\"unmatched\"],\"rows\":[[0,0,0,0],[0,0,0,0],[0,0,0,0],[0,6,0,0]],\"precision\":[0.0,0.0,0.0,0.0],\"recall\":[0.0,0.0,0.0,0.0],\"accuracy\":[0.0,0.0,0.0,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.0,\"precision\":0.0,\"recall\":0.0},\"annotation_components\":{\"shape\":{\"valid_count\":0,\"missing_count\":6,\"extra_count\":0,\"total_count\":6,\"ds_count\":0,\"gt_count\":6,\"mean_iou\":0.0,\"accuracy\":0.0},\"label\":{\"valid_count\":0,\"invalid_count\":0,\"total_count\":0,\"accuracy\":0.0}},\"conflict_count\":6,\"warning_count\":0,\"error_count\":6,\"conflicts_by_type\":{\"missing_annotation\":6}},\"2\":{\"conflicts\":[],\"annotations\":{\"valid_count\":0,\"missing_count\":0,\"extra_count\":0,\"total_count\":0,\"ds_count\":0,\"gt_count\":0,\"confusion_matrix\":{\"labels\":[\"dog\",\"cat\",\"skele\",\"unmatched\"],\"rows\":[[0,0,0,0],[0,0,0,0],[0,0,0,0],[0,0,0,0]],\"precision\":[0.0,0.0,0.0,0.0],\"recall\":[0.0,0.0,0.0,0.0],\"accuracy\":[0.0,0.0,0.0,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.0,\"precision\":0.0,\"recall\":0.0},\"annotation_components\":{\"shape\":{\"valid_count\":0,\"missing_count\":0,\"extra_count\":0,\"total_count\":0,\"ds_count\":0,\"gt_count\":0,\"mean_iou\":0,\"accuracy\":0.0},\"label\":{\"valid_count\":0,\"invalid_count\":0,\"total_count\":0,\"accuracy\":0.0}},\"conflict_count\":0,\"warning_count\":0,\"error_count\":0,\"conflicts_by_type\":{}}}}" } }, @@ -11920,6 +11924,7 @@ "target_last_updated": "2023-11-24T15:23:30.045Z", "gt_last_updated": "2023-11-24T15:18:55.216Z", "assignee": null, + "assignee_last_updated": null, "data": "{\"parameters\":{\"included_annotation_types\":[\"bbox\",\"points\",\"mask\",\"polygon\",\"polyline\",\"skeleton\"],\"compare_attributes\":true,\"ignored_attributes\":[],\"iou_threshold\":0.4,\"low_overlap_threshold\":0.8,\"oks_sigma\":0.09,\"line_thickness\":0.01,\"compare_line_orientation\":true,\"line_orientation_threshold\":0.1,\"compare_groups\":true,\"group_match_threshold\":0.5,\"check_covered_annotations\":true,\"object_visibility_threshold\":0.05,\"panoptic_comparison\":true},\"comparison_summary\":{\"frame_share\":0.2727272727272727,\"frames\":[0,1,2],\"conflict_count\":40,\"warning_count\":16,\"error_count\":24,\"conflicts_by_type\":{\"low_overlap\":7,\"missing_annotation\":10,\"extra_annotation\":11,\"mismatching_label\":3,\"mismatching_direction\":1,\"covered_annotation\":1,\"mismatching_attributes\":1,\"mismatching_groups\":6},\"annotations\":{\"valid_count\":22,\"missing_count\":10,\"extra_count\":11,\"total_count\":46,\"ds_count\":36,\"gt_count\":35,\"confusion_matrix\":{\"labels\":[\"dog\",\"cat\",\"skele\",\"unmatched\"],\"rows\":[[0,2,0,3],[1,21,0,7],[0,0,1,1],[1,8,1,0]],\"precision\":[0.0,0.7241379310344828,0.5,0.0],\"recall\":[0.0,0.6774193548387096,0.5,0.0],\"accuracy\":[0.8478260869565217,0.6086956521739131,0.9565217391304348,0.5434782608695652],\"jaccard_index\":[0.0,0.5384615384615384,0.3333333333333333,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.4782608695652174,\"precision\":0.6111111111111112,\"recall\":0.6285714285714286},\"annotation_components\":{\"shape\":{\"valid_count\":25,\"missing_count\":10,\"extra_count\":11,\"total_count\":46,\"ds_count\":36,\"gt_count\":35,\"mean_iou\":0.18567508032031,\"accuracy\":0.5434782608695652},\"label\":{\"valid_count\":22,\"invalid_count\":3,\"total_count\":25,\"accuracy\":0.88}},\"frame_count\":3,\"mean_conflict_count\":13.333333333333334},\"frame_results\":{\"0\":{\"conflicts\":[{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":65,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":101,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":91,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":118,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":88,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"},{\"obj_id\":102,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":68,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polygon\"},{\"obj_id\":98,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":67,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"},{\"obj_id\":99,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":141,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"skeleton\"},{\"obj_id\":132,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"skeleton\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":92,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"},{\"obj_id\":119,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":107,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":121,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":122,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":131,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"skeleton\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":70,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":76,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":93,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":74,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":95,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":73,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":96,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"ellipse\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":64,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":66,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":94,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":142,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"skeleton\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"mismatching_label\",\"annotation_ids\":[{\"obj_id\":92,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"},{\"obj_id\":119,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"mismatching_label\",\"annotation_ids\":[{\"obj_id\":89,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"},{\"obj_id\":123,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"mismatching_label\",\"annotation_ids\":[{\"obj_id\":69,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"mask\"},{\"obj_id\":97,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"mismatching_direction\",\"annotation_ids\":[{\"obj_id\":67,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"},{\"obj_id\":99,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"covered_annotation\",\"annotation_ids\":[{\"obj_id\":69,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"mask\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_attributes\",\"annotation_ids\":[{\"obj_id\":65,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":101,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":82,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":103,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":81,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":104,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":83,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":105,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":87,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":106,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":80,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":111,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":77,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":114,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"}],\"annotations\":{\"valid_count\":22,\"missing_count\":4,\"extra_count\":11,\"total_count\":40,\"ds_count\":36,\"gt_count\":29,\"confusion_matrix\":{\"labels\":[\"dog\",\"cat\",\"skele\",\"unmatched\"],\"rows\":[[0,2,0,3],[1,21,0,7],[0,0,1,1],[1,2,1,0]],\"precision\":[0.0,0.7241379310344828,0.5,0.0],\"recall\":[0.0,0.84,0.5,0.0],\"accuracy\":[0.825,0.7,0.95,0.625],\"jaccard_index\":[0.0,0.6363636363636364,0.3333333333333333,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.55,\"precision\":0.6111111111111112,\"recall\":0.7586206896551724},\"annotation_components\":{\"shape\":{\"valid_count\":25,\"missing_count\":4,\"extra_count\":11,\"total_count\":40,\"ds_count\":36,\"gt_count\":29,\"mean_iou\":0.55702524096093,\"accuracy\":0.625},\"label\":{\"valid_count\":22,\"invalid_count\":3,\"total_count\":25,\"accuracy\":0.88}},\"conflict_count\":34,\"warning_count\":16,\"error_count\":18,\"conflicts_by_type\":{\"low_overlap\":7,\"missing_annotation\":4,\"extra_annotation\":11,\"mismatching_label\":3,\"mismatching_direction\":1,\"covered_annotation\":1,\"mismatching_attributes\":1,\"mismatching_groups\":6}},\"1\":{\"conflicts\":[{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":130,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":128,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":124,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":125,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"ellipse\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":127,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"mask\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":129,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"error\"}],\"annotations\":{\"valid_count\":0,\"missing_count\":6,\"extra_count\":0,\"total_count\":6,\"ds_count\":0,\"gt_count\":6,\"confusion_matrix\":{\"labels\":[\"dog\",\"cat\",\"skele\",\"unmatched\"],\"rows\":[[0,0,0,0],[0,0,0,0],[0,0,0,0],[0,6,0,0]],\"precision\":[0.0,0.0,0.0,0.0],\"recall\":[0.0,0.0,0.0,0.0],\"accuracy\":[1.0,0.0,1.0,0.0],\"jaccard_index\":[0.0,0.0,0.0,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.0,\"precision\":0.0,\"recall\":0.0},\"annotation_components\":{\"shape\":{\"valid_count\":0,\"missing_count\":6,\"extra_count\":0,\"total_count\":6,\"ds_count\":0,\"gt_count\":6,\"mean_iou\":0.0,\"accuracy\":0.0},\"label\":{\"valid_count\":0,\"invalid_count\":0,\"total_count\":0,\"accuracy\":0.0}},\"conflict_count\":6,\"warning_count\":0,\"error_count\":6,\"conflicts_by_type\":{\"missing_annotation\":6}},\"2\":{\"conflicts\":[],\"annotations\":{\"valid_count\":0,\"missing_count\":0,\"extra_count\":0,\"total_count\":0,\"ds_count\":0,\"gt_count\":0,\"confusion_matrix\":{\"labels\":[\"dog\",\"cat\",\"skele\",\"unmatched\"],\"rows\":[[0,0,0,0],[0,0,0,0],[0,0,0,0],[0,0,0,0]],\"precision\":[0.0,0.0,0.0,0.0],\"recall\":[0.0,0.0,0.0,0.0],\"accuracy\":[0.0,0.0,0.0,0.0],\"jaccard_index\":[0.0,0.0,0.0,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.0,\"precision\":0.0,\"recall\":0.0},\"annotation_components\":{\"shape\":{\"valid_count\":0,\"missing_count\":0,\"extra_count\":0,\"total_count\":0,\"ds_count\":0,\"gt_count\":0,\"mean_iou\":0,\"accuracy\":0.0},\"label\":{\"valid_count\":0,\"invalid_count\":0,\"total_count\":0,\"accuracy\":0.0}},\"conflict_count\":0,\"warning_count\":0,\"error_count\":0,\"conflicts_by_type\":{}}}}" } }, @@ -11934,6 +11939,7 @@ "target_last_updated": "2023-11-24T15:23:30.269Z", "gt_last_updated": "2023-11-24T15:18:55.216Z", "assignee": null, + "assignee_last_updated": null, "data": "{\"parameters\":{\"included_annotation_types\":[\"bbox\",\"points\",\"mask\",\"polygon\",\"polyline\",\"skeleton\"],\"compare_attributes\":true,\"ignored_attributes\":[],\"iou_threshold\":0.4,\"low_overlap_threshold\":0.8,\"oks_sigma\":0.09,\"line_thickness\":0.01,\"compare_line_orientation\":true,\"line_orientation_threshold\":0.1,\"compare_groups\":true,\"group_match_threshold\":0.5,\"check_covered_annotations\":true,\"object_visibility_threshold\":0.05,\"panoptic_comparison\":true},\"comparison_summary\":{\"frame_share\":0.2727272727272727,\"frames\":[0,1,2],\"conflict_count\":40,\"warning_count\":16,\"error_count\":24,\"conflicts_by_type\":{\"low_overlap\":7,\"missing_annotation\":10,\"extra_annotation\":11,\"mismatching_label\":3,\"mismatching_direction\":1,\"covered_annotation\":1,\"mismatching_attributes\":1,\"mismatching_groups\":6},\"annotations\":{\"valid_count\":22,\"missing_count\":10,\"extra_count\":11,\"total_count\":46,\"ds_count\":36,\"gt_count\":35,\"confusion_matrix\":{\"labels\":[\"dog\",\"cat\",\"skele\",\"unmatched\"],\"rows\":[[0,2,0,3],[1,21,0,7],[0,0,1,1],[1,8,1,0]],\"precision\":[0.0,0.7241379310344828,0.5,0.0],\"recall\":[0.0,0.6774193548387096,0.5,0.0],\"accuracy\":[0.8478260869565217,0.6086956521739131,0.9565217391304348,0.5434782608695652],\"jaccard_index\":[0.0,0.5384615384615384,0.3333333333333333,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.4782608695652174,\"precision\":0.6111111111111112,\"recall\":0.6285714285714286},\"annotation_components\":{\"shape\":{\"valid_count\":25,\"missing_count\":10,\"extra_count\":11,\"total_count\":46,\"ds_count\":36,\"gt_count\":35,\"mean_iou\":0.18567508032031,\"accuracy\":0.5434782608695652},\"label\":{\"valid_count\":22,\"invalid_count\":3,\"total_count\":25,\"accuracy\":0.88}},\"frame_count\":3,\"mean_conflict_count\":13.333333333333334},\"frame_results\":{\"0\":{\"conflicts\":[{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":65,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":101,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":91,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":118,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":88,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"},{\"obj_id\":102,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":68,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polygon\"},{\"obj_id\":98,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":67,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"},{\"obj_id\":99,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":141,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"skeleton\"},{\"obj_id\":132,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"skeleton\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":92,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"},{\"obj_id\":119,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":107,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":121,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":122,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":131,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"skeleton\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":70,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":76,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":93,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":74,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":95,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":73,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":96,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"ellipse\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":64,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":66,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":94,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":142,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"skeleton\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"mismatching_label\",\"annotation_ids\":[{\"obj_id\":92,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"},{\"obj_id\":119,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"mismatching_label\",\"annotation_ids\":[{\"obj_id\":89,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"},{\"obj_id\":123,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"mismatching_label\",\"annotation_ids\":[{\"obj_id\":69,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"mask\"},{\"obj_id\":97,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"mismatching_direction\",\"annotation_ids\":[{\"obj_id\":67,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"},{\"obj_id\":99,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"covered_annotation\",\"annotation_ids\":[{\"obj_id\":69,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"mask\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_attributes\",\"annotation_ids\":[{\"obj_id\":65,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":101,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":82,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":103,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":81,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":104,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":83,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":105,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":87,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":106,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":80,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":111,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":77,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":114,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"}],\"annotations\":{\"valid_count\":22,\"missing_count\":4,\"extra_count\":11,\"total_count\":40,\"ds_count\":36,\"gt_count\":29,\"confusion_matrix\":{\"labels\":[\"dog\",\"cat\",\"skele\",\"unmatched\"],\"rows\":[[0,2,0,3],[1,21,0,7],[0,0,1,1],[1,2,1,0]],\"precision\":[0.0,0.7241379310344828,0.5,0.0],\"recall\":[0.0,0.84,0.5,0.0],\"accuracy\":[0.825,0.7,0.95,0.625],\"jaccard_index\":[0.0,0.6363636363636364,0.3333333333333333,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.55,\"precision\":0.6111111111111112,\"recall\":0.7586206896551724},\"annotation_components\":{\"shape\":{\"valid_count\":25,\"missing_count\":4,\"extra_count\":11,\"total_count\":40,\"ds_count\":36,\"gt_count\":29,\"mean_iou\":0.55702524096093,\"accuracy\":0.625},\"label\":{\"valid_count\":22,\"invalid_count\":3,\"total_count\":25,\"accuracy\":0.88}},\"conflict_count\":34,\"warning_count\":16,\"error_count\":18,\"conflicts_by_type\":{\"low_overlap\":7,\"missing_annotation\":4,\"extra_annotation\":11,\"mismatching_label\":3,\"mismatching_direction\":1,\"covered_annotation\":1,\"mismatching_attributes\":1,\"mismatching_groups\":6}},\"1\":{\"conflicts\":[{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":130,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":128,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":124,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":125,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"ellipse\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":127,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"mask\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":129,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"error\"}],\"annotations\":{\"valid_count\":0,\"missing_count\":6,\"extra_count\":0,\"total_count\":6,\"ds_count\":0,\"gt_count\":6,\"confusion_matrix\":{\"labels\":[\"dog\",\"cat\",\"skele\",\"unmatched\"],\"rows\":[[0,0,0,0],[0,0,0,0],[0,0,0,0],[0,6,0,0]],\"precision\":[0.0,0.0,0.0,0.0],\"recall\":[0.0,0.0,0.0,0.0],\"accuracy\":[1.0,0.0,1.0,0.0],\"jaccard_index\":[0.0,0.0,0.0,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.0,\"precision\":0.0,\"recall\":0.0},\"annotation_components\":{\"shape\":{\"valid_count\":0,\"missing_count\":6,\"extra_count\":0,\"total_count\":6,\"ds_count\":0,\"gt_count\":6,\"mean_iou\":0.0,\"accuracy\":0.0},\"label\":{\"valid_count\":0,\"invalid_count\":0,\"total_count\":0,\"accuracy\":0.0}},\"conflict_count\":6,\"warning_count\":0,\"error_count\":6,\"conflicts_by_type\":{\"missing_annotation\":6}},\"2\":{\"conflicts\":[],\"annotations\":{\"valid_count\":0,\"missing_count\":0,\"extra_count\":0,\"total_count\":0,\"ds_count\":0,\"gt_count\":0,\"confusion_matrix\":{\"labels\":[\"dog\",\"cat\",\"skele\",\"unmatched\"],\"rows\":[[0,0,0,0],[0,0,0,0],[0,0,0,0],[0,0,0,0]],\"precision\":[0.0,0.0,0.0,0.0],\"recall\":[0.0,0.0,0.0,0.0],\"accuracy\":[0.0,0.0,0.0,0.0],\"jaccard_index\":[0.0,0.0,0.0,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.0,\"precision\":0.0,\"recall\":0.0},\"annotation_components\":{\"shape\":{\"valid_count\":0,\"missing_count\":0,\"extra_count\":0,\"total_count\":0,\"ds_count\":0,\"gt_count\":0,\"mean_iou\":0,\"accuracy\":0.0},\"label\":{\"valid_count\":0,\"invalid_count\":0,\"total_count\":0,\"accuracy\":0.0}},\"conflict_count\":0,\"warning_count\":0,\"error_count\":0,\"conflicts_by_type\":{}}}}" } }, @@ -11948,6 +11954,7 @@ "target_last_updated": "2024-03-21T20:50:05.947Z", "gt_last_updated": "2024-03-21T20:50:20.020Z", "assignee": null, + "assignee_last_updated": null, "data": "{\"parameters\":{\"included_annotation_types\":[\"bbox\",\"points\",\"mask\",\"polygon\",\"polyline\",\"skeleton\"],\"compare_attributes\":true,\"ignored_attributes\":[],\"iou_threshold\":0.4,\"low_overlap_threshold\":0.8,\"oks_sigma\":0.09,\"line_thickness\":0.01,\"compare_line_orientation\":true,\"line_orientation_threshold\":0.1,\"compare_groups\":true,\"group_match_threshold\":0.5,\"check_covered_annotations\":true,\"object_visibility_threshold\":0.05,\"panoptic_comparison\":true},\"comparison_summary\":{\"frame_share\":0.2727272727272727,\"frames\":[4,5,7],\"conflict_count\":3,\"warning_count\":1,\"error_count\":2,\"conflicts_by_type\":{\"low_overlap\":1,\"missing_annotation\":1,\"extra_annotation\":1},\"annotations\":{\"valid_count\":2,\"missing_count\":1,\"extra_count\":1,\"total_count\":4,\"ds_count\":3,\"gt_count\":3,\"confusion_matrix\":{\"labels\":[\"cat\",\"dog\",\"unmatched\"],\"rows\":[[2,0,1],[0,0,0],[1,0,0]],\"precision\":[0.0,0.0,0.0],\"recall\":[0.0,0.0,0.0],\"accuracy\":[0.0,0.0,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.5,\"precision\":0.6666666666666666,\"recall\":0.6666666666666666},\"annotation_components\":{\"shape\":{\"valid_count\":2,\"missing_count\":1,\"extra_count\":1,\"total_count\":4,\"ds_count\":3,\"gt_count\":3,\"mean_iou\":null,\"accuracy\":0.5},\"label\":{\"valid_count\":2,\"invalid_count\":0,\"total_count\":2,\"accuracy\":1.0}},\"frame_count\":3,\"mean_conflict_count\":1.0},\"frame_results\":{\"5\":{\"conflicts\":[{\"frame_id\":5,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":156,\"job_id\":30,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":163,\"job_id\":32,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"}],\"annotations\":{\"valid_count\":1,\"missing_count\":0,\"extra_count\":0,\"total_count\":1,\"ds_count\":1,\"gt_count\":1,\"confusion_matrix\":{\"labels\":[\"cat\",\"dog\",\"unmatched\"],\"rows\":[[1,0,0],[0,0,0],[0,0,0]],\"precision\":[1.0,0.0,0.0],\"recall\":[1.0,0.0,0.0],\"accuracy\":[1.0,0.0,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":1.0,\"precision\":1.0,\"recall\":1.0},\"annotation_components\":{\"shape\":{\"valid_count\":1,\"missing_count\":0,\"extra_count\":0,\"total_count\":1,\"ds_count\":1,\"gt_count\":1,\"mean_iou\":0.6077449295377204,\"accuracy\":1.0},\"label\":{\"valid_count\":1,\"invalid_count\":0,\"total_count\":1,\"accuracy\":1.0}},\"conflict_count\":1,\"warning_count\":1,\"error_count\":0,\"conflicts_by_type\":{\"low_overlap\":1}},\"7\":{\"conflicts\":[],\"annotations\":{\"valid_count\":1,\"missing_count\":0,\"extra_count\":0,\"total_count\":1,\"ds_count\":1,\"gt_count\":1,\"confusion_matrix\":{\"labels\":[\"cat\",\"dog\",\"unmatched\"],\"rows\":[[1,0,0],[0,0,0],[0,0,0]],\"precision\":[1.0,0.0,0.0],\"recall\":[1.0,0.0,0.0],\"accuracy\":[1.0,0.0,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":1.0,\"precision\":1.0,\"recall\":1.0},\"annotation_components\":{\"shape\":{\"valid_count\":1,\"missing_count\":0,\"extra_count\":0,\"total_count\":1,\"ds_count\":1,\"gt_count\":1,\"mean_iou\":0.8588610910105674,\"accuracy\":1.0},\"label\":{\"valid_count\":1,\"invalid_count\":0,\"total_count\":1,\"accuracy\":1.0}},\"conflict_count\":0,\"warning_count\":0,\"error_count\":0,\"conflicts_by_type\":{}},\"4\":{\"conflicts\":[{\"frame_id\":4,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":162,\"job_id\":32,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"},{\"frame_id\":4,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":155,\"job_id\":29,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"}],\"annotations\":{\"valid_count\":0,\"missing_count\":1,\"extra_count\":1,\"total_count\":2,\"ds_count\":1,\"gt_count\":1,\"confusion_matrix\":{\"labels\":[\"cat\",\"dog\",\"unmatched\"],\"rows\":[[0,0,1],[0,0,0],[1,0,0]],\"precision\":[0.0,0.0,0.0],\"recall\":[0.0,0.0,0.0],\"accuracy\":[0.0,0.0,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.0,\"precision\":0.0,\"recall\":0.0},\"annotation_components\":{\"shape\":{\"valid_count\":0,\"missing_count\":1,\"extra_count\":1,\"total_count\":2,\"ds_count\":1,\"gt_count\":1,\"mean_iou\":0.1548852356623416,\"accuracy\":0.0},\"label\":{\"valid_count\":0,\"invalid_count\":0,\"total_count\":0,\"accuracy\":0.0}},\"conflict_count\":2,\"warning_count\":0,\"error_count\":2,\"conflicts_by_type\":{\"missing_annotation\":1,\"extra_annotation\":1}}}}" } }, @@ -11962,6 +11969,7 @@ "target_last_updated": "2024-03-21T20:50:27.594Z", "gt_last_updated": "2024-03-21T20:50:20.020Z", "assignee": null, + "assignee_last_updated": null, "data": "{\"parameters\":{\"included_annotation_types\":[\"bbox\",\"points\",\"mask\",\"polygon\",\"polyline\",\"skeleton\"],\"compare_attributes\":true,\"ignored_attributes\":[],\"iou_threshold\":0.4,\"low_overlap_threshold\":0.8,\"oks_sigma\":0.09,\"line_thickness\":0.01,\"compare_line_orientation\":true,\"line_orientation_threshold\":0.1,\"compare_groups\":true,\"group_match_threshold\":0.5,\"check_covered_annotations\":true,\"object_visibility_threshold\":0.05,\"panoptic_comparison\":true},\"comparison_summary\":{\"frame_share\":0.0,\"frames\":[],\"conflict_count\":0,\"warning_count\":0,\"error_count\":0,\"conflicts_by_type\":{},\"annotations\":{\"valid_count\":0,\"missing_count\":0,\"extra_count\":0,\"total_count\":0,\"ds_count\":0,\"gt_count\":0,\"confusion_matrix\":{\"labels\":[\"cat\",\"dog\",\"unmatched\"],\"rows\":[[0,0,0],[0,0,0],[0,0,0]],\"precision\":[0.0,0.0,0.0],\"recall\":[0.0,0.0,0.0],\"accuracy\":[0.0,0.0,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.0,\"precision\":0.0,\"recall\":0.0},\"annotation_components\":{\"shape\":{\"valid_count\":0,\"missing_count\":0,\"extra_count\":0,\"total_count\":0,\"ds_count\":0,\"gt_count\":0,\"mean_iou\":null,\"accuracy\":0.0},\"label\":{\"valid_count\":0,\"invalid_count\":0,\"total_count\":0,\"accuracy\":0.0}},\"frame_count\":0,\"mean_conflict_count\":0.0},\"frame_results\":{}}" } }, @@ -11976,6 +11984,7 @@ "target_last_updated": "2024-03-21T20:50:33.610Z", "gt_last_updated": "2024-03-21T20:50:20.020Z", "assignee": null, + "assignee_last_updated": null, "data": "{\"parameters\":{\"included_annotation_types\":[\"bbox\",\"points\",\"mask\",\"polygon\",\"polyline\",\"skeleton\"],\"compare_attributes\":true,\"ignored_attributes\":[],\"iou_threshold\":0.4,\"low_overlap_threshold\":0.8,\"oks_sigma\":0.09,\"line_thickness\":0.01,\"compare_line_orientation\":true,\"line_orientation_threshold\":0.1,\"compare_groups\":true,\"group_match_threshold\":0.5,\"check_covered_annotations\":true,\"object_visibility_threshold\":0.05,\"panoptic_comparison\":true},\"comparison_summary\":{\"frame_share\":0.4,\"frames\":[5,7],\"conflict_count\":1,\"warning_count\":1,\"error_count\":0,\"conflicts_by_type\":{\"low_overlap\":1},\"annotations\":{\"valid_count\":2,\"missing_count\":0,\"extra_count\":0,\"total_count\":2,\"ds_count\":2,\"gt_count\":2,\"confusion_matrix\":{\"labels\":[\"cat\",\"dog\",\"unmatched\"],\"rows\":[[2,0,0],[0,0,0],[0,0,0]],\"precision\":[1.0,0.0,0.0],\"recall\":[1.0,0.0,0.0],\"accuracy\":[1.0,0.0,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":1.0,\"precision\":1.0,\"recall\":1.0},\"annotation_components\":{\"shape\":{\"valid_count\":2,\"missing_count\":0,\"extra_count\":0,\"total_count\":2,\"ds_count\":2,\"gt_count\":2,\"mean_iou\":0.7333030102741439,\"accuracy\":1.0},\"label\":{\"valid_count\":2,\"invalid_count\":0,\"total_count\":2,\"accuracy\":1.0}},\"frame_count\":2,\"mean_conflict_count\":0.5},\"frame_results\":{\"5\":{\"conflicts\":[{\"frame_id\":5,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":156,\"job_id\":30,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":163,\"job_id\":32,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"}],\"annotations\":{\"valid_count\":1,\"missing_count\":0,\"extra_count\":0,\"total_count\":1,\"ds_count\":1,\"gt_count\":1,\"confusion_matrix\":{\"labels\":[\"cat\",\"dog\",\"unmatched\"],\"rows\":[[1,0,0],[0,0,0],[0,0,0]],\"precision\":[1.0,0.0,0.0],\"recall\":[1.0,0.0,0.0],\"accuracy\":[1.0,0.0,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":1.0,\"precision\":1.0,\"recall\":1.0},\"annotation_components\":{\"shape\":{\"valid_count\":1,\"missing_count\":0,\"extra_count\":0,\"total_count\":1,\"ds_count\":1,\"gt_count\":1,\"mean_iou\":0.6077449295377204,\"accuracy\":1.0},\"label\":{\"valid_count\":1,\"invalid_count\":0,\"total_count\":1,\"accuracy\":1.0}},\"conflict_count\":1,\"warning_count\":1,\"error_count\":0,\"conflicts_by_type\":{\"low_overlap\":1}},\"7\":{\"conflicts\":[],\"annotations\":{\"valid_count\":1,\"missing_count\":0,\"extra_count\":0,\"total_count\":1,\"ds_count\":1,\"gt_count\":1,\"confusion_matrix\":{\"labels\":[\"cat\",\"dog\",\"unmatched\"],\"rows\":[[1,0,0],[0,0,0],[0,0,0]],\"precision\":[1.0,0.0,0.0],\"recall\":[1.0,0.0,0.0],\"accuracy\":[1.0,0.0,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":1.0,\"precision\":1.0,\"recall\":1.0},\"annotation_components\":{\"shape\":{\"valid_count\":1,\"missing_count\":0,\"extra_count\":0,\"total_count\":1,\"ds_count\":1,\"gt_count\":1,\"mean_iou\":0.8588610910105674,\"accuracy\":1.0},\"label\":{\"valid_count\":1,\"invalid_count\":0,\"total_count\":1,\"accuracy\":1.0}},\"conflict_count\":0,\"warning_count\":0,\"error_count\":0,\"conflicts_by_type\":{}}}}" } }, @@ -11990,6 +11999,7 @@ "target_last_updated": "2024-03-21T20:50:39.585Z", "gt_last_updated": "2024-03-21T20:50:20.020Z", "assignee": null, + "assignee_last_updated": null, "data": "{\"parameters\":{\"included_annotation_types\":[\"bbox\",\"points\",\"mask\",\"polygon\",\"polyline\",\"skeleton\"],\"compare_attributes\":true,\"ignored_attributes\":[],\"iou_threshold\":0.4,\"low_overlap_threshold\":0.8,\"oks_sigma\":0.09,\"line_thickness\":0.01,\"compare_line_orientation\":true,\"line_orientation_threshold\":0.1,\"compare_groups\":true,\"group_match_threshold\":0.5,\"check_covered_annotations\":true,\"object_visibility_threshold\":0.05,\"panoptic_comparison\":true},\"comparison_summary\":{\"frame_share\":0.2,\"frames\":[4],\"conflict_count\":2,\"warning_count\":0,\"error_count\":2,\"conflicts_by_type\":{\"missing_annotation\":1,\"extra_annotation\":1},\"annotations\":{\"valid_count\":0,\"missing_count\":1,\"extra_count\":1,\"total_count\":2,\"ds_count\":1,\"gt_count\":1,\"confusion_matrix\":{\"labels\":[\"cat\",\"dog\",\"unmatched\"],\"rows\":[[0,0,1],[0,0,0],[1,0,0]],\"precision\":[0.0,0.0,0.0],\"recall\":[0.0,0.0,0.0],\"accuracy\":[0.0,0.0,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.0,\"precision\":0.0,\"recall\":0.0},\"annotation_components\":{\"shape\":{\"valid_count\":0,\"missing_count\":1,\"extra_count\":1,\"total_count\":2,\"ds_count\":1,\"gt_count\":1,\"mean_iou\":0.1548852356623416,\"accuracy\":0.0},\"label\":{\"valid_count\":0,\"invalid_count\":0,\"total_count\":0,\"accuracy\":0.0}},\"frame_count\":1,\"mean_conflict_count\":2.0},\"frame_results\":{\"4\":{\"conflicts\":[{\"frame_id\":4,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":162,\"job_id\":32,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"},{\"frame_id\":4,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":155,\"job_id\":29,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"}],\"annotations\":{\"valid_count\":0,\"missing_count\":1,\"extra_count\":1,\"total_count\":2,\"ds_count\":1,\"gt_count\":1,\"confusion_matrix\":{\"labels\":[\"cat\",\"dog\",\"unmatched\"],\"rows\":[[0,0,1],[0,0,0],[1,0,0]],\"precision\":[0.0,0.0,0.0],\"recall\":[0.0,0.0,0.0],\"accuracy\":[0.0,0.0,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.0,\"precision\":0.0,\"recall\":0.0},\"annotation_components\":{\"shape\":{\"valid_count\":0,\"missing_count\":1,\"extra_count\":1,\"total_count\":2,\"ds_count\":1,\"gt_count\":1,\"mean_iou\":0.1548852356623416,\"accuracy\":0.0},\"label\":{\"valid_count\":0,\"invalid_count\":0,\"total_count\":0,\"accuracy\":0.0}},\"conflict_count\":2,\"warning_count\":0,\"error_count\":2,\"conflicts_by_type\":{\"missing_annotation\":1,\"extra_annotation\":1}}}}" } }, @@ -12004,6 +12014,7 @@ "target_last_updated": "2023-11-24T15:23:30.045Z", "gt_last_updated": "2023-11-24T15:18:55.216Z", "assignee": null, + "assignee_last_updated": null, "data": "{\"parameters\":{\"included_annotation_types\":[\"bbox\",\"points\",\"mask\",\"polygon\",\"polyline\",\"skeleton\",\"label\"],\"compare_attributes\":true,\"ignored_attributes\":[],\"iou_threshold\":0.4,\"low_overlap_threshold\":0.8,\"oks_sigma\":0.09,\"line_thickness\":0.01,\"compare_line_orientation\":true,\"line_orientation_threshold\":0.1,\"compare_groups\":true,\"group_match_threshold\":0.5,\"check_covered_annotations\":true,\"object_visibility_threshold\":0.05,\"panoptic_comparison\":true},\"comparison_summary\":{\"frame_share\":0.2727272727272727,\"frames\":[0,1,2],\"conflict_count\":42,\"warning_count\":16,\"error_count\":26,\"conflicts_by_type\":{\"low_overlap\":7,\"missing_annotation\":12,\"extra_annotation\":11,\"mismatching_label\":3,\"mismatching_direction\":1,\"covered_annotation\":1,\"mismatching_attributes\":1,\"mismatching_groups\":6},\"annotations\":{\"valid_count\":22,\"missing_count\":12,\"extra_count\":11,\"total_count\":48,\"ds_count\":36,\"gt_count\":37,\"confusion_matrix\":{\"labels\":[\"dog\",\"cat\",\"skele\",\"unmatched\"],\"rows\":[[0,2,0,3],[1,21,0,7],[0,0,1,1],[1,10,1,0]],\"precision\":[0.0,0.7241379310344828,0.5,0.0],\"recall\":[0.0,0.6363636363636364,0.5,0.0],\"accuracy\":[0.8541666666666666,0.5833333333333334,0.9583333333333334,0.5208333333333334],\"jaccard_index\":[0.0,0.5121951219512195,0.3333333333333333,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.4583333333333333,\"precision\":0.6111111111111112,\"recall\":0.5945945945945946},\"annotation_components\":{\"shape\":{\"valid_count\":25,\"missing_count\":10,\"extra_count\":11,\"total_count\":46,\"ds_count\":36,\"gt_count\":35,\"mean_iou\":0.18567508032031,\"accuracy\":0.5434782608695652},\"label\":{\"valid_count\":22,\"invalid_count\":3,\"total_count\":25,\"accuracy\":0.88}},\"frame_count\":3,\"mean_conflict_count\":14.0},\"frame_results\":{\"0\":{\"conflicts\":[{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":65,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":101,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":91,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":118,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":88,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"},{\"obj_id\":102,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":68,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polygon\"},{\"obj_id\":98,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":67,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"},{\"obj_id\":99,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":141,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"skeleton\"},{\"obj_id\":132,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"skeleton\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":92,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"},{\"obj_id\":119,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":107,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":121,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":122,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":131,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"skeleton\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":7,\"job_id\":28,\"type\":\"tag\",\"shape_type\":null}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":70,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":76,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":93,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":74,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":95,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":73,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":96,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"ellipse\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":64,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":66,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":94,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":142,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"skeleton\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"mismatching_label\",\"annotation_ids\":[{\"obj_id\":92,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"},{\"obj_id\":119,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"mismatching_label\",\"annotation_ids\":[{\"obj_id\":89,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"},{\"obj_id\":123,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"mismatching_label\",\"annotation_ids\":[{\"obj_id\":69,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"mask\"},{\"obj_id\":97,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"mismatching_direction\",\"annotation_ids\":[{\"obj_id\":67,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"},{\"obj_id\":99,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"covered_annotation\",\"annotation_ids\":[{\"obj_id\":69,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"mask\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_attributes\",\"annotation_ids\":[{\"obj_id\":65,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":101,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":82,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":103,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":81,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":104,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":83,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":105,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":87,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":106,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":80,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":111,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":77,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":114,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"}],\"annotations\":{\"valid_count\":22,\"missing_count\":5,\"extra_count\":11,\"total_count\":41,\"ds_count\":36,\"gt_count\":30,\"confusion_matrix\":{\"labels\":[\"dog\",\"cat\",\"skele\",\"unmatched\"],\"rows\":[[0,2,0,3],[1,21,0,7],[0,0,1,1],[1,3,1,0]],\"precision\":[0.0,0.7241379310344828,0.5,0.0],\"recall\":[0.0,0.8076923076923077,0.5,0.0],\"accuracy\":[0.8292682926829268,0.6829268292682927,0.9512195121951219,0.6097560975609756],\"jaccard_index\":[0.0,0.6176470588235294,0.3333333333333333,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.5365853658536586,\"precision\":0.6111111111111112,\"recall\":0.7333333333333333},\"annotation_components\":{\"shape\":{\"valid_count\":25,\"missing_count\":4,\"extra_count\":11,\"total_count\":40,\"ds_count\":36,\"gt_count\":29,\"mean_iou\":0.55702524096093,\"accuracy\":0.625},\"label\":{\"valid_count\":22,\"invalid_count\":3,\"total_count\":25,\"accuracy\":0.88}},\"conflict_count\":35,\"warning_count\":16,\"error_count\":19,\"conflicts_by_type\":{\"low_overlap\":7,\"missing_annotation\":5,\"extra_annotation\":11,\"mismatching_label\":3,\"mismatching_direction\":1,\"covered_annotation\":1,\"mismatching_attributes\":1,\"mismatching_groups\":6}},\"1\":{\"conflicts\":[{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":130,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":128,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":124,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":125,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"ellipse\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":127,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"mask\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":129,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":8,\"job_id\":28,\"type\":\"tag\",\"shape_type\":null}],\"severity\":\"error\"}],\"annotations\":{\"valid_count\":0,\"missing_count\":7,\"extra_count\":0,\"total_count\":7,\"ds_count\":0,\"gt_count\":7,\"confusion_matrix\":{\"labels\":[\"dog\",\"cat\",\"skele\",\"unmatched\"],\"rows\":[[0,0,0,0],[0,0,0,0],[0,0,0,0],[0,7,0,0]],\"precision\":[0.0,0.0,0.0,0.0],\"recall\":[0.0,0.0,0.0,0.0],\"accuracy\":[1.0,0.0,1.0,0.0],\"jaccard_index\":[0.0,0.0,0.0,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.0,\"precision\":0.0,\"recall\":0.0},\"annotation_components\":{\"shape\":{\"valid_count\":0,\"missing_count\":6,\"extra_count\":0,\"total_count\":6,\"ds_count\":0,\"gt_count\":6,\"mean_iou\":0.0,\"accuracy\":0.0},\"label\":{\"valid_count\":0,\"invalid_count\":0,\"total_count\":0,\"accuracy\":0.0}},\"conflict_count\":7,\"warning_count\":0,\"error_count\":7,\"conflicts_by_type\":{\"missing_annotation\":7}},\"2\":{\"conflicts\":[],\"annotations\":{\"valid_count\":0,\"missing_count\":0,\"extra_count\":0,\"total_count\":0,\"ds_count\":0,\"gt_count\":0,\"confusion_matrix\":{\"labels\":[\"dog\",\"cat\",\"skele\",\"unmatched\"],\"rows\":[[0,0,0,0],[0,0,0,0],[0,0,0,0],[0,0,0,0]],\"precision\":[0.0,0.0,0.0,0.0],\"recall\":[0.0,0.0,0.0,0.0],\"accuracy\":[0.0,0.0,0.0,0.0],\"jaccard_index\":[0.0,0.0,0.0,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.0,\"precision\":0.0,\"recall\":0.0},\"annotation_components\":{\"shape\":{\"valid_count\":0,\"missing_count\":0,\"extra_count\":0,\"total_count\":0,\"ds_count\":0,\"gt_count\":0,\"mean_iou\":0,\"accuracy\":0.0},\"label\":{\"valid_count\":0,\"invalid_count\":0,\"total_count\":0,\"accuracy\":0.0}},\"conflict_count\":0,\"warning_count\":0,\"error_count\":0,\"conflicts_by_type\":{}}}}" } }, @@ -12018,6 +12029,7 @@ "target_last_updated": "2023-11-24T15:23:30.269Z", "gt_last_updated": "2023-11-24T15:18:55.216Z", "assignee": null, + "assignee_last_updated": null, "data": "{\"parameters\":{\"included_annotation_types\":[\"bbox\",\"points\",\"mask\",\"polygon\",\"polyline\",\"skeleton\",\"label\"],\"compare_attributes\":true,\"ignored_attributes\":[],\"iou_threshold\":0.4,\"low_overlap_threshold\":0.8,\"oks_sigma\":0.09,\"line_thickness\":0.01,\"compare_line_orientation\":true,\"line_orientation_threshold\":0.1,\"compare_groups\":true,\"group_match_threshold\":0.5,\"check_covered_annotations\":true,\"object_visibility_threshold\":0.05,\"panoptic_comparison\":true},\"comparison_summary\":{\"frame_share\":0.2727272727272727,\"frames\":[0,1,2],\"conflict_count\":42,\"warning_count\":16,\"error_count\":26,\"conflicts_by_type\":{\"low_overlap\":7,\"missing_annotation\":12,\"extra_annotation\":11,\"mismatching_label\":3,\"mismatching_direction\":1,\"covered_annotation\":1,\"mismatching_attributes\":1,\"mismatching_groups\":6},\"annotations\":{\"valid_count\":22,\"missing_count\":12,\"extra_count\":11,\"total_count\":48,\"ds_count\":36,\"gt_count\":37,\"confusion_matrix\":{\"labels\":[\"dog\",\"cat\",\"skele\",\"unmatched\"],\"rows\":[[0,2,0,3],[1,21,0,7],[0,0,1,1],[1,10,1,0]],\"precision\":[0.0,0.7241379310344828,0.5,0.0],\"recall\":[0.0,0.6363636363636364,0.5,0.0],\"accuracy\":[0.8541666666666666,0.5833333333333334,0.9583333333333334,0.5208333333333334],\"jaccard_index\":[0.0,0.5121951219512195,0.3333333333333333,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.4583333333333333,\"precision\":0.6111111111111112,\"recall\":0.5945945945945946},\"annotation_components\":{\"shape\":{\"valid_count\":25,\"missing_count\":10,\"extra_count\":11,\"total_count\":46,\"ds_count\":36,\"gt_count\":35,\"mean_iou\":0.18567508032031,\"accuracy\":0.5434782608695652},\"label\":{\"valid_count\":22,\"invalid_count\":3,\"total_count\":25,\"accuracy\":0.88}},\"frame_count\":3,\"mean_conflict_count\":14.0},\"frame_results\":{\"0\":{\"conflicts\":[{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":65,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":101,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":91,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":118,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":88,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"},{\"obj_id\":102,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":68,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polygon\"},{\"obj_id\":98,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":67,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"},{\"obj_id\":99,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":141,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"skeleton\"},{\"obj_id\":132,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"skeleton\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":92,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"},{\"obj_id\":119,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":107,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":121,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":122,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":131,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"skeleton\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":7,\"job_id\":28,\"type\":\"tag\",\"shape_type\":null}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":70,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":76,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":93,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":74,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":95,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":73,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":96,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"ellipse\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":64,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":66,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":94,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":142,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"skeleton\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"mismatching_label\",\"annotation_ids\":[{\"obj_id\":92,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"},{\"obj_id\":119,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"mismatching_label\",\"annotation_ids\":[{\"obj_id\":89,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"},{\"obj_id\":123,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"mismatching_label\",\"annotation_ids\":[{\"obj_id\":69,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"mask\"},{\"obj_id\":97,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"mismatching_direction\",\"annotation_ids\":[{\"obj_id\":67,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"},{\"obj_id\":99,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"covered_annotation\",\"annotation_ids\":[{\"obj_id\":69,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"mask\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_attributes\",\"annotation_ids\":[{\"obj_id\":65,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":101,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":82,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":103,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":81,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":104,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":83,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":105,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":87,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":106,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":80,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":111,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":77,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":114,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"}],\"annotations\":{\"valid_count\":22,\"missing_count\":5,\"extra_count\":11,\"total_count\":41,\"ds_count\":36,\"gt_count\":30,\"confusion_matrix\":{\"labels\":[\"dog\",\"cat\",\"skele\",\"unmatched\"],\"rows\":[[0,2,0,3],[1,21,0,7],[0,0,1,1],[1,3,1,0]],\"precision\":[0.0,0.7241379310344828,0.5,0.0],\"recall\":[0.0,0.8076923076923077,0.5,0.0],\"accuracy\":[0.8292682926829268,0.6829268292682927,0.9512195121951219,0.6097560975609756],\"jaccard_index\":[0.0,0.6176470588235294,0.3333333333333333,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.5365853658536586,\"precision\":0.6111111111111112,\"recall\":0.7333333333333333},\"annotation_components\":{\"shape\":{\"valid_count\":25,\"missing_count\":4,\"extra_count\":11,\"total_count\":40,\"ds_count\":36,\"gt_count\":29,\"mean_iou\":0.55702524096093,\"accuracy\":0.625},\"label\":{\"valid_count\":22,\"invalid_count\":3,\"total_count\":25,\"accuracy\":0.88}},\"conflict_count\":35,\"warning_count\":16,\"error_count\":19,\"conflicts_by_type\":{\"low_overlap\":7,\"missing_annotation\":5,\"extra_annotation\":11,\"mismatching_label\":3,\"mismatching_direction\":1,\"covered_annotation\":1,\"mismatching_attributes\":1,\"mismatching_groups\":6}},\"1\":{\"conflicts\":[{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":130,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":128,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":124,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":125,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"ellipse\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":127,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"mask\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":129,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":8,\"job_id\":28,\"type\":\"tag\",\"shape_type\":null}],\"severity\":\"error\"}],\"annotations\":{\"valid_count\":0,\"missing_count\":7,\"extra_count\":0,\"total_count\":7,\"ds_count\":0,\"gt_count\":7,\"confusion_matrix\":{\"labels\":[\"dog\",\"cat\",\"skele\",\"unmatched\"],\"rows\":[[0,0,0,0],[0,0,0,0],[0,0,0,0],[0,7,0,0]],\"precision\":[0.0,0.0,0.0,0.0],\"recall\":[0.0,0.0,0.0,0.0],\"accuracy\":[1.0,0.0,1.0,0.0],\"jaccard_index\":[0.0,0.0,0.0,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.0,\"precision\":0.0,\"recall\":0.0},\"annotation_components\":{\"shape\":{\"valid_count\":0,\"missing_count\":6,\"extra_count\":0,\"total_count\":6,\"ds_count\":0,\"gt_count\":6,\"mean_iou\":0.0,\"accuracy\":0.0},\"label\":{\"valid_count\":0,\"invalid_count\":0,\"total_count\":0,\"accuracy\":0.0}},\"conflict_count\":7,\"warning_count\":0,\"error_count\":7,\"conflicts_by_type\":{\"missing_annotation\":7}},\"2\":{\"conflicts\":[],\"annotations\":{\"valid_count\":0,\"missing_count\":0,\"extra_count\":0,\"total_count\":0,\"ds_count\":0,\"gt_count\":0,\"confusion_matrix\":{\"labels\":[\"dog\",\"cat\",\"skele\",\"unmatched\"],\"rows\":[[0,0,0,0],[0,0,0,0],[0,0,0,0],[0,0,0,0]],\"precision\":[0.0,0.0,0.0,0.0],\"recall\":[0.0,0.0,0.0,0.0],\"accuracy\":[0.0,0.0,0.0,0.0],\"jaccard_index\":[0.0,0.0,0.0,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.0,\"precision\":0.0,\"recall\":0.0},\"annotation_components\":{\"shape\":{\"valid_count\":0,\"missing_count\":0,\"extra_count\":0,\"total_count\":0,\"ds_count\":0,\"gt_count\":0,\"mean_iou\":0,\"accuracy\":0.0},\"label\":{\"valid_count\":0,\"invalid_count\":0,\"total_count\":0,\"accuracy\":0.0}},\"conflict_count\":0,\"warning_count\":0,\"error_count\":0,\"conflicts_by_type\":{}}}}" } }, @@ -16231,7 +16243,10 @@ "check_covered_annotations": true, "object_visibility_threshold": 0.05, "panoptic_comparison": true, - "compare_attributes": true + "compare_attributes": true, + "target_metric": "accuracy", + "target_metric_threshold": 0.7, + "max_validations_per_job": 0 } }, { @@ -16250,7 +16265,10 @@ "check_covered_annotations": true, "object_visibility_threshold": 0.05, "panoptic_comparison": true, - "compare_attributes": true + "compare_attributes": true, + "target_metric": "accuracy", + "target_metric_threshold": 0.7, + "max_validations_per_job": 0 } }, { @@ -16269,7 +16287,10 @@ "check_covered_annotations": true, "object_visibility_threshold": 0.05, "panoptic_comparison": true, - "compare_attributes": true + "compare_attributes": true, + "target_metric": "accuracy", + "target_metric_threshold": 0.7, + "max_validations_per_job": 0 } }, { @@ -16288,7 +16309,10 @@ "check_covered_annotations": true, "object_visibility_threshold": 0.05, "panoptic_comparison": true, - "compare_attributes": true + "compare_attributes": true, + "target_metric": "accuracy", + "target_metric_threshold": 0.7, + "max_validations_per_job": 0 } }, { @@ -16307,7 +16331,10 @@ "check_covered_annotations": true, "object_visibility_threshold": 0.05, "panoptic_comparison": true, - "compare_attributes": true + "compare_attributes": true, + "target_metric": "accuracy", + "target_metric_threshold": 0.7, + "max_validations_per_job": 0 } }, { @@ -16326,7 +16353,10 @@ "check_covered_annotations": true, "object_visibility_threshold": 0.05, "panoptic_comparison": true, - "compare_attributes": true + "compare_attributes": true, + "target_metric": "accuracy", + "target_metric_threshold": 0.7, + "max_validations_per_job": 0 } }, { @@ -16345,7 +16375,10 @@ "check_covered_annotations": true, "object_visibility_threshold": 0.05, "panoptic_comparison": true, - "compare_attributes": true + "compare_attributes": true, + "target_metric": "accuracy", + "target_metric_threshold": 0.7, + "max_validations_per_job": 0 } }, { @@ -16364,7 +16397,10 @@ "check_covered_annotations": true, "object_visibility_threshold": 0.05, "panoptic_comparison": true, - "compare_attributes": true + "compare_attributes": true, + "target_metric": "accuracy", + "target_metric_threshold": 0.7, + "max_validations_per_job": 0 } }, { @@ -16383,7 +16419,10 @@ "check_covered_annotations": true, "object_visibility_threshold": 0.05, "panoptic_comparison": true, - "compare_attributes": true + "compare_attributes": true, + "target_metric": "accuracy", + "target_metric_threshold": 0.7, + "max_validations_per_job": 0 } }, { @@ -16402,7 +16441,10 @@ "check_covered_annotations": true, "object_visibility_threshold": 0.05, "panoptic_comparison": true, - "compare_attributes": true + "compare_attributes": true, + "target_metric": "accuracy", + "target_metric_threshold": 0.7, + "max_validations_per_job": 0 } }, { @@ -16421,7 +16463,10 @@ "check_covered_annotations": true, "object_visibility_threshold": 0.05, "panoptic_comparison": true, - "compare_attributes": true + "compare_attributes": true, + "target_metric": "accuracy", + "target_metric_threshold": 0.7, + "max_validations_per_job": 0 } }, { @@ -16440,7 +16485,10 @@ "check_covered_annotations": true, "object_visibility_threshold": 0.05, "panoptic_comparison": true, - "compare_attributes": true + "compare_attributes": true, + "target_metric": "accuracy", + "target_metric_threshold": 0.7, + "max_validations_per_job": 0 } }, { @@ -16459,7 +16507,10 @@ "check_covered_annotations": true, "object_visibility_threshold": 0.05, "panoptic_comparison": true, - "compare_attributes": true + "compare_attributes": true, + "target_metric": "accuracy", + "target_metric_threshold": 0.7, + "max_validations_per_job": 0 } }, { @@ -16478,7 +16529,10 @@ "check_covered_annotations": true, "object_visibility_threshold": 0.05, "panoptic_comparison": true, - "compare_attributes": true + "compare_attributes": true, + "target_metric": "accuracy", + "target_metric_threshold": 0.7, + "max_validations_per_job": 0 } }, { @@ -16497,7 +16551,10 @@ "check_covered_annotations": true, "object_visibility_threshold": 0.05, "panoptic_comparison": true, - "compare_attributes": true + "compare_attributes": true, + "target_metric": "accuracy", + "target_metric_threshold": 0.7, + "max_validations_per_job": 0 } }, { @@ -16516,7 +16573,10 @@ "check_covered_annotations": true, "object_visibility_threshold": 0.05, "panoptic_comparison": true, - "compare_attributes": true + "compare_attributes": true, + "target_metric": "accuracy", + "target_metric_threshold": 0.7, + "max_validations_per_job": 0 } }, { @@ -16535,7 +16595,10 @@ "check_covered_annotations": true, "object_visibility_threshold": 0.05, "panoptic_comparison": true, - "compare_attributes": true + "compare_attributes": true, + "target_metric": "accuracy", + "target_metric_threshold": 0.7, + "max_validations_per_job": 0 } }, { @@ -16554,7 +16617,10 @@ "check_covered_annotations": true, "object_visibility_threshold": 0.05, "panoptic_comparison": true, - "compare_attributes": true + "compare_attributes": true, + "target_metric": "accuracy", + "target_metric_threshold": 0.7, + "max_validations_per_job": 0 } }, { @@ -16573,7 +16639,10 @@ "check_covered_annotations": true, "object_visibility_threshold": 0.05, "panoptic_comparison": true, - "compare_attributes": true + "compare_attributes": true, + "target_metric": "accuracy", + "target_metric_threshold": 0.7, + "max_validations_per_job": 0 } }, { @@ -16592,7 +16661,10 @@ "check_covered_annotations": true, "object_visibility_threshold": 0.05, "panoptic_comparison": true, - "compare_attributes": true + "compare_attributes": true, + "target_metric": "accuracy", + "target_metric_threshold": 0.7, + "max_validations_per_job": 0 } }, { diff --git a/tests/python/shared/assets/quality_reports.json b/tests/python/shared/assets/quality_reports.json index b1fa8173dfe..64ed156e6da 100644 --- a/tests/python/shared/assets/quality_reports.json +++ b/tests/python/shared/assets/quality_reports.json @@ -11,6 +11,7 @@ "job_id": null, "parent_id": null, "summary": { + "accuracy": 0.4883720930232558, "conflict_count": 37, "conflicts_by_type": { "covered_annotation": 1, @@ -27,6 +28,8 @@ "frame_count": 3, "frame_share": 0.2727272727272727, "gt_count": 33, + "precision": 0.6176470588235294, + "recall": 0.6363636363636364, "total_count": 43, "valid_count": 21, "warning_count": 15 @@ -43,6 +46,7 @@ "job_id": 27, "parent_id": 1, "summary": { + "accuracy": 0.4883720930232558, "conflict_count": 37, "conflicts_by_type": { "covered_annotation": 1, @@ -59,6 +63,8 @@ "frame_count": 3, "frame_share": 0.2727272727272727, "gt_count": 33, + "precision": 0.6176470588235294, + "recall": 0.6363636363636364, "total_count": 43, "valid_count": 21, "warning_count": 15 @@ -75,6 +81,7 @@ "job_id": null, "parent_id": null, "summary": { + "accuracy": 0.4782608695652174, "conflict_count": 40, "conflicts_by_type": { "covered_annotation": 1, @@ -91,6 +98,8 @@ "frame_count": 3, "frame_share": 0.2727272727272727, "gt_count": 35, + "precision": 0.6111111111111112, + "recall": 0.6285714285714286, "total_count": 46, "valid_count": 22, "warning_count": 16 @@ -107,6 +116,7 @@ "job_id": 27, "parent_id": 3, "summary": { + "accuracy": 0.4782608695652174, "conflict_count": 40, "conflicts_by_type": { "covered_annotation": 1, @@ -123,6 +133,8 @@ "frame_count": 3, "frame_share": 0.2727272727272727, "gt_count": 35, + "precision": 0.6111111111111112, + "recall": 0.6285714285714286, "total_count": 46, "valid_count": 22, "warning_count": 16 @@ -139,6 +151,7 @@ "job_id": null, "parent_id": null, "summary": { + "accuracy": 0.4782608695652174, "conflict_count": 40, "conflicts_by_type": { "covered_annotation": 1, @@ -155,6 +168,8 @@ "frame_count": 3, "frame_share": 0.2727272727272727, "gt_count": 35, + "precision": 0.6111111111111112, + "recall": 0.6285714285714286, "total_count": 46, "valid_count": 22, "warning_count": 16 @@ -171,6 +186,7 @@ "job_id": 27, "parent_id": 5, "summary": { + "accuracy": 0.4782608695652174, "conflict_count": 40, "conflicts_by_type": { "covered_annotation": 1, @@ -187,6 +203,8 @@ "frame_count": 3, "frame_share": 0.2727272727272727, "gt_count": 35, + "precision": 0.6111111111111112, + "recall": 0.6285714285714286, "total_count": 46, "valid_count": 22, "warning_count": 16 @@ -203,6 +221,7 @@ "job_id": null, "parent_id": null, "summary": { + "accuracy": 0.5, "conflict_count": 3, "conflicts_by_type": { "extra_annotation": 1, @@ -214,6 +233,8 @@ "frame_count": 3, "frame_share": 0.2727272727272727, "gt_count": 3, + "precision": 0.6666666666666666, + "recall": 0.6666666666666666, "total_count": 4, "valid_count": 2, "warning_count": 1 @@ -230,6 +251,7 @@ "job_id": 31, "parent_id": 7, "summary": { + "accuracy": 0.0, "conflict_count": 0, "conflicts_by_type": {}, "ds_count": 0, @@ -237,6 +259,8 @@ "frame_count": 0, "frame_share": 0.0, "gt_count": 0, + "precision": 0.0, + "recall": 0.0, "total_count": 0, "valid_count": 0, "warning_count": 0 @@ -253,6 +277,7 @@ "job_id": 30, "parent_id": 7, "summary": { + "accuracy": 1.0, "conflict_count": 1, "conflicts_by_type": { "low_overlap": 1 @@ -262,6 +287,8 @@ "frame_count": 2, "frame_share": 0.4, "gt_count": 2, + "precision": 1.0, + "recall": 1.0, "total_count": 2, "valid_count": 2, "warning_count": 1 @@ -278,6 +305,7 @@ "job_id": 29, "parent_id": 7, "summary": { + "accuracy": 0.0, "conflict_count": 2, "conflicts_by_type": { "extra_annotation": 1, @@ -288,6 +316,8 @@ "frame_count": 1, "frame_share": 0.2, "gt_count": 1, + "precision": 0.0, + "recall": 0.0, "total_count": 2, "valid_count": 0, "warning_count": 0 @@ -304,6 +334,7 @@ "job_id": null, "parent_id": null, "summary": { + "accuracy": 0.4583333333333333, "conflict_count": 42, "conflicts_by_type": { "covered_annotation": 1, @@ -320,6 +351,8 @@ "frame_count": 3, "frame_share": 0.2727272727272727, "gt_count": 37, + "precision": 0.6111111111111112, + "recall": 0.5945945945945946, "total_count": 48, "valid_count": 22, "warning_count": 16 @@ -336,6 +369,7 @@ "job_id": 27, "parent_id": 11, "summary": { + "accuracy": 0.4583333333333333, "conflict_count": 42, "conflicts_by_type": { "covered_annotation": 1, @@ -352,6 +386,8 @@ "frame_count": 3, "frame_share": 0.2727272727272727, "gt_count": 37, + "precision": 0.6111111111111112, + "recall": 0.5945945945945946, "total_count": 48, "valid_count": 22, "warning_count": 16 diff --git a/tests/python/shared/assets/quality_settings.json b/tests/python/shared/assets/quality_settings.json index e23c786e737..e38feba4b2e 100644 --- a/tests/python/shared/assets/quality_settings.json +++ b/tests/python/shared/assets/quality_settings.json @@ -14,9 +14,12 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, "panoptic_comparison": true, + "target_metric": "accuracy", + "target_metric_threshold": 0.7, "task_id": 2 }, { @@ -30,9 +33,12 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, "panoptic_comparison": true, + "target_metric": "accuracy", + "target_metric_threshold": 0.7, "task_id": 5 }, { @@ -46,9 +52,12 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, "panoptic_comparison": true, + "target_metric": "accuracy", + "target_metric_threshold": 0.7, "task_id": 6 }, { @@ -62,9 +71,12 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, "panoptic_comparison": true, + "target_metric": "accuracy", + "target_metric_threshold": 0.7, "task_id": 7 }, { @@ -78,9 +90,12 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, "panoptic_comparison": true, + "target_metric": "accuracy", + "target_metric_threshold": 0.7, "task_id": 8 }, { @@ -94,9 +109,12 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, "panoptic_comparison": true, + "target_metric": "accuracy", + "target_metric_threshold": 0.7, "task_id": 9 }, { @@ -110,9 +128,12 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, "panoptic_comparison": true, + "target_metric": "accuracy", + "target_metric_threshold": 0.7, "task_id": 11 }, { @@ -126,9 +147,12 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, "panoptic_comparison": true, + "target_metric": "accuracy", + "target_metric_threshold": 0.7, "task_id": 12 }, { @@ -142,9 +166,12 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, "panoptic_comparison": true, + "target_metric": "accuracy", + "target_metric_threshold": 0.7, "task_id": 13 }, { @@ -158,9 +185,12 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, "panoptic_comparison": true, + "target_metric": "accuracy", + "target_metric_threshold": 0.7, "task_id": 14 }, { @@ -174,9 +204,12 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, "panoptic_comparison": true, + "target_metric": "accuracy", + "target_metric_threshold": 0.7, "task_id": 15 }, { @@ -190,9 +223,12 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, "panoptic_comparison": true, + "target_metric": "accuracy", + "target_metric_threshold": 0.7, "task_id": 17 }, { @@ -206,9 +242,12 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, "panoptic_comparison": true, + "target_metric": "accuracy", + "target_metric_threshold": 0.7, "task_id": 18 }, { @@ -222,9 +261,12 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, "panoptic_comparison": true, + "target_metric": "accuracy", + "target_metric_threshold": 0.7, "task_id": 19 }, { @@ -238,9 +280,12 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, "panoptic_comparison": true, + "target_metric": "accuracy", + "target_metric_threshold": 0.7, "task_id": 20 }, { @@ -254,9 +299,12 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, "panoptic_comparison": true, + "target_metric": "accuracy", + "target_metric_threshold": 0.7, "task_id": 21 }, { @@ -270,9 +318,12 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, "panoptic_comparison": true, + "target_metric": "accuracy", + "target_metric_threshold": 0.7, "task_id": 22 }, { @@ -286,9 +337,12 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, "panoptic_comparison": true, + "target_metric": "accuracy", + "target_metric_threshold": 0.7, "task_id": 23 }, { @@ -302,9 +356,12 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, "panoptic_comparison": true, + "target_metric": "accuracy", + "target_metric_threshold": 0.7, "task_id": 24 }, { @@ -318,9 +375,12 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, "panoptic_comparison": true, + "target_metric": "accuracy", + "target_metric_threshold": 0.7, "task_id": 25 } ] From ac05a6d03e044ecdd872c4c3ec0da9fd6169386a Mon Sep 17 00:00:00 2001 From: Dmitrii Lavrukhin Date: Fri, 23 Aug 2024 15:05:03 +0400 Subject: [PATCH 18/22] YOLOv8 support (#8240) ## Summary by CodeRabbit - **New Features** - Introduced support for YOLOv8 formats, enhancing object detection capabilities. - Added new export and import functions for YOLOv8 formats within the dataset manager. - Expanded documentation to cover YOLOv8 format specifications and export processes. - **Bug Fixes** - Improved handling of various YOLOv8 annotation formats to ensure accurate processing. - **Tests** - Enhanced test coverage for YOLOv8 formats in both dataset export/import and REST API functionalities. - **Documentation** - Updated existing links in the YOLO format documentation for clarity. - Added new documentation detailing YOLOv8 formats and their export processes. Co-authored-by: Roman Donchenko --- ..._160939_dmitrii.lavrukhin_yolo8_support.md | 4 + cvat/apps/dataset_manager/formats/yolo.py | 110 +++++++++++++-- cvat/apps/dataset_manager/project.py | 2 +- .../tests/assets/annotations.json | 125 +++++++++++++++++ .../dataset_manager/tests/assets/tasks.json | 41 ++++++ .../dataset_manager/tests/test_formats.py | 14 +- .../tests/test_rest_api_formats.py | 81 ++++++++--- cvat/apps/engine/tests/test_rest_api.py | 11 +- cvat/requirements/base.in | 2 +- cvat/requirements/base.txt | 10 +- cvat/requirements/production.txt | 2 +- .../manual/advanced/formats/format-yolov8.md | 128 ++++++++++++++++++ tests/python/rest_api/test_projects.py | 5 + tests/python/rest_api/test_tasks.py | 5 + 14 files changed, 495 insertions(+), 45 deletions(-) create mode 100644 changelog.d/20240730_160939_dmitrii.lavrukhin_yolo8_support.md create mode 100644 site/content/en/docs/manual/advanced/formats/format-yolov8.md diff --git a/changelog.d/20240730_160939_dmitrii.lavrukhin_yolo8_support.md b/changelog.d/20240730_160939_dmitrii.lavrukhin_yolo8_support.md new file mode 100644 index 00000000000..4d4e91f321b --- /dev/null +++ b/changelog.d/20240730_160939_dmitrii.lavrukhin_yolo8_support.md @@ -0,0 +1,4 @@ +### Added + +- Added support for YOLOv8 formats + () diff --git a/cvat/apps/dataset_manager/formats/yolo.py b/cvat/apps/dataset_manager/formats/yolo.py index 9f0e4655811..a8c8177b055 100644 --- a/cvat/apps/dataset_manager/formats/yolo.py +++ b/cvat/apps/dataset_manager/formats/yolo.py @@ -2,36 +2,54 @@ # Copyright (C) 2023-2024 CVAT.ai Corporation # # SPDX-License-Identifier: MIT - import os.path as osp from glob import glob from pyunpack import Archive -from cvat.apps.dataset_manager.bindings import (GetCVATDataExtractor, detect_dataset, - import_dm_annotations, match_dm_item, find_dataset_root) +from cvat.apps.dataset_manager.bindings import ( + GetCVATDataExtractor, + detect_dataset, + import_dm_annotations, + match_dm_item, + find_dataset_root, +) from cvat.apps.dataset_manager.util import make_zip_archive +from datumaro.components.annotation import AnnotationType from datumaro.components.extractor import DatasetItem from datumaro.components.project import Dataset -from datumaro.plugins.yolo_format.extractor import YoloExtractor from .registry import dm_env, exporter, importer -@exporter(name='YOLO', ext='ZIP', version='1.1') -def _export(dst_file, temp_dir, instance_data, save_images=False): +def _export_common(dst_file, temp_dir, instance_data, format_name, *, save_images=False): with GetCVATDataExtractor(instance_data, include_images=save_images) as extractor: dataset = Dataset.from_extractors(extractor, env=dm_env) - dataset.export(temp_dir, 'yolo', save_images=save_images) + dataset.export(temp_dir, format_name, save_images=save_images) make_zip_archive(temp_dir, dst_file) -@importer(name='YOLO', ext='ZIP', version='1.1') -def _import(src_file, temp_dir, instance_data, load_data_callback=None, **kwargs): + +@exporter(name='YOLO', ext='ZIP', version='1.1') +def _export_yolo(*args, **kwargs): + _export_common(*args, format_name='yolo', **kwargs) + + +def _import_common( + src_file, + temp_dir, + instance_data, + format_name, + *, + load_data_callback=None, + import_kwargs=None, + **kwargs +): Archive(src_file.name).extractall(temp_dir) image_info = {} - frames = [YoloExtractor.name_from_path(osp.relpath(p, temp_dir)) + extractor = dm_env.extractors.get(format_name) + frames = [extractor.name_from_path(osp.relpath(p, temp_dir)) for p in glob(osp.join(temp_dir, '**', '*.txt'), recursive=True)] root_hint = find_dataset_root( [DatasetItem(id=frame) for frame in frames], instance_data) @@ -44,11 +62,75 @@ def _import(src_file, temp_dir, instance_data, load_data_callback=None, **kwargs except Exception: # nosec pass if frame_info is not None: - image_info[frame] = (frame_info['height'], frame_info['width']) + image_info[frame] = (frame_info["height"], frame_info["width"]) - detect_dataset(temp_dir, format_name='yolo', importer=dm_env.importers.get('yolo')) - dataset = Dataset.import_from(temp_dir, 'yolo', - env=dm_env, image_info=image_info) + detect_dataset(temp_dir, format_name=format_name, importer=dm_env.importers.get(format_name)) + dataset = Dataset.import_from(temp_dir, format_name, + env=dm_env, image_info=image_info, **(import_kwargs or {})) if load_data_callback is not None: load_data_callback(dataset, instance_data) import_dm_annotations(dataset, instance_data) + + +@importer(name='YOLO', ext='ZIP', version='1.1') +def _import_yolo(*args, **kwargs): + _import_common(*args, format_name="yolo", **kwargs) + + +@exporter(name='YOLOv8 Detection', ext='ZIP', version='1.0') +def _export_yolov8_detection(*args, **kwargs): + _export_common(*args, format_name='yolov8_detection', **kwargs) + + +@exporter(name='YOLOv8 Oriented Bounding Boxes', ext='ZIP', version='1.0') +def _export_yolov8_oriented_boxes(*args, **kwargs): + _export_common(*args, format_name='yolov8_oriented_boxes', **kwargs) + + +@exporter(name='YOLOv8 Segmentation', ext='ZIP', version='1.0') +def _export_yolov8_segmentation(dst_file, temp_dir, instance_data, *, save_images=False): + with GetCVATDataExtractor(instance_data, include_images=save_images) as extractor: + dataset = Dataset.from_extractors(extractor, env=dm_env) + dataset = dataset.transform('masks_to_polygons') + dataset.export(temp_dir, 'yolov8_segmentation', save_images=save_images) + + make_zip_archive(temp_dir, dst_file) + + +@exporter(name='YOLOv8 Pose', ext='ZIP', version='1.0') +def _export_yolov8_pose(*args, **kwargs): + _export_common(*args, format_name='yolov8_pose', **kwargs) + + +@importer(name='YOLOv8 Detection', ext="ZIP", version="1.0") +def _import_yolov8_detection(*args, **kwargs): + _import_common(*args, format_name="yolov8_detection", **kwargs) + + +@importer(name='YOLOv8 Segmentation', ext="ZIP", version="1.0") +def _import_yolov8_segmentation(*args, **kwargs): + _import_common(*args, format_name="yolov8_segmentation", **kwargs) + + +@importer(name='YOLOv8 Oriented Bounding Boxes', ext="ZIP", version="1.0") +def _import_yolov8_oriented_boxes(*args, **kwargs): + _import_common(*args, format_name="yolov8_oriented_boxes", **kwargs) + + +@importer(name='YOLOv8 Pose', ext="ZIP", version="1.0") +def _import_yolov8_pose(src_file, temp_dir, instance_data, **kwargs): + with GetCVATDataExtractor(instance_data) as extractor: + point_categories = extractor.categories().get(AnnotationType.points) + label_categories = extractor.categories().get(AnnotationType.label) + true_skeleton_point_labels = { + label_categories[label_id].name: category.labels + for label_id, category in point_categories.items.items() + } + _import_common( + src_file, + temp_dir, + instance_data, + format_name="yolov8_pose", + import_kwargs=dict(skeleton_sub_labels=true_skeleton_point_labels), + **kwargs + ) diff --git a/cvat/apps/dataset_manager/project.py b/cvat/apps/dataset_manager/project.py index 7579f241043..977b1fbad3b 100644 --- a/cvat/apps/dataset_manager/project.py +++ b/cvat/apps/dataset_manager/project.py @@ -158,7 +158,7 @@ def import_dataset(self, dataset_file, importer, **options): os.makedirs(temp_dir_base, exist_ok=True) with TemporaryDirectory(dir=temp_dir_base) as temp_dir: try: - importer(dataset_file, temp_dir, project_data, self.load_dataset_data, **options) + importer(dataset_file, temp_dir, project_data, load_data_callback=self.load_dataset_data, **options) except (DatasetNotFoundError, CvatDatasetNotFoundError) as not_found: if settings.CVAT_LOG_IMPORT_ERRORS: dlogger.log_import_error( diff --git a/cvat/apps/dataset_manager/tests/assets/annotations.json b/cvat/apps/dataset_manager/tests/assets/annotations.json index 6035e40fbd3..9dd05415b14 100644 --- a/cvat/apps/dataset_manager/tests/assets/annotations.json +++ b/cvat/apps/dataset_manager/tests/assets/annotations.json @@ -719,6 +719,131 @@ ], "tracks": [] }, + "YOLOv8 Detection 1.0": { + "version": 0, + "tags": [], + "shapes": [ + { + "type": "rectangle", + "occluded": false, + "z_order": 0, + "points": [8.3, 9.1, 19.2, 14.8], + "frame": 0, + "label_id": null, + "group": 0, + "source": "manual", + "attributes": [] + } + ], + "tracks": [] + }, + "YOLOv8 Oriented Bounding Boxes 1.0": { + "version": 0, + "tags": [], + "shapes": [ + { + "type": "rectangle", + "occluded": false, + "z_order": 0, + "points": [8.3, 9.1, 19.2, 14.8], + "frame": 0, + "label_id": null, + "group": 0, + "source": "manual", + "rotation": 30.0, + "attributes": [] + } + ], + "tracks": [] + }, + "YOLOv8 Segmentation 1.0": { + "version": 0, + "tags": [], + "shapes": [ + { + "type": "polygon", + "occluded": false, + "z_order": 0, + "points": [25.04, 13.7, 35.85, 20.2, 16.65, 19.8], + "frame": 0, + "label_id": null, + "group": 0, + "source": "manual", + "attributes": [] + } + ], + "tracks": [] + }, + "YOLOv8 Pose 1.0": { + "version": 0, + "tags": [], + "shapes": [ + { + "type": "skeleton", + "occluded": false, + "outside": false, + "z_order": 0, + "rotation": 0, + "points": [], + "frame": 0, + "label_id": 0, + "group": 0, + "source": "manual", + "attributes": [], + "elements": [ + { + "type": "points", + "occluded": false, + "outside": true, + "z_order": 0, + "rotation": 0, + "points": [ + 223.02, + 72.83 + ], + "frame": 0, + "label_id": 1, + "group": 0, + "source": "manual", + "attributes": [] + }, + { + "type": "points", + "occluded": false, + "outside": false, + "z_order": 0, + "rotation": 0, + "points": [ + 232.98, + 124.6 + ], + "frame": 0, + "label_id": 2, + "group": 0, + "source": "manual", + "attributes": [] + }, + { + "type": "points", + "occluded": false, + "outside": false, + "z_order": 0, + "rotation": 0, + "points": [ + 281.22, + 36.63 + ], + "frame": 0, + "label_id": 3, + "group": 0, + "source": "manual", + "attributes": [] + } + ] + } + ], + "tracks": [] + }, "VGGFace2 1.0": { "version": 0, "tags": [], diff --git a/cvat/apps/dataset_manager/tests/assets/tasks.json b/cvat/apps/dataset_manager/tests/assets/tasks.json index ce9b2dd8c6a..0bba5ef07d8 100644 --- a/cvat/apps/dataset_manager/tests/assets/tasks.json +++ b/cvat/apps/dataset_manager/tests/assets/tasks.json @@ -592,5 +592,46 @@ "svg": "" } ] + }, + "YOLOv8 Pose 1.0": { + "name": "YOLOv8 pose task", + "overlap": 0, + "segment_size": 100, + "labels": [ + { + "name": "skeleton", + "color": "#2080c0", + "type": "skeleton", + "attributes": [ + { + "name": "attr", + "mutable": false, + "input_type": "select", + "values": ["0", "1", "2"] + } + ], + "sublabels": [ + { + "name": "1", + "color": "#d12345", + "attributes": [], + "type": "points" + }, + { + "name": "2", + "color": "#350dea", + "attributes": [], + "type": "points" + }, + { + "name": "3", + "color": "#479ffe", + "attributes": [], + "type": "points" + } + ], + "svg": "" + } + ] } } diff --git a/cvat/apps/dataset_manager/tests/test_formats.py b/cvat/apps/dataset_manager/tests/test_formats.py index 1c7db60814d..6a03e41c8aa 100644 --- a/cvat/apps/dataset_manager/tests/test_formats.py +++ b/cvat/apps/dataset_manager/tests/test_formats.py @@ -309,7 +309,11 @@ def test_export_formats_query(self): 'KITTI 1.0', 'LFW 1.0', 'Cityscapes 1.0', - 'Open Images V6 1.0' + 'Open Images V6 1.0', + 'YOLOv8 Oriented Bounding Boxes 1.0', + 'YOLOv8 Detection 1.0', + 'YOLOv8 Pose 1.0', + 'YOLOv8 Segmentation 1.0', }) def test_import_formats_query(self): @@ -342,6 +346,10 @@ def test_import_formats_query(self): 'Open Images V6 1.0', 'Datumaro 1.0', 'Datumaro 3D 1.0', + 'YOLOv8 Oriented Bounding Boxes 1.0', + 'YOLOv8 Detection 1.0', + 'YOLOv8 Pose 1.0', + 'YOLOv8 Segmentation 1.0', }) def test_exports(self): @@ -391,6 +399,10 @@ def test_empty_images_are_exported(self): # ('KITTI 1.0', 'kitti') format does not support empty annotations ('LFW 1.0', 'lfw'), # ('Cityscapes 1.0', 'cityscapes'), does not support, empty annotations + ('YOLOv8 Oriented Bounding Boxes 1.0', 'yolov8_oriented_boxes'), + ('YOLOv8 Detection 1.0', 'yolov8_detection'), + ('YOLOv8 Pose 1.0', 'yolov8_pose'), + ('YOLOv8 Segmentation 1.0', 'yolov8_segmentation'), ]: with self.subTest(format=format_name): if not dm.formats.registry.EXPORT_FORMATS[format_name].ENABLED: diff --git a/cvat/apps/dataset_manager/tests/test_rest_api_formats.py b/cvat/apps/dataset_manager/tests/test_rest_api_formats.py index 767fb07fe96..26c7a7f30ef 100644 --- a/cvat/apps/dataset_manager/tests/test_rest_api_formats.py +++ b/cvat/apps/dataset_manager/tests/test_rest_api_formats.py @@ -25,8 +25,10 @@ from unittest.mock import MagicMock, patch, DEFAULT as MOCK_DEFAULT from attr import define, field +from datumaro import AnnotationType, Points from datumaro.components.dataset import Dataset -from datumaro.util.test_utils import compare_datasets, TestDir +from datumaro.components.operations import ExactComparator +from datumaro.util.test_utils import TestDir from django.contrib.auth.models import Group, User from PIL import Image from rest_framework import status @@ -97,6 +99,29 @@ def generate_video_file(filename, width=1280, height=720, duration=1, fps=25, co return [(width, height)] * total_frames, f +def compare_datasets(expected: Dataset, actual: Dataset): + # we need this function to allow for a bit of variation in a rotation attribute and in skeleton elements order + comparator = ExactComparator(ignored_fields=["elements"], ignored_attrs=["rotation"]) + _, unmatched, expected_extra, actual_extra, errors = comparator.compare_datasets( + expected, actual + ) + assert not unmatched, f"Datasets have unmatched items: {unmatched}" + assert not actual_extra, f"Actual has following extra items: {actual_extra}" + assert not expected_extra, f"Expected has following extra items: {expected_extra}" + assert not errors, f"There were following errors while comparing datasets: {errors}" + + for item_a, item_b in zip(expected, actual): + for ann_a, ann_b in zip(item_a.annotations, item_b.annotations): + assert ( + abs(ann_a.attributes.get("rotation", 0) - ann_b.attributes.get("rotation", 0)) + < 0.01 + ) + if ann_a.type == AnnotationType.skeleton: + elements_a = sorted(filter(lambda p: p.visibility[0] != Points.Visibility.absent, ann_a.elements), key=lambda s: s.label) + elements_b = sorted(filter(lambda p: p.visibility[0] != Points.Visibility.absent, ann_b.elements), key=lambda s: s.label) + assert elements_a == elements_b + + class _DbTestBase(ApiTestBase): @classmethod def setUpTestData(cls): @@ -399,7 +424,8 @@ def test_api_v2_dump_and_upload_annotations_with_objects_type_is_shape(self): if dump_format_name in [ "Cityscapes 1.0", "COCO Keypoints 1.0", "ICDAR Localization 1.0", "ICDAR Recognition 1.0", - "ICDAR Segmentation 1.0", "Market-1501 1.0", "MOT 1.1" + "ICDAR Segmentation 1.0", "Market-1501 1.0", "MOT 1.1", + "YOLOv8 Pose 1.0", ]: task = self._create_task(tasks[dump_format_name], images) else: @@ -410,7 +436,9 @@ def test_api_v2_dump_and_upload_annotations_with_objects_type_is_shape(self): "ImageNet 1.0", "MOTS PNG 1.0", "PASCAL VOC 1.1", "Segmentation mask 1.1", "VGGFace2 1.0", - "WiderFace 1.0", "YOLO 1.1" + "WiderFace 1.0", "YOLO 1.1", + "YOLOv8 Detection 1.0", "YOLOv8 Segmentation 1.0", + "YOLOv8 Oriented Bounding Boxes 1.0", "YOLOv8 Pose 1.0", ]: self._create_annotations(task, dump_format_name, "default") else: @@ -463,7 +491,8 @@ def test_api_v2_dump_and_upload_annotations_with_objects_type_is_shape(self): if upload_format_name in [ "Cityscapes 1.0", "COCO Keypoints 1.0", "ICDAR Localization 1.0", "ICDAR Recognition 1.0", - "ICDAR Segmentation 1.0", "Market-1501 1.0", "MOT 1.1" + "ICDAR Segmentation 1.0", "Market-1501 1.0", "MOT 1.1", + "YOLOv8 Pose 1.0", ]: task = self._create_task(tasks[upload_format_name], images) else: @@ -506,7 +535,8 @@ def test_api_v2_dump_annotations_with_objects_type_is_track(self): if dump_format_name in [ "Cityscapes 1.0", "COCO Keypoints 1.0", "ICDAR Localization 1.0", "ICDAR Recognition 1.0", - "ICDAR Segmentation 1.0", "Market-1501 1.0", "MOT 1.1" + "ICDAR Segmentation 1.0", "Market-1501 1.0", "MOT 1.1", + "YOLOv8 Pose 1.0", ]: task = self._create_task(tasks[dump_format_name], video) else: @@ -517,7 +547,9 @@ def test_api_v2_dump_annotations_with_objects_type_is_track(self): "Cityscapes 1.0", "ImageNet 1.0", "MOTS PNG 1.0", "PASCAL VOC 1.1", "Segmentation mask 1.1", - "VGGFace2 1.0", "WiderFace 1.0", "YOLO 1.1" + "VGGFace2 1.0", "WiderFace 1.0", "YOLO 1.1", + "YOLOv8 Detection 1.0", "YOLOv8 Segmentation 1.0", + "YOLOv8 Oriented Bounding Boxes 1.0", "YOLOv8 Pose 1.0", ]: self._create_annotations(task, dump_format_name, "default") else: @@ -569,7 +601,8 @@ def test_api_v2_dump_annotations_with_objects_type_is_track(self): if upload_format_name in [ "Cityscapes 1.0", "COCO Keypoints 1.0", "ICDAR Localization 1.0", "ICDAR Recognition 1.0", - "ICDAR Segmentation 1.0", "Market-1501 1.0", "MOT 1.1" + "ICDAR Segmentation 1.0", "Market-1501 1.0", "MOT 1.1", + "YOLOv8 Pose 1.0", ]: task = self._create_task(tasks[upload_format_name], video) else: @@ -848,7 +881,8 @@ def test_api_v2_export_dataset(self): if dump_format_name in [ "Cityscapes 1.0", "COCO Keypoints 1.0", "ICDAR Localization 1.0", "ICDAR Recognition 1.0", - "ICDAR Segmentation 1.0", "Market-1501 1.0", "MOT 1.1" + "ICDAR Segmentation 1.0", "Market-1501 1.0", "MOT 1.1", + "YOLOv8 Pose 1.0", ]: task = self._create_task(tasks[dump_format_name], images) else: @@ -951,8 +985,8 @@ def test_api_v2_rewriting_annotations(self): images = self._generate_task_images(3) if dump_format_name in [ "Market-1501 1.0", - "ICDAR Localization 1.0", "ICDAR Recognition 1.0", \ - "ICDAR Segmentation 1.0", "COCO Keypoints 1.0", + "ICDAR Localization 1.0", "ICDAR Recognition 1.0", + "ICDAR Segmentation 1.0", "COCO Keypoints 1.0", "YOLOv8 Pose 1.0", ]: task = self._create_task(tasks[dump_format_name], images) else: @@ -963,7 +997,9 @@ def test_api_v2_rewriting_annotations(self): "MOT 1.1", "PASCAL VOC 1.1", "Segmentation mask 1.1", "YOLO 1.1", "ImageNet 1.0", "WiderFace 1.0", "VGGFace2 1.0", - "Datumaro 1.0", "Open Images V6 1.0", "KITTI 1.0" + "Datumaro 1.0", "Open Images V6 1.0", "KITTI 1.0", + "YOLOv8 Detection 1.0", "YOLOv8 Segmentation 1.0", + "YOLOv8 Oriented Bounding Boxes 1.0", "YOLOv8 Pose 1.0", ]: self._create_annotations(task, dump_format_name, "default") else: @@ -1034,7 +1070,7 @@ def test_api_v2_tasks_annotations_dump_and_upload_many_jobs_with_datumaro(self): # equals annotations data_from_task_after_upload = self._get_data_from_task(task_id, include_images) - compare_datasets(self, data_from_task_before_upload, data_from_task_after_upload) + compare_datasets(data_from_task_before_upload, data_from_task_after_upload) def test_api_v2_tasks_annotations_dump_and_upload_with_datumaro(self): test_name = self._testMethodName @@ -1063,9 +1099,10 @@ def test_api_v2_tasks_annotations_dump_and_upload_with_datumaro(self): # create task images = self._generate_task_images(3) if dump_format_name in [ - "Market-1501 1.0", "Cityscapes 1.0", \ - "ICDAR Localization 1.0", "ICDAR Recognition 1.0", \ - "ICDAR Segmentation 1.0", "COCO Keypoints 1.0" + "Market-1501 1.0", "Cityscapes 1.0", + "ICDAR Localization 1.0", "ICDAR Recognition 1.0", + "ICDAR Segmentation 1.0", "COCO Keypoints 1.0", + "YOLOv8 Pose 1.0", ]: task = self._create_task(tasks[dump_format_name], images) else: @@ -1077,7 +1114,9 @@ def test_api_v2_tasks_annotations_dump_and_upload_with_datumaro(self): "PASCAL VOC 1.1", "Segmentation mask 1.1", "YOLO 1.1", "ImageNet 1.0", "WiderFace 1.0", "VGGFace2 1.0", "LFW 1.0", - "Open Images V6 1.0", "Datumaro 1.0", "KITTI 1.0" + "Open Images V6 1.0", "Datumaro 1.0", "KITTI 1.0", + "YOLOv8 Detection 1.0", "YOLOv8 Segmentation 1.0", + "YOLOv8 Oriented Bounding Boxes 1.0", "YOLOv8 Pose 1.0", ]: self._create_annotations(task, dump_format_name, "default") else: @@ -1110,7 +1149,7 @@ def test_api_v2_tasks_annotations_dump_and_upload_with_datumaro(self): # equals annotations data_from_task_after_upload = self._get_data_from_task(task_id, include_images) - compare_datasets(self, data_from_task_before_upload, data_from_task_after_upload) + compare_datasets(data_from_task_before_upload, data_from_task_after_upload) def test_api_v2_check_duplicated_polygon_points(self): test_name = self._testMethodName @@ -1176,7 +1215,7 @@ def test_api_v2_check_widerface_with_all_attributes(self): # equals annotations data_from_task_after_upload = self._get_data_from_task(task_id, include_images) - compare_datasets(self, data_from_task_before_upload, data_from_task_after_upload) + compare_datasets(data_from_task_before_upload, data_from_task_after_upload) def test_api_v2_check_mot_with_shapes_only(self): test_name = self._testMethodName @@ -1212,7 +1251,7 @@ def test_api_v2_check_mot_with_shapes_only(self): # equals annotations data_from_task_after_upload = self._get_data_from_task(task_id, include_images) - compare_datasets(self, data_from_task_before_upload, data_from_task_after_upload) + compare_datasets(data_from_task_before_upload, data_from_task_after_upload) def test_api_v2_check_attribute_import_in_tracks(self): test_name = self._testMethodName @@ -1249,7 +1288,7 @@ def test_api_v2_check_attribute_import_in_tracks(self): # equals annotations data_from_task_after_upload = self._get_data_from_task(task_id, include_images) - compare_datasets(self, data_from_task_before_upload, data_from_task_after_upload) + compare_datasets(data_from_task_before_upload, data_from_task_after_upload) class ExportBehaviorTest(_DbTestBase): @define @@ -2051,7 +2090,7 @@ def test_api_v2_export_import_dataset(self): "Cityscapes 1.0", "Datumaro 1.0", "ImageNet 1.0", "MOT 1.1", "MOTS PNG 1.0", "PASCAL VOC 1.1", "Segmentation mask 1.1", "VGGFace2 1.0", - "WiderFace 1.0", "YOLO 1.1" + "WiderFace 1.0", "YOLO 1.1", "YOLOv8 Detection 1.0", ]: self._create_annotations(task, dump_format_name, "default") else: diff --git a/cvat/apps/engine/tests/test_rest_api.py b/cvat/apps/engine/tests/test_rest_api.py index 47758be11d1..ae0200b6a2a 100644 --- a/cvat/apps/engine/tests/test_rest_api.py +++ b/cvat/apps/engine/tests/test_rest_api.py @@ -6114,6 +6114,15 @@ def _get_initial_annotation(annotation_format): elif annotation_format == "YOLO 1.1": annotations["shapes"] = rectangle_shapes_wo_attrs + elif annotation_format == "YOLOv8 Detection 1.0": + annotations["shapes"] = rectangle_shapes_wo_attrs + + elif annotation_format == "YOLOv8 Oriented Bounding Boxes 1.0": + annotations["shapes"] = rectangle_shapes_wo_attrs + + elif annotation_format == "YOLOv8 Segmentation 1.0": + annotations["shapes"] = polygon_shapes_wo_attrs + elif annotation_format == "COCO 1.0": annotations["shapes"] = polygon_shapes_wo_attrs @@ -6471,7 +6480,7 @@ def etree_to_dict(t): self.assertEqual(meta["task"]["name"], task["name"]) elif format_name == "PASCAL VOC 1.1": self.assertTrue(zipfile.is_zipfile(content)) - elif format_name == "YOLO 1.1": + elif format_name in ["YOLO 1.1", "YOLOv8 Detection 1.0", "YOLOv8 Segmentation 1.0", "YOLOv8 Oriented Bounding Boxes 1.0", "YOLOv8 Pose 1.0"]: self.assertTrue(zipfile.is_zipfile(content)) elif format_name in ['Kitti Raw Format 1.0','Sly Point Cloud Format 1.0']: self.assertTrue(zipfile.is_zipfile(content)) diff --git a/cvat/requirements/base.in b/cvat/requirements/base.in index 0143384d50b..5fe8a915522 100644 --- a/cvat/requirements/base.in +++ b/cvat/requirements/base.in @@ -6,7 +6,7 @@ azure-storage-blob==12.13.0 boto3==1.17.61 clickhouse-connect==0.6.8 coreapi==2.3.3 -datumaro @ git+https://github.com/cvat-ai/datumaro.git@04832fb0bd2b54ed4f32887a8720dc8cc933119d +datumaro @ git+https://github.com/cvat-ai/datumaro.git@125840fc6b28875cce4c85626a5c36bb9e0d2a83 dj-pagination==2.5.0 # Despite direct indication allauth in requirements we should keep 'with_social' for dj-rest-auth # to avoid possible further versions conflicts (we use registration functionality) diff --git a/cvat/requirements/base.txt b/cvat/requirements/base.txt index 4e4130de921..0ec6e5e9b34 100644 --- a/cvat/requirements/base.txt +++ b/cvat/requirements/base.txt @@ -1,4 +1,4 @@ -# SHA1:f9af11b7b937b0496d96a979165025e7de42de8d +# SHA1:e4ac293d721d613911f12dc1095ab4dced2072d8 # # This file is autogenerated by pip-compile-multi # To update, run: @@ -28,7 +28,7 @@ botocore==1.20.112 # via # boto3 # s3transfer -cachetools==5.4.0 +cachetools==5.5.0 # via google-auth certifi==2024.7.4 # via @@ -57,7 +57,7 @@ cryptography==43.0.0 # pyjwt cycler==0.12.1 # via matplotlib -datumaro @ git+https://github.com/cvat-ai/datumaro.git@04832fb0bd2b54ed4f32887a8720dc8cc933119d +datumaro @ git+https://github.com/cvat-ai/datumaro.git@125840fc6b28875cce4c85626a5c36bb9e0d2a83 # via -r cvat/requirements/base.in defusedxml==0.7.1 # via @@ -127,7 +127,7 @@ google-api-core==2.19.1 # via # google-cloud-core # google-cloud-storage -google-auth==2.33.0 +google-auth==2.34.0 # via # google-api-core # google-cloud-core @@ -148,7 +148,7 @@ idna==3.7 # via requests importlib-metadata==8.2.0 # via clickhouse-connect -importlib-resources==6.4.2 +importlib-resources==6.4.3 # via limits inflection==0.5.1 # via drf-spectacular diff --git a/cvat/requirements/production.txt b/cvat/requirements/production.txt index 0659df392c9..3c73c292466 100644 --- a/cvat/requirements/production.txt +++ b/cvat/requirements/production.txt @@ -25,7 +25,7 @@ sniffio==1.3.1 # via anyio uvicorn[standard]==0.22.0 # via -r cvat/requirements/production.in -uvloop==0.19.0 +uvloop==0.20.0 # via uvicorn watchfiles==0.23.0 # via uvicorn diff --git a/site/content/en/docs/manual/advanced/formats/format-yolov8.md b/site/content/en/docs/manual/advanced/formats/format-yolov8.md new file mode 100644 index 00000000000..4d2975900ab --- /dev/null +++ b/site/content/en/docs/manual/advanced/formats/format-yolov8.md @@ -0,0 +1,128 @@ +--- +title: 'YOLOv8' +linkTitle: 'YOLOv8' +weight: 7 +description: 'How to export and import data in YOLOv8 formats' +--- + +YOLOv8 is a format family which consists of four formats: +- [Detection](https://docs.ultralytics.com/datasets/detect/) +- [Oriented bounding Box](https://docs.ultralytics.com/datasets/obb/) +- [Segmentation](https://docs.ultralytics.com/datasets/segment/) +- [Pose](https://docs.ultralytics.com/datasets/pose/) + +Dataset examples: +- [Detection](https://github.com/cvat-ai/datumaro/tree/develop/tests/assets/yolo_dataset/yolov8_detection) +- [Oriented Bounding Boxes](https://github.com/cvat-ai/datumaro/tree/develop/tests/assets/yolo_dataset/yolov8_oriented_boxes) +- [Segmentation](https://github.com/cvat-ai/datumaro/tree/develop/tests/assets/yolo_dataset/yolov8_segmentation) +- [Pose](https://github.com/cvat-ai/datumaro/tree/develop/tests/assets/yolo_dataset/yolov8_pose) + + +## YOLOv8 export + +For export of images: + +- Supported annotations + - Detection: Bounding Boxes + - Oriented bounding box: Oriented Bounding Boxes + - Segmentation: Polygons, Masks + - Pose: Skeletons +- Attributes: Not supported. +- Tracks: Not supported. + +The downloaded file is a .zip archive with the following structure: + +```bash +archive.zip/ + ├── data.yaml # configuration file + ├── train.txt # list of train subset image paths + │ + ├── images/ + │ ├── train/ # directory with images for train subset + │ │ ├── image1.jpg + │ │ ├── image2.jpg + │ │ ├── image3.jpg + │ │ └── ... + ├── labels/ + │ ├── train/ # directory with annotations for train subset + │ │ ├── image1.txt + │ │ ├── image2.txt + │ │ ├── image3.txt + │ │ └── ... + +# train.txt: +images//image1.jpg +images//image2.jpg +... + +# data.yaml: +path: ./ # dataset root dir +train: train.txt # train images (relative to 'path') + +# YOLOv8 Pose specific field +# First number is the number of points in a skeleton. +# If there are several skeletons with different number of points, it is the greatest number of points +# Second number defines the format of point info in annotation txt files +kpt_shape: [17, 3] + +# Classes +names: + 0: person + 1: bicycle + 2: car + # ... + +# .txt: +# content depends on format + +# YOLOv8 Detection: +# label_id - id from names field of data.yaml +# cx, cy - relative coordinates of the bbox center +# rw, rh - relative size of the bbox +# label_id cx cy rw rh +1 0.3 0.8 0.1 0.3 +2 0.7 0.2 0.3 0.1 + +# YOLOv8 Oriented Bounding Boxes: +# xn, yn - relative coordinates of the n-th point +# label_id x1 y1 x2 y2 x3 y3 x4 y4 +1 0.3 0.8 0.1 0.3 0.4 0.5 0.7 0.5 +2 0.7 0.2 0.3 0.1 0.4 0.5 0.5 0.6 + +# YOLOv8 Segmentation: +# xn, yn - relative coordinates of the n-th point +# label_id x1 y1 x2 y2 x3 y3 ... +1 0.3 0.8 0.1 0.3 0.4 0.5 +2 0.7 0.2 0.3 0.1 0.4 0.5 0.5 0.6 0.7 0.5 + +# YOLOv8 Pose: +# cx, cy - relative coordinates of the bbox center +# rw, rh - relative size of the bbox +# xn, yn - relative coordinates of the n-th point +# vn - visibility of n-th point. 2 - visible, 1 - partially visible, 0 - not visible +# if second value in kpt_shape is 3: +# label_id cx cy rw rh x1 y1 v1 x2 y2 v2 x3 y3 v3 ... +1 0.3 0.8 0.1 0.3 0.3 0.8 2 0.1 0.3 2 0.4 0.5 2 0.0 0.0 0 0.0 0.0 0 +2 0.3 0.8 0.1 0.3 0.7 0.2 2 0.3 0.1 1 0.4 0.5 0 0.5 0.6 2 0.7 0.5 2 + +# if second value in kpt_shape is 2: +# label_id cx cy rw rh x1 y1 x2 y2 x3 y3 ... +1 0.3 0.8 0.1 0.3 0.3 0.8 0.1 0.3 0.4 0.5 0.0 0.0 0.0 0.0 +2 0.3 0.8 0.1 0.3 0.7 0.2 0.3 0.1 0.4 0.5 0.5 0.6 0.7 0.5 + +# Note, that if there are several skeletons with different number of points, +# smaller skeletons are padded with points with coordinates 0.0 0.0 and visibility = 0 +``` + +All coordinates must be normalized. +It can be achieved by dividing x coordinates and widths by image width, +and y coordinates and heights by image height. +> Note, that in CVAT you can place an object or some parts of it outside the image, +> which will cause the coordinates to be outside the \[0, 1\] range. +> YOLOv8 framework ignores labels with such coordinates. + +Each annotation file, with the `.txt` extension, +is named to correspond with its associated image file. + +For example, `frame_000001.txt` serves as the annotation for the +`frame_000001.jpg` image. diff --git a/tests/python/rest_api/test_projects.py b/tests/python/rest_api/test_projects.py index 31f4bf8023a..3105388b153 100644 --- a/tests/python/rest_api/test_projects.py +++ b/tests/python/rest_api/test_projects.py @@ -695,6 +695,7 @@ def test_can_import_dataset_in_org(self, admin_user: str): ("CVAT for images 1.1", "CVAT 1.1"), ("CVAT for video 1.1", "CVAT 1.1"), ("Datumaro 1.0", "Datumaro 1.0"), + ("YOLOv8 Pose 1.0", "YOLOv8 Pose 1.0"), ), ) def test_can_export_and_import_dataset_with_skeletons( @@ -984,6 +985,10 @@ def test_can_export_and_import_dataset_after_deleting_related_storage( ("LFW 1.0", "{subset}/images/"), ("Cityscapes 1.0", "imgsFine/leftImg8bit/{subset}/"), ("Open Images V6 1.0", "images/{subset}/"), + ("YOLOv8 Detection 1.0", "images/{subset}/"), + ("YOLOv8 Oriented Bounding Boxes 1.0", "images/{subset}/"), + ("YOLOv8 Segmentation 1.0", "images/{subset}/"), + ("YOLOv8 Pose 1.0", "images/{subset}/"), ], ) @pytest.mark.parametrize("api_version", (1, 2)) diff --git a/tests/python/rest_api/test_tasks.py b/tests/python/rest_api/test_tasks.py index 6cb0ea0568b..7ce1978ae43 100644 --- a/tests/python/rest_api/test_tasks.py +++ b/tests/python/rest_api/test_tasks.py @@ -866,6 +866,7 @@ def test_export_dataset_after_deleting_related_cloud_storage( [ ("Datumaro 1.0", "", "images/{subset}"), ("YOLO 1.1", "train", "obj_{subset}_data"), + ("YOLOv8 Detection 1.0", "train", "images/{subset}"), ], ) @pytest.mark.parametrize("api_version", (1, 2)) @@ -3009,6 +3010,10 @@ def test_import_annotations_after_deleting_related_cloud_storage( "Open Images V6 1.0", "Datumaro 1.0", "Datumaro 3D 1.0", + "YOLOv8 Oriented Bounding Boxes 1.0", + "YOLOv8 Detection 1.0", + "YOLOv8 Pose 1.0", + "YOLOv8 Segmentation 1.0", ], ) def test_check_import_error_on_wrong_file_structure(self, tasks_with_shapes, format_name): From f93d58c1ca9401daeee5beba5d5f79ace975c02b Mon Sep 17 00:00:00 2001 From: Roman Donchenko Date: Fri, 23 Aug 2024 17:38:24 +0300 Subject: [PATCH 19/22] Prepare auto-annotation RQ jobs for migration to the general request API (#8331) * Define an RQ ID format for auto-annotation jobs, and make use of it. * Add a `function_id` field to `RequestSerializer`, so that the general request API can expose the same information as the lambda request API. (In truth, the lambda request API also exposes the "threshold" field, but the UI doesn't use it, and I don't see the point in having it.) Note that this doesn't actually _enable_ the general request API for auto-annotation requests. This is because a similar patch needs to first be applied to the Enterprise version, otherwise requests for Roboflow/Hugging Face jobs will be invisible. --- cvat-core/src/request.ts | 2 ++ cvat/apps/engine/models.py | 1 + cvat/apps/engine/rq_job_handler.py | 7 +++++-- cvat/apps/engine/serializers.py | 2 ++ cvat/apps/lambda_manager/views.py | 28 ++++++++++++++++++++++++---- cvat/schema.yml | 4 ++++ 6 files changed, 38 insertions(+), 6 deletions(-) diff --git a/cvat-core/src/request.ts b/cvat-core/src/request.ts index 2c04402c8df..66ae49b4c96 100644 --- a/cvat-core/src/request.ts +++ b/cvat-core/src/request.ts @@ -13,6 +13,7 @@ type Operation = { jobID: number | null; taskID: number | null; projectID: number | null; + functionID: string | null; }; export class Request { @@ -72,6 +73,7 @@ export class Request { jobID: this.#operation.job_id, taskID: this.#operation.task_id, projectID: this.#operation.project_id, + functionID: this.#operation.function_id, }; } diff --git a/cvat/apps/engine/models.py b/cvat/apps/engine/models.py index 84ab337e931..eda765e6beb 100644 --- a/cvat/apps/engine/models.py +++ b/cvat/apps/engine/models.py @@ -1204,6 +1204,7 @@ class RequestStatus(TextChoices): FINISHED = "finished" class RequestAction(TextChoices): + AUTOANNOTATE = "autoannotate" CREATE = "create" IMPORT = "import" EXPORT = "export" diff --git a/cvat/apps/engine/rq_job_handler.py b/cvat/apps/engine/rq_job_handler.py index dfbfbbf6ae2..25900fba20a 100644 --- a/cvat/apps/engine/rq_job_handler.py +++ b/cvat/apps/engine/rq_job_handler.py @@ -27,6 +27,7 @@ class RQJobMetaField: TASK_PROGRESS = 'task_progress' # export specific fields RESULT_URL = 'result_url' + FUNCTION_ID = 'function_id' def is_rq_job_owner(rq_job: RQJob, user_id: int) -> bool: @@ -59,6 +60,7 @@ class RQId: ) _OPTIONAL_FIELD_REQUIREMENTS = { + RequestAction.AUTOANNOTATE: {"subresource": False, "format": False, "user_id": False}, RequestAction.CREATE: {"subresource": False, "format": False, "user_id": False}, RequestAction.EXPORT: {"subresource": True, "user_id": True}, RequestAction.IMPORT: {"subresource": True, "format": False, "user_id": False}, @@ -74,6 +76,7 @@ def __attrs_post_init__(self) -> None: raise ValueError(f"{field} is not allowed for the {self.action} action") # RQ ID templates: + # autoannotate:task- # import:-- # create:task- # export:---in--format-by- @@ -94,7 +97,7 @@ def render( format_to_be_used_in_urls = self.format.replace(" ", "_").replace(".", "@") return f"{common_prefix}-{self.subresource}-in-{format_to_be_used_in_urls}-format-by-{self.user_id}" - elif RequestAction.CREATE == self.action: + elif self.action in {RequestAction.CREATE, RequestAction.AUTOANNOTATE}: return common_prefix else: assert False, f"Unsupported action {self.action!r} was found" @@ -112,7 +115,7 @@ def parse(rq_id: str) -> RQId: action = RequestAction(action_str) target = RequestTarget(target_str) - if RequestAction.CREATE == action: + if action in {RequestAction.CREATE, RequestAction.AUTOANNOTATE}: identifier = unparsed elif RequestAction.IMPORT == action: identifier, subresource_str = unparsed.rsplit("-", maxsplit=1) diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index d0177140e45..9d66b1716c1 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -2208,6 +2208,7 @@ class RequestDataOperationSerializer(serializers.Serializer): task_id = serializers.IntegerField(required=False, allow_null=True) job_id = serializers.IntegerField(required=False, allow_null=True) format = serializers.CharField(required=False, allow_null=True) + function_id = serializers.CharField(required=False, allow_null=True) def to_representation(self, rq_job: RQJob) -> Dict[str, Any]: parsed_rq_id: RQId = rq_job.parsed_rq_id @@ -2224,6 +2225,7 @@ def to_representation(self, rq_job: RQJob) -> Dict[str, Any]: "task_id": rq_job.meta[RQJobMetaField.TASK_ID], "job_id": rq_job.meta[RQJobMetaField.JOB_ID], "format": parsed_rq_id.format, + "function_id": rq_job.meta.get(RQJobMetaField.FUNCTION_ID), } class RequestSerializer(serializers.Serializer): diff --git a/cvat/apps/lambda_manager/views.py b/cvat/apps/lambda_manager/views.py index 419701e95c6..306ba6f8c86 100644 --- a/cvat/apps/lambda_manager/views.py +++ b/cvat/apps/lambda_manager/views.py @@ -33,7 +33,10 @@ import cvat.apps.dataset_manager as dm from cvat.apps.engine.frame_provider import FrameProvider -from cvat.apps.engine.models import Job, ShapeType, SourceType, Task, Label +from cvat.apps.engine.models import ( + Job, ShapeType, SourceType, Task, Label, RequestAction, RequestTarget, +) +from cvat.apps.engine.rq_job_handler import RQId, RQJobMetaField from cvat.apps.engine.serializers import LabeledDataSerializer from cvat.apps.lambda_manager.permissions import LambdaPermission from cvat.apps.lambda_manager.serializers import ( @@ -525,16 +528,31 @@ def enqueue(self, *, job: Optional[int] = None ) -> LambdaJob: - jobs = self.get_jobs() + queue = self._get_queue() + rq_id = RQId(RequestAction.AUTOANNOTATE, RequestTarget.TASK, task).render() + # It is still possible to run several concurrent jobs for the same task. # But the race isn't critical. The filtration is just a light-weight # protection. - if list(filter(lambda job: job.get_task() == task and not job.is_finished, jobs)): + rq_job = queue.fetch_job(rq_id) + + have_conflict = rq_job and \ + rq_job.get_status(refresh=False) not in {rq.job.JobStatus.FAILED, rq.job.JobStatus.FINISHED} + + # There could be some jobs left over from before the current naming convention was adopted. + # TODO: remove this check after a few releases. + have_legacy_conflict = any( + job.get_task() == task and not (job.is_finished or job.is_failed) + for job in self.get_jobs() + ) + if have_conflict or have_legacy_conflict: raise ValidationError( "Only one running request is allowed for the same task #{}".format(task), code=status.HTTP_409_CONFLICT) - queue = self._get_queue() + if rq_job: + rq_job.delete() + # LambdaJob(None) is a workaround for python-rq. It has multiple issues # with invocation of non-trivial functions. For example, it cannot run # staticmethod, it cannot run a callable class. Thus I provide an object @@ -543,6 +561,7 @@ def enqueue(self, with get_rq_lock_by_user(queue, user_id): rq_job = queue.create_job(LambdaJob(None), + job_id=rq_id, meta={ **get_rq_job_meta( request, @@ -550,6 +569,7 @@ def enqueue(self, Job.objects.get(pk=job) if job else Task.objects.get(pk=task) ), ), + RQJobMetaField.FUNCTION_ID: lambda_func.id, "lambda": True, }, kwargs={ diff --git a/cvat/schema.yml b/cvat/schema.yml index 9135c153fc0..1459ff77a6c 100644 --- a/cvat/schema.yml +++ b/cvat/schema.yml @@ -4529,6 +4529,7 @@ paths: schema: type: string enum: + - autoannotate - create - import - export @@ -9995,6 +9996,9 @@ components: format: type: string nullable: true + function_id: + type: string + nullable: true required: - target - type From ac258aeb3ce4f822bd84f374ba520ded7667276a Mon Sep 17 00:00:00 2001 From: Mariia Acoca <39969264+mdacoca@users.noreply.github.com> Date: Tue, 27 Aug 2024 12:22:38 +0200 Subject: [PATCH 20/22] Propagate filtered shapes Documentation (#8181) --- site/content/en/docs/enterprise/shapes-converter.md | 2 +- .../manual/basics/CVAT-annotation-Interface/objects-sidebar.md | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/site/content/en/docs/enterprise/shapes-converter.md b/site/content/en/docs/enterprise/shapes-converter.md index 45a69ada5d1..fbceb757c3f 100644 --- a/site/content/en/docs/enterprise/shapes-converter.md +++ b/site/content/en/docs/enterprise/shapes-converter.md @@ -36,7 +36,7 @@ With the following fields: | Field | Description | | ---------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **Select action** | Drop-down list with available actions:
  • **Remove filtered shapes** - removes all shapes in alignment with the set-up filter. Doesn't work with tracks.
  • **Shapes converter: masks to polygons** - converts all masks to polygons.
  • **Shapes converter: masks to rectangles** - converts all masks to rectangles in alignment with the set-up filter.
  • **Shapes converter: polygon to masks** - converts all polygons to masks.
  • **Shapes converter: polygon to rectangles** - converts all polygons to rectangles.
  • **Shapes converter: rectangles to masks** - converts all rectangles to masks.
  • **Shapes converter: rectangles to polygons** - converts all rectangles to polygons.

    • **Note:** only **Remove filtered shapes** is available on the **Free** plan. | +| **Select action** | Drop-down list with available actions:
  • **Remove filtered shapes** - removes all shapes in alignment with the set-up filter. Doesn't work with tracks.
  • **Propagate shapes** - propagates all the filtered shapes from the current frame to the target frame.
  • **Shapes converter: masks to polygons** - converts all masks to polygons.
  • **Shapes converter: masks to rectangles** - converts all masks to rectangles in alignment with the set-up filter.
  • **Shapes converter: polygon to masks** - converts all polygons to masks.
  • **Shapes converter: polygon to rectangles** - converts all polygons to rectangles.
  • **Shapes converter: rectangles to masks** - converts all rectangles to masks.
  • **Shapes converter: rectangles to polygons** - converts all rectangles to polygons.

    • **Note:** only **Propagate shapes** and **Remove filtered shapes** is available in the community version. | | **Specify frames to run action** | Field where you can specify the frame range for the selected action. Enter the starting frame in the **Starting from frame:** field, and the ending frame in the **up to frame** field.

    If nothing is selected here or in **Choose one of the predefined options** section, the action will be applied to all fields. | | **Choose one of the predefined options** | Predefined options to apply to frames. Selection here is mutually exclusive with **Specify frames to run action**.

    If nothing is selected here or in **Specify frames to run action** section, the action will be applied to all fields. | diff --git a/site/content/en/docs/manual/basics/CVAT-annotation-Interface/objects-sidebar.md b/site/content/en/docs/manual/basics/CVAT-annotation-Interface/objects-sidebar.md index eda3a33dd6e..e97d5ccad37 100644 --- a/site/content/en/docs/manual/basics/CVAT-annotation-Interface/objects-sidebar.md +++ b/site/content/en/docs/manual/basics/CVAT-annotation-Interface/objects-sidebar.md @@ -52,7 +52,8 @@ The action menu contains: - **Propagate** function copies the form to multiple frames and displays a dialog box where you can specify the number of copies or the frame to which you want to copy the object. - The keyboard shortcut is **Ctr**l + **B**.
    There are two options available: + The keyboard shortcut is **Ctr**l + **B**. On how to propagate + only filtered shapes, see [Shapes converter](/docs/enterprise/shapes-converter/)
    There are two options available: - **Propagate forward** (![Fw propagate](/images/propagate_fw.png)) creates a copy of the object on `N` _subsequent_ frames at the same position. From ec2807692a788858334293470a3b9cc0c4f0cd24 Mon Sep 17 00:00:00 2001 From: Roman Donchenko Date: Tue, 27 Aug 2024 18:02:10 +0300 Subject: [PATCH 21/22] Upgrade Silk (#8357) Mainly to get the fix for jazzband/django-silk#688. --- cvat/requirements/development.in | 2 +- cvat/requirements/development.txt | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/cvat/requirements/development.in b/cvat/requirements/development.in index bea1c0c783e..ad5a5b6557e 100644 --- a/cvat/requirements/development.in +++ b/cvat/requirements/development.in @@ -2,7 +2,7 @@ black>=24.1 django-extensions==3.0.8 -django-silk==5.0.3 +django-silk==5.* pylint-django==2.5.3 pylint-plugin-utils==0.7 pylint==2.14.5 diff --git a/cvat/requirements/development.txt b/cvat/requirements/development.txt index 63f262430eb..c7613610fef 100644 --- a/cvat/requirements/development.txt +++ b/cvat/requirements/development.txt @@ -1,4 +1,4 @@ -# SHA1:27238f2f377debba1b5fe910878f0cc0cfaf6e7d +# SHA1:b71f4fe955f645187b7ccdf82b05f6a8d61eb3ab # # This file is autogenerated by pip-compile-multi # To update, run: @@ -6,6 +6,8 @@ # pip-compile-multi # -r base.txt +--no-binary lxml +--no-binary xmlsec astroid==2.11.7 # via pylint @@ -17,7 +19,7 @@ dill==0.3.8 # via pylint django-extensions==3.0.8 # via -r cvat/requirements/development.in -django-silk==5.0.3 +django-silk==5.2.0 # via -r cvat/requirements/development.in gprof2dot==2024.6.6 # via django-silk From 4d8621a250ea86c71cf556d26e04e3cba8fdd1e7 Mon Sep 17 00:00:00 2001 From: "cvat-bot[bot]" <147643061+cvat-bot[bot]@users.noreply.github.com> Date: Tue, 27 Aug 2024 16:38:51 +0000 Subject: [PATCH 22/22] Prepare release v2.17.0 --- CHANGELOG.md | 52 +++++++++++++++++++ ..._160939_dmitrii.lavrukhin_yolo8_support.md | 4 -- .../20240813_115132_klakhov_fix_go_back.md | 4 -- .../20240814_151947_boris_update_events.md | 3 -- .../20240814_153835_boris_update_events.md | 4 -- .../20240815_143525_roman_no_token_in_ui_2.md | 15 ------ ...20240819_210200_mzhiltso_validation_api.md | 4 -- .../20240820_134050_maria_rest_api_tests.md | 9 ---- ...22_111601_sekachev.bs_fixed_minor_issue.md | 4 -- ...240822_134319_roman_rm_login_with_token.md | 4 -- cvat/__init__.py | 2 +- docker-compose.yml | 18 +++---- helm-chart/values.yaml | 4 +- 13 files changed, 64 insertions(+), 63 deletions(-) delete mode 100644 changelog.d/20240730_160939_dmitrii.lavrukhin_yolo8_support.md delete mode 100644 changelog.d/20240813_115132_klakhov_fix_go_back.md delete mode 100644 changelog.d/20240814_151947_boris_update_events.md delete mode 100644 changelog.d/20240814_153835_boris_update_events.md delete mode 100644 changelog.d/20240815_143525_roman_no_token_in_ui_2.md delete mode 100644 changelog.d/20240819_210200_mzhiltso_validation_api.md delete mode 100644 changelog.d/20240820_134050_maria_rest_api_tests.md delete mode 100644 changelog.d/20240822_111601_sekachev.bs_fixed_minor_issue.md delete mode 100644 changelog.d/20240822_134319_roman_rm_login_with_token.md diff --git a/CHANGELOG.md b/CHANGELOG.md index d14eaca7895..02550f9ee8e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,58 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 +     +## \[2.17.0\] - 2024-08-27 + +### Added + +- Added support for YOLOv8 formats + () + +- Last assignee update date in quality reports, new options in quality settings + () + +### Changed + +- User sessions now expire after two weeks of inactivity + () + +- A user changing their password will now invalidate all of their sessions + except for the current one + () + +### Deprecated + +- Client events `upload:annotations`, `lock:object`, `change:attribute`, `change:label` + () + +### Removed + +- Client event `restore:job` () + +- Removed the `/auth/login-with-token` page + () + +### Fixed + +- Go back button behavior on analytics page + () + +- Logging out of one session will no longer log the user out of all their + other sessions + () + +- Prevent export process from restarting when downloading a result file, + that resulted in downloading a file with new request ID + () +- Race condition occurred while handling parallel export requests + () +- Requests filtering using format and target filters + () + +- Sometimes it is not possible to switch workspace because active control broken after +trying to create a tag with a shortcut () + ## \[2.16.3\] - 2024-08-13 diff --git a/changelog.d/20240730_160939_dmitrii.lavrukhin_yolo8_support.md b/changelog.d/20240730_160939_dmitrii.lavrukhin_yolo8_support.md deleted file mode 100644 index 4d4e91f321b..00000000000 --- a/changelog.d/20240730_160939_dmitrii.lavrukhin_yolo8_support.md +++ /dev/null @@ -1,4 +0,0 @@ -### Added - -- Added support for YOLOv8 formats - () diff --git a/changelog.d/20240813_115132_klakhov_fix_go_back.md b/changelog.d/20240813_115132_klakhov_fix_go_back.md deleted file mode 100644 index 6eb6cb9b945..00000000000 --- a/changelog.d/20240813_115132_klakhov_fix_go_back.md +++ /dev/null @@ -1,4 +0,0 @@ -### Fixed - -- Go back button behavior on analytics page - () diff --git a/changelog.d/20240814_151947_boris_update_events.md b/changelog.d/20240814_151947_boris_update_events.md deleted file mode 100644 index d65e561e065..00000000000 --- a/changelog.d/20240814_151947_boris_update_events.md +++ /dev/null @@ -1,3 +0,0 @@ -### Removed - -- Client event `restore:job` () diff --git a/changelog.d/20240814_153835_boris_update_events.md b/changelog.d/20240814_153835_boris_update_events.md deleted file mode 100644 index 605d93f187e..00000000000 --- a/changelog.d/20240814_153835_boris_update_events.md +++ /dev/null @@ -1,4 +0,0 @@ -### Deprecated - -- Client events `upload:annotations`, `lock:object`, `change:attribute`, `change:label` - () diff --git a/changelog.d/20240815_143525_roman_no_token_in_ui_2.md b/changelog.d/20240815_143525_roman_no_token_in_ui_2.md deleted file mode 100644 index 2c54d4b4c9e..00000000000 --- a/changelog.d/20240815_143525_roman_no_token_in_ui_2.md +++ /dev/null @@ -1,15 +0,0 @@ -### Fixed - -- Logging out of one session will no longer log the user out of all their - other sessions - () - -### Changed - -- User sessions now expire after two weeks of inactivity - () - -- A user changing their password will now invalidate all of their sessions - except for the current one - () - diff --git a/changelog.d/20240819_210200_mzhiltso_validation_api.md b/changelog.d/20240819_210200_mzhiltso_validation_api.md deleted file mode 100644 index 2031da1456a..00000000000 --- a/changelog.d/20240819_210200_mzhiltso_validation_api.md +++ /dev/null @@ -1,4 +0,0 @@ -### Added - -- Last assignee update date in quality reports, new options in quality settings - () diff --git a/changelog.d/20240820_134050_maria_rest_api_tests.md b/changelog.d/20240820_134050_maria_rest_api_tests.md deleted file mode 100644 index 42b9bccf4fb..00000000000 --- a/changelog.d/20240820_134050_maria_rest_api_tests.md +++ /dev/null @@ -1,9 +0,0 @@ -### Fixed - -- Prevent export process from restarting when downloading a result file, - that resulted in downloading a file with new request ID - () -- Race condition occurred while handling parallel export requests - () -- Requests filtering using format and target filters - () diff --git a/changelog.d/20240822_111601_sekachev.bs_fixed_minor_issue.md b/changelog.d/20240822_111601_sekachev.bs_fixed_minor_issue.md deleted file mode 100644 index 7ae4d38588b..00000000000 --- a/changelog.d/20240822_111601_sekachev.bs_fixed_minor_issue.md +++ /dev/null @@ -1,4 +0,0 @@ -### Fixed - -- Sometimes it is not possible to switch workspace because active control broken after -trying to create a tag with a shortcut () diff --git a/changelog.d/20240822_134319_roman_rm_login_with_token.md b/changelog.d/20240822_134319_roman_rm_login_with_token.md deleted file mode 100644 index 45d1631c893..00000000000 --- a/changelog.d/20240822_134319_roman_rm_login_with_token.md +++ /dev/null @@ -1,4 +0,0 @@ -### Removed - -- Removed the `/auth/login-with-token` page - () diff --git a/cvat/__init__.py b/cvat/__init__.py index dff5c95b13b..70840afb3c0 100644 --- a/cvat/__init__.py +++ b/cvat/__init__.py @@ -4,6 +4,6 @@ from cvat.utils.version import get_version -VERSION = (2, 17, 0, 'alpha', 0) +VERSION = (2, 17, 0, 'final', 0) __version__ = get_version(VERSION) diff --git a/docker-compose.yml b/docker-compose.yml index 051bd0bfd8c..ebd9e5cbdc9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -78,7 +78,7 @@ services: cvat_server: container_name: cvat_server - image: cvat/server:${CVAT_VERSION:-dev} + image: cvat/server:${CVAT_VERSION:-v2.17.0} restart: always depends_on: <<: *backend-deps @@ -112,7 +112,7 @@ services: cvat_utils: container_name: cvat_utils - image: cvat/server:${CVAT_VERSION:-dev} + image: cvat/server:${CVAT_VERSION:-v2.17.0} restart: always depends_on: *backend-deps environment: @@ -129,7 +129,7 @@ services: cvat_worker_import: container_name: cvat_worker_import - image: cvat/server:${CVAT_VERSION:-dev} + image: cvat/server:${CVAT_VERSION:-v2.17.0} restart: always depends_on: *backend-deps environment: @@ -145,7 +145,7 @@ services: cvat_worker_export: container_name: cvat_worker_export - image: cvat/server:${CVAT_VERSION:-dev} + image: cvat/server:${CVAT_VERSION:-v2.17.0} restart: always depends_on: *backend-deps environment: @@ -161,7 +161,7 @@ services: cvat_worker_annotation: container_name: cvat_worker_annotation - image: cvat/server:${CVAT_VERSION:-dev} + image: cvat/server:${CVAT_VERSION:-v2.17.0} restart: always depends_on: *backend-deps environment: @@ -177,7 +177,7 @@ services: cvat_worker_webhooks: container_name: cvat_worker_webhooks - image: cvat/server:${CVAT_VERSION:-dev} + image: cvat/server:${CVAT_VERSION:-v2.17.0} restart: always depends_on: *backend-deps environment: @@ -193,7 +193,7 @@ services: cvat_worker_quality_reports: container_name: cvat_worker_quality_reports - image: cvat/server:${CVAT_VERSION:-dev} + image: cvat/server:${CVAT_VERSION:-v2.17.0} restart: always depends_on: *backend-deps environment: @@ -209,7 +209,7 @@ services: cvat_worker_analytics_reports: container_name: cvat_worker_analytics_reports - image: cvat/server:${CVAT_VERSION:-dev} + image: cvat/server:${CVAT_VERSION:-v2.17.0} restart: always depends_on: *backend-deps environment: @@ -225,7 +225,7 @@ services: cvat_ui: container_name: cvat_ui - image: cvat/ui:${CVAT_VERSION:-dev} + image: cvat/ui:${CVAT_VERSION:-v2.17.0} restart: always depends_on: - cvat_server diff --git a/helm-chart/values.yaml b/helm-chart/values.yaml index 91e4493258f..becd098c790 100644 --- a/helm-chart/values.yaml +++ b/helm-chart/values.yaml @@ -115,7 +115,7 @@ cvat: additionalVolumeMounts: [] replicas: 1 image: cvat/server - tag: dev + tag: v2.17.0 imagePullPolicy: Always permissionFix: enabled: true @@ -139,7 +139,7 @@ cvat: frontend: replicas: 1 image: cvat/ui - tag: dev + tag: v2.17.0 imagePullPolicy: Always labels: {} # test: test