Quality control page

changelog.d/20240826_093730_klakhov_support_quality_plugin.md

@@ -0,0 +1,14 @@
+### Changed
+
+- Moved quality control from `analytics` page to `quality control` page
+  (<https://github.com/cvat-ai/cvat/pull/8329>)
+
+### Removed
+
+- Quality report no longer available in CVAT community version
+  (<https://github.com/cvat-ai/cvat/pull/8329>)
+
+### Added
+
+- Quality management tab on `quality control` allowing to enable/disable GT frames
+  (<https://github.com/cvat-ai/cvat/pull/8329>)

cvat-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cvat-core",
-  "version": "15.1.2",
+  "version": "15.2.0",
   "type": "module",
   "description": "Part of Computer Vision Tool which presents an interface for client-side integration",
   "main": "src/api.ts",

cvat-core/src/api-implementation.ts

@@ -37,7 +37,7 @@ import {
 import QualityReport from './quality-report';
 import QualityConflict, { ConflictSeverity } from './quality-conflict';
 import QualitySettings from './quality-settings';
-import { FramesMetaData } from './frames';
+import { FramesMetaData, getJobMeta } from './frames';
 import AnalyticsReport from './analytics-report';
 import { listActions, registerAction, runActions } from './annotations-actions';
 import { JobType } from './enums';
@@ -550,11 +550,16 @@ export default function implementAPI(cvat: CVATCore): CVATCore {
         await serverProxy.analytics.performance.calculate(params, onUpdate);
     });
     implementationMixin(cvat.frames.getMeta, async (type, id) => {
-        const result = await serverProxy.frames.getMeta(type, id);
-        return new FramesMetaData({
-            ...result,
-            deleted_frames: Object.fromEntries(result.deleted_frames.map((_frame) => [_frame, true])),
-        });
+        if (type === 'task') {
+            const result = await serverProxy.frames.getMeta(type, id);
+            return new FramesMetaData({
+                ...result,
+                deleted_frames: Object.fromEntries(result.deleted_frames.map((_frame) => [_frame, true])),
+            });
+        }
+
+        const result = await getJobMeta(id, { reload: true });
+        return result;
     });
 
     return cvat;

cvat-core/src/frames.ts

@@ -435,8 +435,8 @@ Object.defineProperty(FrameData.prototype.data, 'implementation', {
     writable: false,
 });
 
-async function getJobMeta(jobID: number): Promise<FramesMetaData> {
-    if (!frameMetaCache[jobID]) {
+export async function getJobMeta(jobID: number, { reload } = { reload: false }): Promise<FramesMetaData> {
+    if (!frameMetaCache[jobID] || reload) {
         frameMetaCache[jobID] = serverProxy.frames.getMeta('job', jobID)
             .then((serverMeta) => new FramesMetaData({
                 ...serverMeta,
@@ -655,24 +655,25 @@ export async function getDeletedFrames(instanceType: 'job' | 'task', id): Promis
     throw new Exception(`getDeletedFrames is not implemented for ${instanceType}`);
 }
 
-export function deleteFrame(jobID: number, frame: number): void {
-    const { meta } = frameDataCache[jobID];
+export async function deleteFrame(jobID: number, frame: number): Promise<void> {
+    const meta = await frameMetaCache[jobID];
     meta.deletedFrames[frame] = true;
 }
 
-export function restoreFrame(jobID: number, frame: number): void {
-    const { meta } = frameDataCache[jobID];
+export async function restoreFrame(jobID: number, frame: number): Promise<void> {
+    const meta = await frameMetaCache[jobID];
     delete meta.deletedFrames[frame];
 }
 
-export async function patchMeta(jobID: number): Promise<void> {
-    const { meta } = frameDataCache[jobID];
+export async function patchMeta(jobID: number): Promise<FramesMetaData> {
+    const meta = await frameMetaCache[jobID];
     const updatedFields = meta.getUpdated();
 
     if (Object.keys(updatedFields).length) {
-        const newMeta = await saveJobMeta(meta, jobID);
-        frameDataCache[jobID].meta = newMeta;
+        frameMetaCache[jobID] = saveJobMeta(meta, jobID);
     }
+    const newMeta = await frameMetaCache[jobID];
+    return newMeta;
 }
 
 export async function findFrame(

cvat-core/src/quality-settings.ts

@@ -6,8 +6,17 @@ import { SerializedQualitySettingsData } from './server-response-types';
 import PluginRegistry from './plugins';
 import serverProxy from './server-proxy';
 
+export enum TargetMetric {
+    ACCURACY = 'accuracy',
+    PRECISION = 'precision',
+    RECALL = 'recall',
+}
+
 export default class QualitySettings {
     #id: number;
+    #targetMetric: TargetMetric;
+    #targetMetricThreshold: number;
+    #maxValidationsPerJob: number;
     #task: number;
     #iouThreshold: number;
     #oksSigma: number;
@@ -25,6 +34,9 @@ export default class QualitySettings {
     constructor(initialData: SerializedQualitySettingsData) {
         this.#id = initialData.id;
         this.#task = initialData.task;
+        this.#targetMetric = initialData.target_metric as TargetMetric;
+        this.#targetMetricThreshold = initialData.target_metric_threshold;
+        this.#maxValidationsPerJob = initialData.max_validations_per_job;
         this.#iouThreshold = initialData.iou_threshold;
         this.#oksSigma = initialData.oks_sigma;
         this.#lineThickness = initialData.line_thickness;
@@ -143,6 +155,30 @@ export default class QualitySettings {
         this.#compareAttributes = newVal;
     }
 
+    get targetMetric(): TargetMetric {
+        return this.#targetMetric;
+    }
+
+    set targetMetric(newVal: TargetMetric) {
+        this.#targetMetric = newVal;
+    }
+
+    get targetMetricThreshold(): number {
+        return this.#targetMetricThreshold;
+    }
+
+    set targetMetricThreshold(newVal: number) {
+        this.#targetMetricThreshold = newVal;
+    }
+
+    get maxValidationsPerJob(): number {
+        return this.#maxValidationsPerJob;
+    }
+
+    set maxValidationsPerJob(newVal: number) {
+        this.#maxValidationsPerJob = newVal;
+    }
+
     public toJSON(): SerializedQualitySettingsData {
         const result: SerializedQualitySettingsData = {
             iou_threshold: this.#iouThreshold,
@@ -157,6 +193,9 @@ export default class QualitySettings {
             object_visibility_threshold: this.#objectVisibilityThreshold,
             panoptic_comparison: this.#panopticComparison,
             compare_attributes: this.#compareAttributes,
+            target_metric: this.#targetMetric,
+            target_metric_threshold: this.#targetMetricThreshold,
+            max_validations_per_job: this.#maxValidationsPerJob,
         };
 
         return result;
@@ -172,8 +211,10 @@ Object.defineProperties(QualitySettings.prototype.save, {
     implementation: {
         writable: false,
         enumerable: false,
-        value: async function implementation() {
-            const result = await serverProxy.analytics.quality.settings.update(this.id, this.toJSON());
+        value: async function implementation(): Promise<QualitySettings> {
+            const result = await serverProxy.analytics.quality.settings.update(
+                this.id, this.toJSON(),
+            );
             return new QualitySettings(result);
         },
     },

cvat-core/src/server-response-types.ts

@@ -241,6 +241,9 @@ export type QualitySettingsFilter = Camelized<APIQualitySettingsFilter>;
 export interface SerializedQualitySettingsData {
     id?: number;
     task?: number;
+    target_metric?: string;
+    target_metric_threshold?: number;
+    max_validations_per_job?: number;
     iou_threshold?: number;
     oks_sigma?: number;
     line_thickness?: number;

cvat-core/src/session-implementation.ts

@@ -44,9 +44,11 @@ async function deleteFrameWrapper(jobID, frame): Promise<void> {
     };
 
     await redo();
-    getHistory(this).do(HistoryActions.REMOVED_FRAME, async () => {
-        restoreFrame(jobID, frame);
-    }, redo, [], frame);
+    try {
+        getHistory(this).do(HistoryActions.REMOVED_FRAME, async () => {
+            restoreFrame(jobID, frame);
+        }, redo, [], frame);
+    } catch (error) { /* empty */ }
 }
 
 async function restoreFrameWrapper(jobID, frame): Promise<void> {
@@ -55,9 +57,11 @@ async function restoreFrameWrapper(jobID, frame): Promise<void> {
     };
 
     await redo();
-    getHistory(this).do(HistoryActions.RESTORED_FRAME, async () => {
-        deleteFrame(jobID, frame);
-    }, redo, [], frame);
+    try {
+        getHistory(this).do(HistoryActions.RESTORED_FRAME, async () => {
+            deleteFrame(jobID, frame);
+        }, redo, [], frame);
+    } catch (error) { /* empty */ }
 }
 
 export function implementJob(Job: typeof JobClass): typeof JobClass {
@@ -232,7 +236,7 @@ export function implementJob(Job: typeof JobClass): typeof JobClass {
         value: function saveFramesImplementation(
             this: JobClass,
         ): ReturnType<typeof JobClass.prototype.frames.save> {
-            return patchMeta(this.id);
+            return patchMeta(this.id).then((meta) => [meta]);
         },
     });
 
@@ -695,6 +699,19 @@ export function implementTask(Task: typeof TaskClass): typeof TaskClass {
                 taskSpec.source_storage = this.sourceStorage.toJSON();
             }
 
+            if (this.validationMethod !== 'undefined') {
+                taskSpec.validation_method = this.validationMethod;
+            }
+            if (typeof this.validationFramesPercent !== 'undefined') {
+                taskSpec.validation_frames_percent = this.validationFramesPercent;
+            }
+            if (typeof this.validationFramesPerJob !== 'undefined') {
+                taskSpec.validation_frames_per_job = this.validationFramesPerJob;
+            }
+            if (typeof this.frameSelectionMethod !== 'undefined') {
+                taskSpec.frame_selection_method = this.frameSelectionMethod;
+            }
+
             const taskDataSpec = {
                 client_files: this.clientFiles,
                 server_files: this.serverFiles,
@@ -899,8 +916,7 @@ export function implementTask(Task: typeof TaskClass): typeof TaskClass {
         value: async function saveFramesImplementation(
             this: TaskClass,
         ): ReturnType<typeof TaskClass.prototype.frames.save> {
-            return Promise.all(this.jobs.map((job) => patchMeta(job.id)))
-                .then(() => Promise.resolve());
+            return Promise.all(this.jobs.map((job) => patchMeta(job.id)));
         },
     });
 

cvat-core/src/session.ts

@@ -22,7 +22,7 @@ import {
     SerializedLabel, SerializedTask,
 } from './server-response-types';
 import AnnotationGuide from './guide';
-import { FrameData } from './frames';
+import { FrameData, FramesMetaData } from './frames';
 import Statistics from './statistics';
 import { Request } from './request';
 import logger from './logger';
@@ -223,10 +223,11 @@ function buildDuplicatedAPI(prototype) {
                     );
                 },
                 async save() {
-                    await PluginRegistry.apiWrapper.call(
+                    const result = await PluginRegistry.apiWrapper.call(
                         this,
                         prototype.frames.save,
                     );
+                    return result;
                 },
                 async cachedChunks() {
                     const result = await PluginRegistry.apiWrapper.call(this, prototype.frames.cachedChunks);
@@ -377,7 +378,7 @@ export class Session {
         get: (frame: number, isPlaying?: boolean, step?: number) => Promise<FrameData>;
         delete: (frame: number) => Promise<void>;
         restore: (frame: number) => Promise<void>;
-        save: () => Promise<void>;
+        save: () => Promise<FramesMetaData[]>;
         cachedChunks: () => Promise<number[]>;
         preview: () => Promise<string>;
         contextImage: (frame: number) => Promise<Record<string, ImageBitmap>>;
@@ -730,6 +731,11 @@ export class Task extends Session {
     public readonly cloudStorageID: number;
     public readonly sortingMethod: string;
 
+    public readonly validationMethod: string;
+    public readonly validationFramesPercent: number;
+    public readonly validationFramesPerJob: number;
+    public readonly frameSelectionMethod: string;
+
     constructor(initialData: Readonly<Omit<SerializedTask, 'labels' | 'jobs'> & {
         labels?: SerializedLabel[];
         progress?: SerializedTask['jobs'];
@@ -775,7 +781,10 @@ export class Task extends Session {
             sorting_method: undefined,
             files: undefined,
 
-            quality_settings: undefined,
+            validation_method: undefined,
+            validation_frames_percent: undefined,
+            validation_frames_per_job: undefined,
+            frame_selection_method: undefined,
         };
 
         const updateTrigger = new FieldUpdateTrigger();
@@ -1103,6 +1112,18 @@ export class Task extends Session {
                 progress: {
                     get: () => data.progress,
                 },
+                validationFramesPercent: {
+                    get: () => data.validation_frames_percent,
+                },
+                validationFramesPerJob: {
+                    get: () => data.validation_frames_per_job,
+                },
+                frameSelectionMethod: {
+                    get: () => data.frame_selection_method,
+                },
+                validationMethod: {
+                    get: () => data.validation_method,
+                },
                 _internalData: {
                     get: () => data,
                 },

cvat-ui/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cvat-ui",
-  "version": "1.65.0",
+  "version": "1.66.0",
   "description": "CVAT single-page application",
   "main": "src/index.tsx",
   "scripts": {
@@ -35,6 +35,7 @@
     "@types/react-router-dom": "^5.1.9",
     "@types/redux-logger": "^3.0.9",
     "@types/resize-observer-browser": "^0.1.6",
+    "@types/react-resizable": "^3.0.8",
     "@uiw/react-md-editor": "^3.22.0",
     "antd": "5.17.1",
     "chart.js": "^4.3.0",
@@ -59,6 +60,7 @@
     "react-markdown": "^8.0.4",
     "react-moment": "^1.1.1",
     "react-redux": "^8.0.2",
+    "react-resizable": "^3.0.5",
     "react-router": "^5.1.0",
     "react-router-dom": "^5.1.0",
     "react-sortable-hoc": "^2.0.0",

cvat-ui/src/actions/tasks-actions.ts

@@ -11,6 +11,7 @@ import {
 import { filterNull } from 'utils/filter-null';
 import { ThunkDispatch, ThunkAction } from 'utils/redux';
 
+import { ValidationMethod } from 'components/create-task-page/quality-configuration-form';
 import { getInferenceStatusAsync } from './models-actions';
 import { updateRequestProgress } from './requests-actions';
 
@@ -254,11 +255,20 @@ ThunkAction {
             description.cloud_storage_id = data.cloudStorageId;
         }
 
+        if (data.quality.validationMethod === ValidationMethod.GT) {
+            description.validation_method = ValidationMethod.GT;
+            description.validation_frames_percent = data.quality.validationFramesPercent;
+            description.frame_selection_method = data.quality.frameSelectionMethod;
+        } else if (data.quality.validationMethod === ValidationMethod.HONEYPOTS) {
+            description.validation_method = ValidationMethod.HONEYPOTS;
+            description.validation_frames_percent = data.quality.validationFramesPercent;
+            description.validation_frames_per_job = data.quality.validationFramesPerJob;
+        }
+
         const taskInstance = new cvat.classes.Task(description);
         taskInstance.clientFiles = data.files.local;
         taskInstance.serverFiles = data.files.share.concat(data.files.cloudStorage);
         taskInstance.remoteFiles = data.files.remote;
-
         try {
             const savedTask = await taskInstance.save({
                 requestStatusCallback(request) {

cvat-ui/src/assets/index.d.ts

@@ -3,3 +3,4 @@
 // SPDX-License-Identifier: MIT
 
 declare module '*.svg';
+declare module '*.png';