From d7de49fc44bf18c2a57f06a600f7870369eac6da Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Sat, 23 Feb 2019 14:36:46 +0300 Subject: [PATCH 1/8] Deep Extreme Cut initial commit --- CHANGELOG.md | 1 + Dockerfile | 10 + cvat/apps/dextr_segmentation/README.md | 18 ++ cvat/apps/dextr_segmentation/__init__.py | 7 + cvat/apps/dextr_segmentation/apps.py | 8 + cvat/apps/dextr_segmentation/dextr.py | 112 ++++++++++ .../docker-compose.dextr.yml | 14 ++ .../dextr_segmentation/js/enginePlugin.js | 209 ++++++++++++++++++ cvat/apps/dextr_segmentation/urls.py | 12 + cvat/apps/dextr_segmentation/views.py | 128 +++++++++++ cvat/settings/base.py | 3 + cvat/urls.py | 3 + 12 files changed, 525 insertions(+) create mode 100644 cvat/apps/dextr_segmentation/README.md create mode 100644 cvat/apps/dextr_segmentation/__init__.py create mode 100644 cvat/apps/dextr_segmentation/apps.py create mode 100644 cvat/apps/dextr_segmentation/dextr.py create mode 100644 cvat/apps/dextr_segmentation/docker-compose.dextr.yml create mode 100644 cvat/apps/dextr_segmentation/static/dextr_segmentation/js/enginePlugin.js create mode 100644 cvat/apps/dextr_segmentation/urls.py create mode 100644 cvat/apps/dextr_segmentation/views.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 549d87e2ca22..dacf0dad3e1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Ability to rotate images/video in the client part (Ctrl+R, Shift+Ctrl+R shortcuts) (#305) - The ReID application for automatic bounding box merging has been added (#299) - Keyboard shortcuts to switch next/previous default shape type (box, polygon etc) [Alt + <, Alt + >] (#316) +- Semi-automatic semantic segmentation with the [Deep Extreme Cut](http://www.vision.ee.ethz.ch/~cvlsegmentation/dextr/) work ### Changed diff --git a/Dockerfile b/Dockerfile index 48dbe177843d..3f9f3e979ca3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -125,6 +125,16 @@ RUN if [ "$OPENVINO_TOOLKIT" = "yes" ]; then \ wget https://download.01.org/openvinotoolkit/2018_R5/open_model_zoo/person-reidentification-retail-0079/FP32/person-reidentification-retail-0079.bin -O reid/reid.bin; \ fi +# TODO: CHANGE URL +ARG WITH_DEXTR +ENV WITH_DEXTR=${WITH_DEXTR} +ENV DEXTR_MODEL_DIR=${HOME}/models/dextr +RUN if [ "$WITH_DEXTR" = "yes" ]; then \ + mkdir ${DEXTR_MODEL_DIR} -p && \ + wget https://download.01.org/openvinotoolkit/models_contrib/cvat/dextr_model_v1.zip -O ${DEXTR_MODEL_DIR}/dextr.zip && \ + unzip ${DEXTR_MODEL_DIR}/dextr.zip -d ${DEXTR_MODEL_DIR} && rm ${DEXTR_MODEL_DIR}/dextr.zip; \ + fi + COPY ssh ${HOME}/.ssh COPY cvat/ ${HOME}/cvat COPY tests ${HOME}/tests diff --git a/cvat/apps/dextr_segmentation/README.md b/cvat/apps/dextr_segmentation/README.md new file mode 100644 index 000000000000..cb23aa943acb --- /dev/null +++ b/cvat/apps/dextr_segmentation/README.md @@ -0,0 +1,18 @@ +# Semi-Automatic Segmentation with [Deep Extreme Cut](http://www.vision.ee.ethz.ch/~cvlsegmentation/dextr/) + +## About the application + +The application allows to use deep learning model for semi-automatic semantic and instance segmentation. +You can get a segmentation polygon from four (or more) extreme points of object. +This application uses the pre-trained DEXTR model which has been converted to Inference Engine format. + +We are grateful to K.K. Maninis, S. Caelles, J. Pont-Tuset, and L. Van Gool who permitted to use their models in our tool. + +## Installation + +```bash +# Build image with Deep Extreme Cut application (OpenVINO component is also needed) +docker-compose -f docker-compose.yml -f components/openvino/docker-compose.openvino.yml -f cvat/apps/dextr_segmentation/docker-compose.dextr.yml build +``` + +## Running diff --git a/cvat/apps/dextr_segmentation/__init__.py b/cvat/apps/dextr_segmentation/__init__.py new file mode 100644 index 000000000000..472c2ac3c42f --- /dev/null +++ b/cvat/apps/dextr_segmentation/__init__.py @@ -0,0 +1,7 @@ +# Copyright (C) 2018 Intel Corporation +# +# SPDX-License-Identifier: MIT + +from cvat.settings.base import JS_3RDPARTY + +JS_3RDPARTY['engine'] = JS_3RDPARTY.get('engine', []) + ['dextr_segmentation/js/enginePlugin.js'] diff --git a/cvat/apps/dextr_segmentation/apps.py b/cvat/apps/dextr_segmentation/apps.py new file mode 100644 index 000000000000..d5d43a88e1c0 --- /dev/null +++ b/cvat/apps/dextr_segmentation/apps.py @@ -0,0 +1,8 @@ +# Copyright (C) 2018 Intel Corporation +# +# SPDX-License-Identifier: MIT + +from django.apps import AppConfig + +class DextrSegmentationConfig(AppConfig): + name = 'dextr_segmentation' diff --git a/cvat/apps/dextr_segmentation/dextr.py b/cvat/apps/dextr_segmentation/dextr.py new file mode 100644 index 000000000000..657137314e01 --- /dev/null +++ b/cvat/apps/dextr_segmentation/dextr.py @@ -0,0 +1,112 @@ + +# Copyright (C) 2018 Intel Corporation +# +# SPDX-License-Identifier: MIT + +from openvino.inference_engine import IENetwork, IEPlugin + +import os +import cv2 +import PIL +import numpy as np + +_IE_CPU_EXTENSION = os.getenv("IE_CPU_EXTENSION", "libcpu_extension_avx2.so") +_IE_PLUGINS_PATH = os.getenv("IE_PLUGINS_PATH", None) + +_DEXTR_MODEL_DIR = os.getenv("DEXTR_MODEL_DIR", None) +_DEXTR_PADDING = 50 +_DEXTR_TRESHOLD = 0.9 +_DEXTR_SIZE = 512 + +class DEXTR_HANDLER: + _plugin = None + _network = None + _exec_network = None + _input_blob = None + _output_blob = None + _xml = None + _bin = None + + def __init__(self): + if not _IE_PLUGINS_PATH: + raise Exception("IE_PLUGINS_PATH is not defined") + if not _DEXTR_MODEL_DIR: + raise Exception("DEXTR_MODEL_DIR is not defined") + self._xml = os.path.join(_DEXTR_MODEL_DIR, 'dextr.xml') + self._bin = os.path.join(_DEXTR_MODEL_DIR, 'dextr.bin') + + def handle(self, im_path, points): + # Lazy initialization + if not self._plugin: + self._plugin = IEPlugin(device="CPU", plugin_dirs=[_IE_PLUGINS_PATH]) + self._plugin.add_cpu_extension(os.path.join(_IE_PLUGINS_PATH, _IE_CPU_EXTENSION)) + self._network = IENetwork.from_ir(model=self._xml, weights=self._bin) + self._input_blob = next(iter(self._network.inputs)) + self._output_blob = next(iter(self._network.outputs)) + self._exec_network = self._plugin.load(network=self._network) + + image = PIL.Image.open(im_path) + numpy_image = np.array(PIL.Image.open(im_path)) + points = np.asarray([[int(p["x"]), int(p["y"])] for p in points], dtype=int) + bounding_box = ( + max(min(points[:, 0]) - _DEXTR_PADDING, 0), + max(min(points[:, 1]) - _DEXTR_PADDING, 0), + min(max(points[:, 0]) + _DEXTR_PADDING, numpy_image.shape[1] - 1), + min(max(points[:, 1]) + _DEXTR_PADDING, numpy_image.shape[0] - 1) + ) + + # Prepare an image + numpy_cropped = np.array(image.crop(bounding_box)) + resized = cv2.resize(numpy_cropped, (_DEXTR_SIZE, _DEXTR_SIZE), + interpolation = cv2.INTER_CUBIC).astype(np.float32) + + # Make a heatmap + points = points - [min(points[:, 0]), min(points[:, 1])] + [_DEXTR_PADDING, _DEXTR_PADDING] + points = (points * [_DEXTR_SIZE / numpy_cropped.shape[1], _DEXTR_SIZE / numpy_cropped.shape[0]]).astype(int) + heatmap = np.zeros(shape=resized.shape[:2], dtype=np.float64) + for point in points: + gaussian_x_axis = np.arange(0, _DEXTR_SIZE, 1, float) - point[0] + gaussian_y_axis = np.arange(0, _DEXTR_SIZE, 1, float)[:, np.newaxis] - point[1] + gaussian = np.exp(-4 * np.log(2) * ((gaussian_x_axis ** 2 + gaussian_y_axis ** 2) / 100)).astype(np.float64) + heatmap = np.maximum(heatmap, gaussian) + cv2.normalize(heatmap, heatmap, 0, 255, cv2.NORM_MINMAX) + + # Concat an image and a heatmap + input_dextr = np.concatenate((resized, heatmap[:, :, np.newaxis].astype(resized.dtype)), axis=2) + input_dextr = input_dextr.transpose((2,0,1)) + + np.set_printoptions(threshold=np.nan) + pred = self._exec_network.infer(inputs={self._input_blob: input_dextr[np.newaxis, ...]})[self._output_blob][0, 0, :, :] + pred = cv2.resize(pred, tuple(reversed(numpy_cropped.shape[:2])), interpolation = cv2.INTER_CUBIC) + result = np.zeros(numpy_image.shape[:2]) + result[bounding_box[1]:bounding_box[1] + pred.shape[0], bounding_box[0]:bounding_box[0] + pred.shape[1]] = pred > _DEXTR_TRESHOLD + + # Convert a mask to a polygon + result = np.array(result, dtype=np.uint8) + cv2.normalize(result,result,0,255,cv2.NORM_MINMAX) + contours = None + if int(cv2.__version__.split('.')[0]) > 3: + contours = cv2.findContours(result, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_TC89_KCOS)[0] + else: + contours = cv2.findContours(result, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_TC89_KCOS)[1] + + contours = max(contours, key=lambda arr: arr.size) + if contours.shape.count(1): + contours = np.squeeze(contours) + if contours.size < 3 * 2: + raise Exception('Less then three point have been detected. Can not build a polygon.') + + result = "" + for point in contours: + result += "{},{} ".format(int(point[0]), int(point[1])) + result = result[:-1] + + return result + + def __del__(self): + if self._exec_network: + del self._exec_network + if self._network: + del self._network + if self._plugin: + del self._plugin diff --git a/cvat/apps/dextr_segmentation/docker-compose.dextr.yml b/cvat/apps/dextr_segmentation/docker-compose.dextr.yml new file mode 100644 index 000000000000..468523b8a05a --- /dev/null +++ b/cvat/apps/dextr_segmentation/docker-compose.dextr.yml @@ -0,0 +1,14 @@ +# +# Copyright (C) 2018 Intel Corporation +# +# SPDX-License-Identifier: MIT +# + +version: "2.3" + +services: + cvat: + build: + context: . + args: + WITH_DEXTR: "yes" diff --git a/cvat/apps/dextr_segmentation/static/dextr_segmentation/js/enginePlugin.js b/cvat/apps/dextr_segmentation/static/dextr_segmentation/js/enginePlugin.js new file mode 100644 index 000000000000..24beb4c08f9a --- /dev/null +++ b/cvat/apps/dextr_segmentation/static/dextr_segmentation/js/enginePlugin.js @@ -0,0 +1,209 @@ +/* + * Copyright (C) 2018 Intel Corporation + * + * SPDX-License-Identifier: MIT + */ + + /* global + AREA_TRESHOLD:false + ShapeCreatorModel:false + ShapeCreatorView:false +*/ + +'use strict'; + +window.addEventListener('DOMContentLoaded', () => { + $('').appendTo('#shapeTypeSelector'); + + const dextrCancelButtonId = 'dextrCancelButton'; + const dextrOverlay = $(` + `).appendTo('body'); + + const dextrCancelButton = $(`#${dextrCancelButtonId}`); + dextrCancelButton.on('click', () => { + dextrCancelButton.prop('disabled', true); + $.ajax({ + url: `/dextr/cancel/${window.cvat.job.id}`, + type: 'GET', + error: (errorData) => { + const message = `Can not cancel segmentation. Code: ${errorData.status}. + Message: ${errorData.responseText || errorData.statusText}`; + showMessage(message); + }, + complete: () => { + dextrCancelButton.prop('disabled', false); + } + }); + }); + + function ShapeCreatorModelWrapper(originalClass) { + // Constructor will patch some properties for a created instance + function constructorDecorator() { + const instance = new originalClass(...arguments); + + // Decorator for the defaultType property + Object.defineProperty(instance, 'defaultType', { + get: function() { + return instance._defaultType; + }, + set: function(type) { + if (!['box', 'points', 'polygon', 'polyline', 'auto_segmentation'].includes(type)) { + throw Error(`Unknown shape type found ${type}`); + } + instance._defaultType = type; + } + }); + + // Decorator for finish method. + const decoratedFinish = instance.finish; + instance.finish = (result) => { + if (instance._defaultType === 'auto_segmentation') { + try { + instance._defaultType = 'polygon'; + decoratedFinish.call(instance, result); + } + finally { + instance._defaultType = 'auto_segmentation'; + } + } + else { + decoratedFinish.call(instance, result); + } + }; + + return instance; + } + + constructorDecorator.prototype = originalClass.prototype; + constructorDecorator.prototype.constructor = constructorDecorator; + return constructorDecorator; + } + + + function ShapeCreatorViewWrapper(originalClass) { + // Constructor will patch some properties for each instance + function constructorDecorator() { + const instance = new originalClass(...arguments); + + // Decorator for the _create() method. + // We save the decorated _create() and we will use it if type != 'auto_segmentation' + const decoratedCreate = instance._create; + instance._create = () => { + if (instance._type != 'auto_segmentation') { + decoratedCreate.call(instance); + return; + } + + instance._drawInstance = instance._frameContent.polyline().draw({snapToGrid: 0.1}).addClass('shapeCreation').attr({ + 'stroke-width': 0, + 'z_order': Number.MAX_SAFE_INTEGER, + }); + instance._createPolyEvents(); + + // the _createPolyEvents method have added "drawdone" event handler which invalid for this case + // because of that reason we remove the handler and create the valid handler instead + instance._drawInstance.off('drawdone').on('drawdone', (e) => { + let actualPoints = window.cvat.translate.points.canvasToActual(e.target.getAttribute('points')); + actualPoints = PolyShapeModel.convertStringToNumberArray(actualPoints); + + if (actualPoints.length < 4) { + showMessage('It is need to specify minimum four extreme points for an object'); + instance._controller.switchCreateMode(true); + return; + } + + const frameWidth = window.cvat.player.geometry.frameWidth; + const frameHeight = window.cvat.player.geometry.frameHeight; + for (let point of actualPoints) { + point.x = Math.clamp(point.x, 0, frameWidth); + point.y = Math.clamp(point.y, 0, frameHeight); + } + + e.target.setAttribute('points', window.cvat.translate.points.actualToCanvas( + PolyShapeModel.convertNumberArrayToString(actualPoints))); + + const polybox = e.target.getBBox(); + const area = polybox.width * polybox.height; + + if (area > AREA_TRESHOLD) { + $.ajax({ + url: `/dextr/create/${window.cvat.job.id}`, + type: 'POST', + data: JSON.stringify({ + frame: window.cvat.player.frames.current, + points: actualPoints + }), + contentType: 'application/json', + success: () => { + function intervalCallback() { + $.ajax({ + url: `/dextr/check/${window.cvat.job.id}`, + type: 'GET', + success: (jobData) => { + if (['queued', 'started'].includes(jobData.status)) { + if (jobData.status === 'queued') { + dextrCancelButton.prop('disabled', false); + } + setTimeout(intervalCallback, 1000); + } else { + dextrOverlay.addClass('hidden'); + if (jobData.status === 'finished') { + if (jobData.result) { + instance._controller.finish({points: jobData.result}, 'polygon'); + } + } else if (jobData.status === 'failed') { + const message = `Segmentation has fallen. Error: '${jobData.stderr}'`; + showMessage(message); + } else { + let message = `Check segmentation request returned "${jobData.status}" status.`; + if (jobData.stderr) { + message += ` Error: ${jobData.stderr}`; + } + showMessage(message); + } + } + }, + error: () => { + dextrOverlay.addClass('hidden'); + const message = `Can not check segmentation. Code: ${errorData.status}. + Message: ${errorData.responseText || errorData.statusText}`; + showMessage(message); + } + }); + } + + dextrCancelButton.prop('disabled', true); + dextrOverlay.removeClass('hidden'); + setTimeout(intervalCallback, 1000); + }, + error: (errorData) => { + const message = `Can not cancel ReID process. Code: ${errorData.status}. + Message: ${errorData.responseText || errorData.statusText}`; + showMessage(message); + } + }); + } + + instance._controller.switchCreateMode(true); + }); // end of "drawdone" handler + }; // end of _create() method + + return instance; + } // end of constructorDecorator() + + constructorDecorator.prototype = originalClass.prototype; + constructorDecorator.prototype.constructor = constructorDecorator; + return constructorDecorator; + } // end of ShapeCreatorViewWrapper + + // Apply patch for classes + ShapeCreatorModel = ShapeCreatorModelWrapper(ShapeCreatorModel); + ShapeCreatorView = ShapeCreatorViewWrapper(ShapeCreatorView); +}); diff --git a/cvat/apps/dextr_segmentation/urls.py b/cvat/apps/dextr_segmentation/urls.py new file mode 100644 index 000000000000..82e415bd7f67 --- /dev/null +++ b/cvat/apps/dextr_segmentation/urls.py @@ -0,0 +1,12 @@ +# Copyright (C) 2018 Intel Corporation +# +# SPDX-License-Identifier: MIT + +from django.urls import path +from . import views + +urlpatterns = [ + path('create/', views.create), + path('cancel/', views.cancel), + path('check/', views.check) +] \ No newline at end of file diff --git a/cvat/apps/dextr_segmentation/views.py b/cvat/apps/dextr_segmentation/views.py new file mode 100644 index 000000000000..de99caeedb5a --- /dev/null +++ b/cvat/apps/dextr_segmentation/views.py @@ -0,0 +1,128 @@ +# Copyright (C) 2018 Intel Corporation +# +# SPDX-License-Identifier: MIT + +from django.http import HttpResponse, HttpResponseBadRequest, JsonResponse +from cvat.apps.authentication.decorators import login_required +from rules.contrib.views import permission_required, objectgetter + +from cvat.apps.engine.models import Job +from cvat.apps.engine.log import slogger +from cvat.apps.engine.task import get_frame_path +from cvat.apps.dextr_segmentation.dextr import DEXTR_HANDLER + +import django_rq +import json +import os +import rq + +__RQ_QUEUE_NAME = "default" +__DEXTR_HANDLER = DEXTR_HANDLER() + +def _dextr_thread(im_path, points): + job = rq.get_current_job() + job.meta["result"] = __DEXTR_HANDLER.handle(im_path, points) + job.save_meta() + + +@login_required +@permission_required(perm=["engine.job.change"], + fn=objectgetter(Job, "jid"), raise_exception=True) +def create(request, jid): + try: + data = json.loads(request.body.decode("utf-8")) + + points = data["points"] + frame = int(data["frame"]) + username = request.user.username + + slogger.job[jid].info("create dextr request for the JOB: {} ".format(jid) + + "by the USER: {} on the FRAME: {}".format(username, frame)) + + tid = Job.objects.select_related("segment__task").get(id=jid).segment.task.id + im_path = os.path.realpath(get_frame_path(tid, frame)) + + queue = django_rq.get_queue(__RQ_QUEUE_NAME) + rq_id = "dextr.create/{}/{}".format(jid, username) + job = queue.fetch_job(rq_id) + + if job is not None and (job.is_started or job.is_queued): + if "cancel" not in job.meta: + raise Exception("Segmentation process has been already run for the " + + "JOB: {} and the USER: {}".format(jid, username)) + else: + job.delete() + + queue.enqueue_call(func=_dextr_thread, + args=(im_path, points), + job_id=rq_id, + timeout=15, + ttl=30) + + return HttpResponse() + except Exception as ex: + slogger.job[jid].error("can't create a dextr request for the job {}".format(jid), exc_info=True) + return HttpResponseBadRequest(str(ex)) + + +@login_required +@permission_required(perm=["engine.job.change"], + fn=objectgetter(Job, "jid"), raise_exception=True) +def cancel(request, jid): + try: + username = request.user.username + slogger.job[jid].info("cancel dextr request for the JOB: {} ".format(jid) + + "by the USER: {}".format(username)) + + queue = django_rq.get_queue(__RQ_QUEUE_NAME) + rq_id = "dextr.create/{}/{}".format(jid, username) + job = queue.fetch_job(rq_id) + + if job is None or job.is_finished or job.is_failed: + raise Exception("Segmentation isn't running now") + elif "cancel" not in job.meta: + job.meta["cancel"] = True + job.save_meta() + + return HttpResponse() + except Exception as ex: + slogger.job[jid].error("can't cancel a dextr request for the job {}".format(jid), exc_info=True) + return HttpResponseBadRequest(str(ex)) + + +@login_required +@permission_required(perm=["engine.job.change"], + fn=objectgetter(Job, "jid"), raise_exception=True) +def check(request, jid): + try: + username = request.user.username + slogger.job[jid].info("check dextr request for the JOB: {} ".format(jid) + + "by the USER: {}".format(username)) + + queue = django_rq.get_queue(__RQ_QUEUE_NAME) + rq_id = "dextr.create/{}/{}".format(jid, username) + job = queue.fetch_job(rq_id) + data = {} + + if job is None: + data["status"] = "unknown" + else: + if "cancel" in job.meta: + data["status"] = "finished" + elif job.is_queued: + data["status"] = "queued" + elif job.is_started: + data["status"] = "started" + elif job.is_finished: + data["status"] = "finished" + data["result"] = job.meta["result"] + job.delete() + else: + data["status"] = "failed" + data["stderr"] = job.exc_info + job.delete() + + return JsonResponse(data) + except Exception as ex: + slogger.job[jid].error("can't check a dextr request for the job {}".format(jid), exc_info=True) + return HttpResponseBadRequest(str(ex)) diff --git a/cvat/settings/base.py b/cvat/settings/base.py index 5c70bb710665..7ba1e668664d 100644 --- a/cvat/settings/base.py +++ b/cvat/settings/base.py @@ -110,6 +110,9 @@ def generate_ssh_keys(): if 'yes' == os.environ.get('OPENVINO_TOOLKIT', 'no'): INSTALLED_APPS += ['cvat.apps.reid'] +if 'yes' == os.environ.get('WITH_DEXTR', 'no'): + INSTALLED_APPS += ['cvat.apps.dextr_segmentation'] + if os.getenv('DJANGO_LOG_VIEWER_HOST'): INSTALLED_APPS += ['cvat.apps.log_viewer'] diff --git a/cvat/urls.py b/cvat/urls.py index a0e2769cbf93..e35ea21566c1 100644 --- a/cvat/urls.py +++ b/cvat/urls.py @@ -46,6 +46,9 @@ if apps.is_installed('cvat.apps.auto_annotation'): urlpatterns.append(path('auto_annotation/', include('cvat.apps.auto_annotation.urls'))) +if apps.is_installed('cvat.apps.dextr_segmentation'): + urlpatterns.append(path('dextr/', include('cvat.apps.dextr_segmentation.urls'))) + if apps.is_installed('cvat.apps.log_viewer'): urlpatterns.append(path('analytics/', include('cvat.apps.log_viewer.urls'))) From c995807deba2e49815f07f55c998feb04f2bf772 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Mon, 25 Feb 2019 13:49:29 +0300 Subject: [PATCH 2/8] Codacy fixes --- README.md | 33 +++---- cvat/apps/dextr_segmentation/README.md | 10 ++- .../dextr_segmentation/js/enginePlugin.js | 86 ++++++++++--------- 3 files changed, 71 insertions(+), 58 deletions(-) diff --git a/README.md b/README.md index 9ce2e3fc296c..155507e9eaa1 100644 --- a/README.md +++ b/README.md @@ -10,19 +10,19 @@ CVAT is completely re-designed and re-implemented version of [Video Annotation T ## Documentation -- [User's guide](cvat/apps/documentation/user_guide.md) -- [XML annotation format](cvat/apps/documentation/xml_format.md) -- [AWS Deployment Guide](cvat/apps/documentation/AWS-Deployment-Guide.md) -- [Questions](#questions) +-[User's guide](cvat/apps/documentation/user_guide.md) +-[XML annotation format](cvat/apps/documentation/xml_format.md) +-[AWS Deployment Guide](cvat/apps/documentation/AWS-Deployment-Guide.md) +-[Questions](#questions) ## Screencasts -- [Introduction](https://youtu.be/L9_IvUIHGwM) -- [Annotation mode](https://youtu.be/6h7HxGL6Ct4) -- [Interpolation mode](https://youtu.be/U3MYDhESHo4) -- [Attribute mode](https://youtu.be/UPNfWl8Egd8) -- [Segmentation mode](https://youtu.be/Fh8oKuSUIPs) -- [Tutorial for polygons](https://www.youtube.com/watch?v=XTwfXDh4clI) +-[Introduction](https://youtu.be/L9_IvUIHGwM) +-[Annotation mode](https://youtu.be/6h7HxGL6Ct4) +-[Interpolation mode](https://youtu.be/U3MYDhESHo4) +-[Attribute mode](https://youtu.be/UPNfWl8Egd8) +-[Segmentation mode](https://youtu.be/Fh8oKuSUIPs) +-[Tutorial for polygons](https://www.youtube.com/watch?v=XTwfXDh4clI) ## Online Demo @@ -30,8 +30,8 @@ CVAT is completely re-designed and re-implemented version of [Video Annotation T After you click the link above: -- Click on "GO TO WORKSPACE" and the CVAT environment will load up -- The environment is backed by a K80 GPU +-Click on "GO TO WORKSPACE" and the CVAT environment will load up +-The environment is backed by a K80 GPU If you have any questions, please contact Onepanel directly at support@onepanel.io. If you are in the Onepanel application, you can also use the chat icon in the bottom right corner. @@ -73,10 +73,11 @@ docker-compose -f docker-compose.yml -f components/cuda/docker-compose.cuda.yml ### Additional optional components -- [Support for Intel OpenVINO: auto annotation](components/openvino/README.md) -- [Analytics: management and monitoring of data annotation team](components/analytics/README.md) -- [TF Object Detection API: auto annotation](components/tf_annotation/README.md) -- [Support for NVIDIA GPUs](components/cuda/README.md) +-[Support for Intel OpenVINO: auto annotation](components/openvino/README.md) +-[Analytics: management and monitoring of data annotation team](components/analytics/README.md) +-[TF Object Detection API: auto annotation](components/tf_annotation/README.md) +-[Support for NVIDIA GPUs](components/cuda/README.md) +-[Semi-automatic segmentation with Deep Extreme Cut](cvat/apps/dextr_segmentation/README.md) ### Create superuser account diff --git a/cvat/apps/dextr_segmentation/README.md b/cvat/apps/dextr_segmentation/README.md index cb23aa943acb..a25f828d1733 100644 --- a/cvat/apps/dextr_segmentation/README.md +++ b/cvat/apps/dextr_segmentation/README.md @@ -6,7 +6,7 @@ The application allows to use deep learning model for semi-automatic semantic an You can get a segmentation polygon from four (or more) extreme points of object. This application uses the pre-trained DEXTR model which has been converted to Inference Engine format. -We are grateful to K.K. Maninis, S. Caelles, J. Pont-Tuset, and L. Van Gool who permitted to use their models in our tool. +We are grateful to K.K. Maninis, S. Caelles, J. Pont-Tuset, and L. Van Gool who permitted using their models in our tool ## Installation @@ -16,3 +16,11 @@ docker-compose -f docker-compose.yml -f components/openvino/docker-compose.openv ``` ## Running + +1. Open a job +2. Select "Auto Segmentation" in the list of shapes +3. Run the draw mode as usually (by press the "Create Shape" button or by "N" shortcut) +4. Click four-six (or more if need) extreme points of an object +5. Close the draw mode (the same way like start it) +6. Wait a moment and you will receive class agnostic annotation polygon +7. You can close an annotation request if it is so long (in case if it is queued to rq worker and all workers are busy) diff --git a/cvat/apps/dextr_segmentation/static/dextr_segmentation/js/enginePlugin.js b/cvat/apps/dextr_segmentation/static/dextr_segmentation/js/enginePlugin.js index 24beb4c08f9a..49dda020d080 100644 --- a/cvat/apps/dextr_segmentation/static/dextr_segmentation/js/enginePlugin.js +++ b/cvat/apps/dextr_segmentation/static/dextr_segmentation/js/enginePlugin.js @@ -4,13 +4,15 @@ * SPDX-License-Identifier: MIT */ - /* global +/* global AREA_TRESHOLD:false - ShapeCreatorModel:false - ShapeCreatorView:false + PolyShapeModel:false + ShapeCreatorModel:true + ShapeCreatorView:true + showMessage:false */ -'use strict'; +/* eslint no-underscore-dangle: 0 */ window.addEventListener('DOMContentLoaded', () => { $('').appendTo('#shapeTypeSelector'); @@ -39,26 +41,24 @@ window.addEventListener('DOMContentLoaded', () => { }, complete: () => { dextrCancelButton.prop('disabled', false); - } + }, }); }); - function ShapeCreatorModelWrapper(originalClass) { + function ShapeCreatorModelWrapper(OriginalClass) { // Constructor will patch some properties for a created instance - function constructorDecorator() { - const instance = new originalClass(...arguments); + function constructorDecorator(...args) { + const instance = new OriginalClass(...args); // Decorator for the defaultType property Object.defineProperty(instance, 'defaultType', { - get: function() { - return instance._defaultType; - }, - set: function(type) { + get: () => instance._defaultType, + set: (type) => { if (!['box', 'points', 'polygon', 'polyline', 'auto_segmentation'].includes(type)) { throw Error(`Unknown shape type found ${type}`); } instance._defaultType = type; - } + }, }); // Decorator for finish method. @@ -68,12 +68,10 @@ window.addEventListener('DOMContentLoaded', () => { try { instance._defaultType = 'polygon'; decoratedFinish.call(instance, result); - } - finally { + } finally { instance._defaultType = 'auto_segmentation'; } - } - else { + } else { decoratedFinish.call(instance, result); } }; @@ -81,34 +79,37 @@ window.addEventListener('DOMContentLoaded', () => { return instance; } - constructorDecorator.prototype = originalClass.prototype; + constructorDecorator.prototype = OriginalClass.prototype; constructorDecorator.prototype.constructor = constructorDecorator; return constructorDecorator; } - function ShapeCreatorViewWrapper(originalClass) { + function ShapeCreatorViewWrapper(OriginalClass) { // Constructor will patch some properties for each instance - function constructorDecorator() { - const instance = new originalClass(...arguments); + function constructorDecorator(...args) { + const instance = new OriginalClass(...args); // Decorator for the _create() method. // We save the decorated _create() and we will use it if type != 'auto_segmentation' const decoratedCreate = instance._create; instance._create = () => { - if (instance._type != 'auto_segmentation') { + if (instance._type !== 'auto_segmentation') { decoratedCreate.call(instance); return; } - instance._drawInstance = instance._frameContent.polyline().draw({snapToGrid: 0.1}).addClass('shapeCreation').attr({ + instance._drawInstance = instance._frameContent.polyline().draw({ snapToGrid: 0.1 }).addClass('shapeCreation').attr({ 'stroke-width': 0, - 'z_order': Number.MAX_SAFE_INTEGER, + z_order: Number.MAX_SAFE_INTEGER, }); instance._createPolyEvents(); - // the _createPolyEvents method have added "drawdone" event handler which invalid for this case - // because of that reason we remove the handler and create the valid handler instead + /* the _createPolyEvents method have added "drawdone" + * event handler which invalid for this case + * because of that reason we remove the handler and + * create the valid handler instead + */ instance._drawInstance.off('drawdone').on('drawdone', (e) => { let actualPoints = window.cvat.translate.points.canvasToActual(e.target.getAttribute('points')); actualPoints = PolyShapeModel.convertStringToNumberArray(actualPoints); @@ -119,15 +120,18 @@ window.addEventListener('DOMContentLoaded', () => { return; } - const frameWidth = window.cvat.player.geometry.frameWidth; - const frameHeight = window.cvat.player.geometry.frameHeight; - for (let point of actualPoints) { + const { frameWidth } = window.cvat.player.geometry; + const { frameHeight } = window.cvat.player.geometry; + for (let idx = 0; idx < actualPoints.length; idx++) { + const point = actualPoints[idx]; point.x = Math.clamp(point.x, 0, frameWidth); point.y = Math.clamp(point.y, 0, frameHeight); } - e.target.setAttribute('points', window.cvat.translate.points.actualToCanvas( - PolyShapeModel.convertNumberArrayToString(actualPoints))); + e.target.setAttribute('points', + window.cvat.translate.points.actualToCanvas( + PolyShapeModel.convertNumberArrayToString(actualPoints), + )); const polybox = e.target.getBBox(); const area = polybox.width * polybox.height; @@ -138,7 +142,7 @@ window.addEventListener('DOMContentLoaded', () => { type: 'POST', data: JSON.stringify({ frame: window.cvat.player.frames.current, - points: actualPoints + points: actualPoints, }), contentType: 'application/json', success: () => { @@ -156,7 +160,7 @@ window.addEventListener('DOMContentLoaded', () => { dextrOverlay.addClass('hidden'); if (jobData.status === 'finished') { if (jobData.result) { - instance._controller.finish({points: jobData.result}, 'polygon'); + instance._controller.finish({ points: jobData.result }, 'polygon'); } } else if (jobData.status === 'failed') { const message = `Segmentation has fallen. Error: '${jobData.stderr}'`; @@ -170,12 +174,12 @@ window.addEventListener('DOMContentLoaded', () => { } } }, - error: () => { + error: (errorData) => { dextrOverlay.addClass('hidden'); - const message = `Can not check segmentation. Code: ${errorData.status}. - Message: ${errorData.responseText || errorData.statusText}`; + const message = `Can not check segmentation. Code: ${errorData.status}.` + + ` Message: ${errorData.responseText || errorData.statusText}`; showMessage(message); - } + }, }); } @@ -184,10 +188,10 @@ window.addEventListener('DOMContentLoaded', () => { setTimeout(intervalCallback, 1000); }, error: (errorData) => { - const message = `Can not cancel ReID process. Code: ${errorData.status}. - Message: ${errorData.responseText || errorData.statusText}`; + const message = `Can not cancel ReID process. Code: ${errorData.status}.` + + ` Message: ${errorData.responseText || errorData.statusText}`; showMessage(message); - } + }, }); } @@ -198,7 +202,7 @@ window.addEventListener('DOMContentLoaded', () => { return instance; } // end of constructorDecorator() - constructorDecorator.prototype = originalClass.prototype; + constructorDecorator.prototype = OriginalClass.prototype; constructorDecorator.prototype.constructor = constructorDecorator; return constructorDecorator; } // end of ShapeCreatorViewWrapper From 2c40270310bf31c91895af59b9e005d6ae12b65d Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Mon, 25 Feb 2019 14:23:15 +0300 Subject: [PATCH 3/8] Codacy fixes --- .../static/dextr_segmentation/js/enginePlugin.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cvat/apps/dextr_segmentation/static/dextr_segmentation/js/enginePlugin.js b/cvat/apps/dextr_segmentation/static/dextr_segmentation/js/enginePlugin.js index 49dda020d080..548f19f64aac 100644 --- a/cvat/apps/dextr_segmentation/static/dextr_segmentation/js/enginePlugin.js +++ b/cvat/apps/dextr_segmentation/static/dextr_segmentation/js/enginePlugin.js @@ -122,7 +122,7 @@ window.addEventListener('DOMContentLoaded', () => { const { frameWidth } = window.cvat.player.geometry; const { frameHeight } = window.cvat.player.geometry; - for (let idx = 0; idx < actualPoints.length; idx++) { + for (let idx = 0; idx < actualPoints.length; idx += 1) { const point = actualPoints[idx]; point.x = Math.clamp(point.x, 0, frameWidth); point.y = Math.clamp(point.y, 0, frameHeight); @@ -131,7 +131,7 @@ window.addEventListener('DOMContentLoaded', () => { e.target.setAttribute('points', window.cvat.translate.points.actualToCanvas( PolyShapeModel.convertNumberArrayToString(actualPoints), - )); + )); const polybox = e.target.getBBox(); const area = polybox.width * polybox.height; From dff3e8271d4abc946eaab49adbbf3064101bce9a Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Wed, 27 Feb 2019 15:49:25 +0300 Subject: [PATCH 4/8] Fixed readme --- README.md | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 155507e9eaa1..c12ec8c0b2a3 100644 --- a/README.md +++ b/README.md @@ -10,19 +10,19 @@ CVAT is completely re-designed and re-implemented version of [Video Annotation T ## Documentation --[User's guide](cvat/apps/documentation/user_guide.md) --[XML annotation format](cvat/apps/documentation/xml_format.md) --[AWS Deployment Guide](cvat/apps/documentation/AWS-Deployment-Guide.md) --[Questions](#questions) +- [User's guide](cvat/apps/documentation/user_guide.md) +- [XML annotation format](cvat/apps/documentation/xml_format.md) +- [AWS Deployment Guide](cvat/apps/documentation/AWS-Deployment-Guide.md) +- [Questions](#questions) ## Screencasts --[Introduction](https://youtu.be/L9_IvUIHGwM) --[Annotation mode](https://youtu.be/6h7HxGL6Ct4) --[Interpolation mode](https://youtu.be/U3MYDhESHo4) --[Attribute mode](https://youtu.be/UPNfWl8Egd8) --[Segmentation mode](https://youtu.be/Fh8oKuSUIPs) --[Tutorial for polygons](https://www.youtube.com/watch?v=XTwfXDh4clI) +- [Introduction](https://youtu.be/L9_IvUIHGwM) +- [Annotation mode](https://youtu.be/6h7HxGL6Ct4) +- [Interpolation mode](https://youtu.be/U3MYDhESHo4) +- [Attribute mode](https://youtu.be/UPNfWl8Egd8) +- [Segmentation mode](https://youtu.be/Fh8oKuSUIPs) +- [Tutorial for polygons](https://www.youtube.com/watch?v=XTwfXDh4clI) ## Online Demo @@ -30,8 +30,8 @@ CVAT is completely re-designed and re-implemented version of [Video Annotation T After you click the link above: --Click on "GO TO WORKSPACE" and the CVAT environment will load up --The environment is backed by a K80 GPU +- Click on "GO TO WORKSPACE" and the CVAT environment will load up +- The environment is backed by a K80 GPU If you have any questions, please contact Onepanel directly at support@onepanel.io. If you are in the Onepanel application, you can also use the chat icon in the bottom right corner. @@ -73,11 +73,11 @@ docker-compose -f docker-compose.yml -f components/cuda/docker-compose.cuda.yml ### Additional optional components --[Support for Intel OpenVINO: auto annotation](components/openvino/README.md) --[Analytics: management and monitoring of data annotation team](components/analytics/README.md) --[TF Object Detection API: auto annotation](components/tf_annotation/README.md) --[Support for NVIDIA GPUs](components/cuda/README.md) --[Semi-automatic segmentation with Deep Extreme Cut](cvat/apps/dextr_segmentation/README.md) +- [Support for Intel OpenVINO: auto annotation](components/openvino/README.md) +- [Analytics: management and monitoring of data annotation team](components/analytics/README.md) +- [TF Object Detection API: auto annotation](components/tf_annotation/README.md) +- [Support for NVIDIA GPUs](components/cuda/README.md) +- [Semi-automatic segmentation with Deep Extreme Cut](cvat/apps/dextr_segmentation/README.md) ### Create superuser account From 7c6118e6a7f5eb7e45ede502139058dc35a83abc Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Wed, 20 Mar 2019 15:12:38 +0300 Subject: [PATCH 5/8] Shared code to create OpenVINO models. Changed README --- cvat/apps/auto_annotation/model_loader.py | 18 ++--------- cvat/apps/dextr_segmentation/README.md | 24 +++++++++------ cvat/apps/dextr_segmentation/dextr.py | 26 ++++++---------- cvat/apps/openvino/ir.py | 37 +++++++++++++++++++++++ cvat/apps/tf_annotation/views.py | 27 +++-------------- 5 files changed, 68 insertions(+), 64 deletions(-) create mode 100644 cvat/apps/openvino/ir.py diff --git a/cvat/apps/auto_annotation/model_loader.py b/cvat/apps/auto_annotation/model_loader.py index 4359d997b893..47908f403306 100644 --- a/cvat/apps/auto_annotation/model_loader.py +++ b/cvat/apps/auto_annotation/model_loader.py @@ -8,7 +8,7 @@ import os import subprocess -from openvino.inference_engine import IENetwork, IEPlugin +from cvat.apps.openvino.ir import make_plugin, make_network class ModelLoader(): def __init__(self, model, weights): @@ -19,15 +19,9 @@ def __init__(self, model, weights): if not IE_PLUGINS_PATH: raise OSError("Inference engine plugin path env not found in the system.") - plugin = IEPlugin(device="CPU", plugin_dirs=[IE_PLUGINS_PATH]) - if (self._check_instruction("avx2")): - plugin.add_cpu_extension(os.path.join(IE_PLUGINS_PATH, "libcpu_extension_avx2.so")) - elif (self._check_instruction("sse4")): - plugin.add_cpu_extension(os.path.join(IE_PLUGINS_PATH, "libcpu_extension_sse4.so")) - else: - raise Exception("Inference engine requires a support of avx2 or sse4.") + plugin = make_plugin() + network = make_network(self._model, self._weights) - network = IENetwork.from_ir(model=self._model, weights=self._weights) supported_layers = plugin.get_supported_layers(network) not_supported_layers = [l for l in network.layers.keys() if l not in supported_layers] if len(not_supported_layers) != 0: @@ -47,12 +41,6 @@ def infer(self, image): in_frame = in_frame.transpose((2, 0, 1)) # Change data layout from HWC to CHW return self._net.infer(inputs={self._input_blob_name: in_frame})[self._output_blob_name].copy() - @staticmethod - def _check_instruction(instruction): - return instruction == str.strip( - subprocess.check_output( - "lscpu | grep -o \"{}\" | head -1".format(instruction), shell=True - ).decode("utf-8")) def load_label_map(labels_path): with open(labels_path, "r") as f: diff --git a/cvat/apps/dextr_segmentation/README.md b/cvat/apps/dextr_segmentation/README.md index a25f828d1733..3a80595a156b 100644 --- a/cvat/apps/dextr_segmentation/README.md +++ b/cvat/apps/dextr_segmentation/README.md @@ -2,25 +2,29 @@ ## About the application -The application allows to use deep learning model for semi-automatic semantic and instance segmentation. -You can get a segmentation polygon from four (or more) extreme points of object. +The application allows to use deep learning models for semi-automatic semantic and instance segmentation. +You can get a segmentation polygon from four (or more) extreme points of an object. This application uses the pre-trained DEXTR model which has been converted to Inference Engine format. We are grateful to K.K. Maninis, S. Caelles, J. Pont-Tuset, and L. Van Gool who permitted using their models in our tool -## Installation - +## Build docker image ```bash -# Build image with Deep Extreme Cut application (OpenVINO component is also needed) +# OpenVINO component is also needed docker-compose -f docker-compose.yml -f components/openvino/docker-compose.openvino.yml -f cvat/apps/dextr_segmentation/docker-compose.dextr.yml build ``` -## Running +## Run docker container +```bash +docker-compose -f docker-compose.yml -f components/openvino/docker-compose.openvino.yml -f cvat/apps/dextr_segmentation/docker-compose.dextr.yml up -d +``` + +## Using 1. Open a job 2. Select "Auto Segmentation" in the list of shapes 3. Run the draw mode as usually (by press the "Create Shape" button or by "N" shortcut) -4. Click four-six (or more if need) extreme points of an object -5. Close the draw mode (the same way like start it) -6. Wait a moment and you will receive class agnostic annotation polygon -7. You can close an annotation request if it is so long (in case if it is queued to rq worker and all workers are busy) +4. Click four-six (or more if it's need) extreme points of an object +5. Close the draw mode as usually (by shortcut or pressing the button "Stop Creation") +6. Wait a moment and you will get a class agnostic annotation polygon +7. You can close an annotation request if it is too long (in case if it is queued to rq worker and all workers are busy) diff --git a/cvat/apps/dextr_segmentation/dextr.py b/cvat/apps/dextr_segmentation/dextr.py index 5414ef0b98b5..e80eb77dace8 100644 --- a/cvat/apps/dextr_segmentation/dextr.py +++ b/cvat/apps/dextr_segmentation/dextr.py @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: MIT -from openvino.inference_engine import IENetwork, IEPlugin +from cvat.apps.openvino.ir import make_plugin, make_network import os import cv2 @@ -19,28 +19,22 @@ _DEXTR_SIZE = 512 class DEXTR_HANDLER: - _plugin = None - _network = None - _exec_network = None - _input_blob = None - _output_blob = None - _xml = None - _bin = None - def __init__(self): - if not _IE_PLUGINS_PATH: - raise Exception("IE_PLUGINS_PATH is not defined") + self._plugin = None + self._network = None + self._exec_network = None + self._input_blob = None + self._output_blob = None if not _DEXTR_MODEL_DIR: raise Exception("DEXTR_MODEL_DIR is not defined") - self._xml = os.path.join(_DEXTR_MODEL_DIR, 'dextr.xml') - self._bin = os.path.join(_DEXTR_MODEL_DIR, 'dextr.bin') + def handle(self, im_path, points): # Lazy initialization if not self._plugin: - self._plugin = IEPlugin(device="CPU", plugin_dirs=[_IE_PLUGINS_PATH]) - self._plugin.add_cpu_extension(os.path.join(_IE_PLUGINS_PATH, _IE_CPU_EXTENSION)) - self._network = IENetwork.from_ir(model=self._xml, weights=self._bin) + self._plugin = make_plugin() + self._network = make_network(os.path.join(_DEXTR_MODEL_DIR, 'dextr.xml'), + os.path.join(_DEXTR_MODEL_DIR, 'dextr.bin')) self._input_blob = next(iter(self._network.inputs)) self._output_blob = next(iter(self._network.outputs)) self._exec_network = self._plugin.load(network=self._network) diff --git a/cvat/apps/openvino/ir.py b/cvat/apps/openvino/ir.py new file mode 100644 index 000000000000..6961511bd451 --- /dev/null +++ b/cvat/apps/openvino/ir.py @@ -0,0 +1,37 @@ +# Copyright (C) 2018 Intel Corporation +# +# SPDX-License-Identifier: MIT + +from openvino.inference_engine import IENetwork, IEPlugin + +import subprocess +import os + +_IE_PLUGINS_PATH = os.getenv("IE_PLUGINS_PATH", None) + + +def _check_instruction(instruction): + return instruction == str.strip( + subprocess.check_output( + 'lscpu | grep -o "{}" | head -1'.format(instruction), shell=True + ).decode('utf-8') + ) + + +def make_plugin(): + if _IE_PLUGINS_PATH is None: + raise OSError('Inference engine plugin path env not found in the system.') + + plugin = IEPlugin(device='CPU', plugin_dirs=[_IE_PLUGINS_PATH]) + if (_check_instruction('avx2')): + plugin.add_cpu_extension(os.path.join(_IE_PLUGINS_PATH, 'libcpu_extension_avx2.so')) + elif (_check_instruction('sse4')): + plugin.add_cpu_extension(os.path.join(_IE_PLUGINS_PATH, 'libcpu_extension_sse4.so')) + else: + raise Exception('Inference engine requires a support of avx2 or sse4.') + + return plugin + + +def make_network(model, weights): + return IENetwork.from_ir(model = model, weights = weights) diff --git a/cvat/apps/tf_annotation/views.py b/cvat/apps/tf_annotation/views.py index dbad68a7b60c..6bbc5d8172ec 100644 --- a/cvat/apps/tf_annotation/views.py +++ b/cvat/apps/tf_annotation/views.py @@ -8,11 +8,12 @@ from django.shortcuts import render from rules.contrib.views import permission_required, objectgetter from cvat.apps.authentication.decorators import login_required +from cvat.apps.openvino.ir import make_plugin, make_network from cvat.apps.engine.models import Task as TaskModel from cvat.apps.engine import annotation, task + import django_rq -import subprocess import fnmatch import logging import json @@ -25,8 +26,6 @@ from PIL import Image from cvat.apps.engine.log import slogger -if os.environ.get('OPENVINO_TOOLKIT') == 'yes': - from openvino.inference_engine import IENetwork, IEPlugin def load_image_into_numpy(image): (im_width, im_height) = image.size @@ -34,13 +33,6 @@ def load_image_into_numpy(image): def run_inference_engine_annotation(image_list, labels_mapping, treshold): - def _check_instruction(instruction): - return instruction == str.strip( - subprocess.check_output( - 'lscpu | grep -o "{}" | head -1'.format(instruction), shell=True - ).decode('utf-8') - ) - def _normalize_box(box, w, h, dw, dh): xmin = min(int(box[0] * dw * w), w) ymin = min(int(box[1] * dh * h), h) @@ -53,19 +45,8 @@ def _normalize_box(box, w, h, dw, dh): if MODEL_PATH is None: raise OSError('Model path env not found in the system.') - IE_PLUGINS_PATH = os.getenv('IE_PLUGINS_PATH') - if IE_PLUGINS_PATH is None: - raise OSError('Inference engine plugin path env not found in the system.') - - plugin = IEPlugin(device='CPU', plugin_dirs=[IE_PLUGINS_PATH]) - if (_check_instruction('avx2')): - plugin.add_cpu_extension(os.path.join(IE_PLUGINS_PATH, 'libcpu_extension_avx2.so')) - elif (_check_instruction('sse4')): - plugin.add_cpu_extension(os.path.join(IE_PLUGINS_PATH, 'libcpu_extension_sse4.so')) - else: - raise Exception('Inference engine requires a support of avx2 or sse4.') - - network = IENetwork.from_ir(model = MODEL_PATH + '.xml', weights = MODEL_PATH + '.bin') + plugin = make_plugin() + network = make_network('{}.xml'.format(MODEL_PATH), '{}.bin'.format(MODEL_PATH)) input_blob_name = next(iter(network.inputs)) output_blob_name = next(iter(network.outputs)) executable_network = plugin.load(network=network) From a9554ac38ff5bbc688b92ab94c68ecf2a7f10d17 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Wed, 20 Mar 2019 16:00:37 +0300 Subject: [PATCH 6/8] Changed filename --- .../{openvino/ir.py => auto_annotation/inference_engine.py} | 0 cvat/apps/auto_annotation/model_loader.py | 2 +- cvat/apps/dextr_segmentation/dextr.py | 2 +- cvat/apps/tf_annotation/views.py | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename cvat/apps/{openvino/ir.py => auto_annotation/inference_engine.py} (100%) diff --git a/cvat/apps/openvino/ir.py b/cvat/apps/auto_annotation/inference_engine.py similarity index 100% rename from cvat/apps/openvino/ir.py rename to cvat/apps/auto_annotation/inference_engine.py diff --git a/cvat/apps/auto_annotation/model_loader.py b/cvat/apps/auto_annotation/model_loader.py index 47908f403306..ee83714fd1f7 100644 --- a/cvat/apps/auto_annotation/model_loader.py +++ b/cvat/apps/auto_annotation/model_loader.py @@ -8,7 +8,7 @@ import os import subprocess -from cvat.apps.openvino.ir import make_plugin, make_network +from cvat.apps.auto_annotation.inference_engine import make_plugin, make_network class ModelLoader(): def __init__(self, model, weights): diff --git a/cvat/apps/dextr_segmentation/dextr.py b/cvat/apps/dextr_segmentation/dextr.py index e80eb77dace8..2affc3379f61 100644 --- a/cvat/apps/dextr_segmentation/dextr.py +++ b/cvat/apps/dextr_segmentation/dextr.py @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: MIT -from cvat.apps.openvino.ir import make_plugin, make_network +cvat.apps.auto_annotation.inference_engine import make_plugin, make_network import os import cv2 diff --git a/cvat/apps/tf_annotation/views.py b/cvat/apps/tf_annotation/views.py index 6bbc5d8172ec..32a629bf8048 100644 --- a/cvat/apps/tf_annotation/views.py +++ b/cvat/apps/tf_annotation/views.py @@ -8,7 +8,7 @@ from django.shortcuts import render from rules.contrib.views import permission_required, objectgetter from cvat.apps.authentication.decorators import login_required -from cvat.apps.openvino.ir import make_plugin, make_network +from cvat.apps.auto_annotation.inference_engine import make_plugin, make_network from cvat.apps.engine.models import Task as TaskModel from cvat.apps.engine import annotation, task From 5bc99975213f314f8b65329d7ae07607f23f9da8 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Wed, 20 Mar 2019 16:27:14 +0300 Subject: [PATCH 7/8] Fixed import --- cvat/apps/dextr_segmentation/dextr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvat/apps/dextr_segmentation/dextr.py b/cvat/apps/dextr_segmentation/dextr.py index 2affc3379f61..35c9b64fa2c7 100644 --- a/cvat/apps/dextr_segmentation/dextr.py +++ b/cvat/apps/dextr_segmentation/dextr.py @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: MIT -cvat.apps.auto_annotation.inference_engine import make_plugin, make_network +from cvat.apps.auto_annotation.inference_engine import make_plugin, make_network import os import cv2 From 3965e5e9c0bb1f18da2068fb70c06ccbe0362c90 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Wed, 20 Mar 2019 16:29:21 +0300 Subject: [PATCH 8/8] Fixed README --- cvat/apps/dextr_segmentation/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cvat/apps/dextr_segmentation/README.md b/cvat/apps/dextr_segmentation/README.md index 3a80595a156b..2382a380861d 100644 --- a/cvat/apps/dextr_segmentation/README.md +++ b/cvat/apps/dextr_segmentation/README.md @@ -27,4 +27,5 @@ docker-compose -f docker-compose.yml -f components/openvino/docker-compose.openv 4. Click four-six (or more if it's need) extreme points of an object 5. Close the draw mode as usually (by shortcut or pressing the button "Stop Creation") 6. Wait a moment and you will get a class agnostic annotation polygon -7. You can close an annotation request if it is too long (in case if it is queued to rq worker and all workers are busy) +7. You can close an annotation request if it is too long +(in case if it is queued to rq worker and all workers are busy)