diff --git a/appinfo/info.xml b/appinfo/info.xml index e0dd641b..8f5fa5ef 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -46,6 +46,7 @@ OCA\FaceRecognition\Command\BackgroundCommand + OCA\FaceRecognition\Command\MigrateCommand OCA\FaceRecognition\Command\ProgressCommand OCA\FaceRecognition\Command\ResetCommand OCA\FaceRecognition\Command\SetupCommand diff --git a/lib/BackgroundJob/Tasks/ImageProcessingTask.php b/lib/BackgroundJob/Tasks/ImageProcessingTask.php index a866b6aa..ef9295c9 100644 --- a/lib/BackgroundJob/Tasks/ImageProcessingTask.php +++ b/lib/BackgroundJob/Tasks/ImageProcessingTask.php @@ -136,25 +136,23 @@ public function execute(FaceRecognitionContext $context) { } // Get faces in the temporary image - $tempImagePath = $tempImage->getTempPath(); + $tempImagePath = $tempImage->getResizedImagePath(); $rawFaces = $this->model->detectFaces($tempImagePath); $this->logInfo('Faces found: ' . count($rawFaces)); $faces = array(); foreach ($rawFaces as $rawFace) { - // Get landmarks of face from model - $rawLandmarks = $this->model->detectLandmarks($tempImage->getTempPath(), $rawFace); - // Get descriptor of face from model - $descriptor = $this->model->computeDescriptor($tempImage->getTempPath(), $rawLandmarks); - - // Normalize face and landmarks from model to original size + // Normalize face and get landmarks of face from model to original size $normFace = $this->getNormalizedFace($rawFace, $tempImage->getRatio()); - $normLandmarks = $this->getNormalizedLandmarks($rawLandmarks['parts'], $tempImage->getRatio()); + $landmarks = $this->model->detectLandmarks($tempImage->getOrientedImagePath(), $normFace); + + // Get descriptor of face from model + $descriptor = $this->model->computeDescriptor($tempImage->getOrientedImagePath(), $landmarks); // Convert from dictionary of faces to our Face Db Entity and put Landmarks and descriptor $face = Face::fromModel($image->getId(), $normFace); - $face->landmarks = $normLandmarks; + $face->landmarks = $landmarks['parts']; $face->descriptor = $descriptor; $faces[] = $face; @@ -260,19 +258,4 @@ private function getNormalizedFace(array $rawFace, float $ratio): array { return $face; } - /** - * Helper method, to normalize landmarks sizes back to original dimensions, based on ratio - * - */ - private function getNormalizedLandmarks(array $rawLandmarks, float $ratio): array { - $landmarks = []; - foreach ($rawLandmarks as $rawLandmark) { - $landmark = []; - $landmark['x'] = intval(round($rawLandmark['x']*$ratio)); - $landmark['y'] = intval(round($rawLandmark['y']*$ratio)); - $landmarks[] = $landmark; - } - return $landmarks; - } - } \ No newline at end of file diff --git a/lib/Command/MigrateCommand.php b/lib/Command/MigrateCommand.php new file mode 100644 index 00000000..a4798ed7 --- /dev/null +++ b/lib/Command/MigrateCommand.php @@ -0,0 +1,249 @@ + + * @copyright Copyright (c) 2019, Branko Kokanovic + * + * @author Branko Kokanovic + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ +namespace OCA\FaceRecognition\Command; + +use OCP\IUserManager; + +use Symfony\Component\Console\Command\Command; +use Symfony\Component\Console\Input\InputOption; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Output\OutputInterface; + +use Symfony\Component\Console\Helper\ProgressBar; + +use OCA\FaceRecognition\Db\Face; +use OCA\FaceRecognition\Db\FaceMapper; + +use OCA\FaceRecognition\Db\Image; +use OCA\FaceRecognition\Db\ImageMapper; + +use OCA\FaceRecognition\Model\ModelManager; + +use OCA\FaceRecognition\Service\FaceManagementService; +use OCA\FaceRecognition\Service\FileService; + +use OCP\Image as OCP_Image; + +class MigrateCommand extends Command { + + /** @var FaceManagementService */ + protected $faceManagementService; + + /** @var FileService */ + protected $fileService; + + /** @var IUserManager */ + protected $userManager; + + /** @var ModelManager */ + protected $modelManager; + + /** @var FaceMapper */ + protected $faceMapper; + + /** @var ImageMapper Image mapper*/ + protected $imageMapper; + + /** + * @param FaceManagementService $faceManagementService + * @param IUserManager $userManager + */ + public function __construct(FaceManagementService $faceManagementService, + FileService $fileService, + IUserManager $userManager, + ModelManager $modelManager, + FaceMapper $faceMapper, + ImageMapper $imageMapper) + { + parent::__construct(); + + $this->faceManagementService = $faceManagementService; + $this->fileService = $fileService; + $this->userManager = $userManager; + $this->modelManager = $modelManager; + $this->faceMapper = $faceMapper; + $this->imageMapper = $imageMapper; + } + + protected function configure() { + $this + ->setName('face:migrate') + ->setDescription( + 'Migrate the faces found in a model and analyze with the current model.') + ->addOption( + 'model', + 'm', + InputOption::VALUE_REQUIRED, + 'The identifier number of the model to migrate', + null, + ) + ->addOption( + 'user_id', + 'u', + InputOption::VALUE_REQUIRED, + 'Migrate data for a given user only. If not given, migrate everything for all users.', + null, + ); + } + + /** + * @param InputInterface $input + * @param OutputInterface $output + * @return int + */ + protected function execute(InputInterface $input, OutputInterface $output) { + // Extract user, if any + // + $userId = $input->getOption('user_id'); + if ($userId === null) { + $output->writeln("You must specify the user to migrate"); + return 1; + } + + $user = $this->userManager->get($userId); + if ($user === null) { + $output->writeln("User with id <$userId> is unknown."); + return 1; + } + + $modelId = $input->getOption('model'); + if (is_null($modelId)) { + $output->writeln("You must indicate the ID of the model to migrate"); + return 1; + } + + $model = $this->modelManager->getModel($modelId); + if (is_null($model)) { + $output->writeln("Invalid model Id"); + return 1; + } + + if (!$model->isInstalled()) { + $output->writeln("The model <$modelId> is not installed"); + return 1; + } + + $currentModel = $this->modelManager->getCurrentModel(); + $currentModelId = (!is_null($currentModel)) ? $currentModel->getId() : -1; + + if ($currentModelId === $modelId) { + $output->writeln("The proposed model <$modelId> to migrate must be other than the current one <$currentModelId>"); + return 1; + } + + if (!$this->faceManagementService->hasDataForUser($userId, $modelId)) { + $output->writeln("The proposed model <$modelId> to migrate is empty"); + return 1; + } + + if ($this->faceManagementService->hasDataForUser($userId, $currentModelId)) { + $output->writeln("The current model <$currentModelId> already has data. You cannot migrate to a used model."); + return 1; + } + + /** + * MIgrate + */ + $currentModel->open(); + + $oldImages = $this->imageMapper->findAll($userId, $modelId); + + $progressBar = new ProgressBar($output, count($oldImages)); + $progressBar->start(); + + foreach ($oldImages as $oldImage) { + $newImage = $this->migrateImage($oldImage, $userId, $currentModelId); + $oldFaces = $this->faceMapper->findFromFile($userId, $modelId, $newImage->getFile()); + if (count($oldFaces) > 0) { + $filePath = $this->getImageFilePath($newImage); + foreach ($oldFaces as $oldFace) { + $this->migrateFace($currentModel, $oldFace, $newImage, $filePath); + } + $this->fileService->clean(); + } + $progressBar->advance(1); + } + + $progressBar->finish(); + } + + private function migrateImage($oldImage, $userId, $modelId): Image { + $image = new Image(); + + $image->setUser($userId); + $image->setFile($oldImage->getFile()); + $image->setModel($modelId); + $image->setIsProcessed($oldImage->getIsProcessed()); + $image->setError($oldImage->getError()); + $image->setLastProcessedTime($oldImage->getLastProcessedTime()); + $image->setProcessingDuration($oldImage->getProcessingDuration()); + + return $this->imageMapper->insert($image); + } + + private function migrateFace($model, $oldFace, $image, $filePath) { + $faceRect = $this->getFaceRect($oldFace); + + $face = Face::fromModel($image->getId(), $faceRect); + + $landmarks = $model->detectLandmarks($filePath, $faceRect); + $descriptor = $model->computeDescriptor($filePath, $landmarks); + + $face->landmarks = $landmarks['parts']; + $face->descriptor = $descriptor; + + $this->faceMapper->insertFace($face); + } + + private function getImageFilePath(Image $image): ?string { + $file = $this->fileService->getFileById($image->getFile(), $image->getUser()); + if (empty($file)) { + return null; + } + + $localPath = $this->fileService->getLocalFile($file); + + $image = new OCP_Image(); + $image->loadFromFile($localPath); + if ($image->getOrientation() > 1) { + $tempPath = $this->fileService->getTemporaryFile(); + $image->fixOrientation(); + $image->save($tempPath, 'image/png'); + return $tempPath; + } + + return $localPath; + } + + private function getFaceRect(Face $face): array { + $rect = []; + $rect['left'] = (int)$face->getLeft(); + $rect['right'] = (int)$face->getRight(); + $rect['top'] = (int)$face->getTop(); + $rect['bottom'] = (int)$face->getBottom(); + $rect['detection_confidence'] = $face->getConfidence(); + return $rect; + } + +} diff --git a/lib/Command/ResetCommand.php b/lib/Command/ResetCommand.php index 6337799d..46ba6cad 100644 --- a/lib/Command/ResetCommand.php +++ b/lib/Command/ResetCommand.php @@ -72,6 +72,13 @@ protected function configure() { 'Reset everything.', null ) + ->addOption( + 'model', + null, + InputOption::VALUE_NONE, + 'Reset current model.', + null + ) ->addOption( 'image-errors', null, @@ -114,6 +121,11 @@ protected function execute(InputInterface $input, OutputInterface $output) { $output->writeln('Reset successfully done'); return 0; } + else if ($input->getOption('model')) { + $this->resetModel($user); + $output->writeln('Reset model successfully done'); + return 0; + } else if ($input->getOption('image-errors')) { $this->resetImageErrors($user); $output->writeln('Reset image errors done'); @@ -142,4 +154,8 @@ private function resetAll($user) { $this->faceManagementService->resetAll($user); } + private function resetModel($user) { + $this->faceManagementService->resetModel($user); + } + } diff --git a/lib/Db/FaceMapper.php b/lib/Db/FaceMapper.php index 09f68e2a..de51d8ba 100644 --- a/lib/Db/FaceMapper.php +++ b/lib/Db/FaceMapper.php @@ -57,7 +57,7 @@ public function find (int $faceId) { */ public function findFromFile(string $userId, int $modelId, int $fileId): array { $qb = $this->db->getQueryBuilder(); - $qb->select('f.id', 'left', 'right', 'top', 'bottom', 'person') + $qb->select('f.id', 'left', 'right', 'top', 'bottom', 'person', 'confidence', 'creation_time') ->from($this->getTableName(), 'f') ->innerJoin('f', 'facerecog_images' ,'i', $qb->expr()->eq('f.image', 'i.id')) ->where($qb->expr()->eq('i.user', $qb->createParameter('user_id'))) @@ -204,6 +204,28 @@ public function deleteUserFaces(string $userId) { ->execute(); } + /** + * Deletes all faces from that user and model + * + * @param string $userId User to drop faces from table. + * @param int $modelId model to drop faces from table. + */ + public function deleteUserModel(string $userId, $modelId) { + $sub = $this->db->getQueryBuilder(); + $sub->select(new Literal('1')); + $sub->from('facerecog_images', 'i') + ->where($sub->expr()->eq('i.id', '*PREFIX*' . $this->getTableName() .'.image')) + ->andWhere($sub->expr()->eq('i.user', $sub->createParameter('user'))) + ->andWhere($sub->expr()->eq('i.model', $sub->createParameter('model'))); + + $qb = $this->db->getQueryBuilder(); + $qb->delete($this->getTableName()) + ->where('EXISTS (' . $sub->getSQL() . ')') + ->setParameter('user', $userId) + ->setParameter('model', $modelId) + ->execute(); + } + /** * Unset relation beetwen faces and persons from that user in order to reset clustering * diff --git a/lib/Db/ImageMapper.php b/lib/Db/ImageMapper.php index 7b408783..8854482e 100644 --- a/lib/Db/ImageMapper.php +++ b/lib/Db/ImageMapper.php @@ -52,6 +52,19 @@ public function find(string $userId, int $imageId) { return $this->findEntity($qb); } + /** + * @param string $userId Id of user + * @param int $modelId Id of model to get + */ + public function findAll(string $userId, int $modelId) { + $qb = $this->db->getQueryBuilder(); + $qb->select('id', 'file', 'is_processed', 'error', 'last_processed_time', 'processing_duration') + ->from($this->getTableName()) + ->where($qb->expr()->eq('user', $qb->createNamedParameter($userId))) + ->andWhere($qb->expr()->eq('model', $qb->createNamedParameter($modelId))); + return $this->findEntities($qb); + } + /** * @param string $userId Id of user * @param int $modelId Id of model @@ -298,4 +311,19 @@ public function deleteUserImages(string $userId) { ->where($qb->expr()->eq('user', $qb->createNamedParameter($userId))) ->execute(); } + + /** + * Deletes all images from that user and Model + * + * @param string $userId User to drop images from table. + * @param int $modelId model to drop images from table. + */ + public function deleteUserModel(string $userId, $modelId) { + $qb = $this->db->getQueryBuilder(); + $qb->delete($this->getTableName()) + ->where($qb->expr()->eq('user', $qb->createNamedParameter($userId))) + ->andWhere($qb->expr()->eq('model', $qb->createNamedParameter($modelId))) + ->execute(); + } + } diff --git a/lib/Db/PersonMapper.php b/lib/Db/PersonMapper.php index 59190605..b733c02f 100644 --- a/lib/Db/PersonMapper.php +++ b/lib/Db/PersonMapper.php @@ -299,6 +299,24 @@ public function deleteUserPersons(string $userId) { ->execute(); } + /** + * Deletes all persons from that user and model + * + * @param string $userId ID of user for drop from table + * @param string $modelId model for drop from table + */ + public function deleteUserModel(string $userId, int $modelId) { + //TODO: Make it atomic + $qb = $this->db->getQueryBuilder(); + $qb->delete($this->getTableName()) + ->where($qb->expr()->eq('id', $qb->createParameter('person'))); + + $persons = $this->findAll($userId, $modelId); + foreach ($persons as $person) { + $qb->setParameter('person', $person->getId())->execute(); + } + } + /** * Deletes person if it is empty (have no faces associated to it) * diff --git a/lib/Helper/TempImage.php b/lib/Helper/TempImage.php index b60dc1a9..52e8aebe 100644 --- a/lib/Helper/TempImage.php +++ b/lib/Helper/TempImage.php @@ -33,8 +33,11 @@ class TempImage extends Image { /** @var string */ private $imagePath; - /** @var string */ - private $tempPath; + /** @var string|null */ + private $orientedImagePath; + + /** @var string|null */ + private $resizedImagePath; /** @var string */ private $preferredMimeType; @@ -71,13 +74,33 @@ public function __construct(string $imagePath, $this->prepareImage(); } + /** + * Get the path of orig image + * + * @return string + */ + public function getImagePath(): string { + return $this->imagePath; + } + + /** + * Get the path of oriented or original image + * + * @return string + */ + public function getOrientedImagePath(): string { + $this->prepareOrientedImage(); + + return $this->orientedImagePath; + } + /** * Get the path of temporary image * * @return string */ - public function getTempPath(): string { - return $this->tempPath; + public function getResizedImagePath(): string { + return $this->resizedImagePath; } /** @@ -110,7 +133,6 @@ public function clean() { */ private function prepareImage() { $this->loadFromFile($this->imagePath); - $this->fixOrientation(); if (!$this->valid()) { throw new \RuntimeException("Image is not valid, probably cannot be loaded"); @@ -122,10 +144,11 @@ private function prepareImage() { return; } + $this->fixOrientation(); $this->ratio = $this->resizeImage(); - $this->tempPath = $this->tempManager->getTemporaryFile(); - $this->save($this->tempPath, $this->preferredMimeType); + $this->resizedImagePath = $this->tempManager->getTemporaryFile(); + $this->save($this->resizedImagePath, $this->preferredMimeType); } /** @@ -156,4 +179,29 @@ private function resizeImage(): float { return 1 / $scaleFactor; } + /** + * Obtain a temporary image according to the imposed restrictions. + * + */ + private function prepareOrientedImage() { + if (!is_null($this->orientedImagePath)) { + return; + } + + $this->loadFromFile($this->imagePath); + + if (!$this->valid()) { + throw new \RuntimeException("Image is not valid, probably cannot be loaded"); + } + + if ($this->getOrientation() > 1) { + $this->fixOrientation(); + + $this->orientedImagePath = $this->tempManager->getTemporaryFile(); + $this->save($this->orientedImagePath, $this->preferredMimeType); + } else { + $this->orientedImagePath = $this->imagePath; + } + } + } \ No newline at end of file diff --git a/lib/Service/FaceManagementService.php b/lib/Service/FaceManagementService.php index 1caee95f..c7cbd9ca 100644 --- a/lib/Service/FaceManagementService.php +++ b/lib/Service/FaceManagementService.php @@ -72,6 +72,36 @@ public function __construct(IUserManager $userManager, $this->settingsService = $settingsService; } + /** + * Check if the current model has data on db + * + * @param IUser|null $user Optional user to check + * @param Int $modelId Optional model to check + */ + public function hasData(IUser $user = null, int $modelId = -1) { + if ($modelId === -1) { + $modelId = $this->settingsService->getCurrentFaceModel(); + } + $eligible_users = $this->getEligiblesUserId($user); + foreach ($eligible_users as $userId) { + if ($this->hasDataForUser($userId, $modelId)) { + return true; + } + } + return false; + } + + /** + * Check if the current model has data on db for user + * + * @param string $user ID of user to check + * @param Int $modelId model to check + */ + public function hasDataForUser(string $userId, int $modelId) { + $facesCount = $this->faceMapper->countFaces($userId, $modelId); + return ($facesCount > 0); + } + /** * Deletes all faces, images and persons found. IF no user is given, resetting is executed for all users. * @@ -97,6 +127,36 @@ public function resetAllForUser(string $userId) { $this->settingsService->setUserFullScanDone(false, $userId); } + /** + * Deletes all faces, images and persons found. If no user is given, resetting is executed for all users. + * + * @param IUser|null $user Optional user to execute resetting for + * @param Int $modelId Optional model to clean + */ + public function resetModel(IUser $user = null, int $modelId = -1) { + if ($modelId === -1) { + $modelId = $this->settingsService->getCurrentFaceModel(); + } + $eligible_users = $this->getEligiblesUserId($user); + foreach($eligible_users as $userId) { + $this->resetModelForUser($userId, $modelId); + } + } + + /** + * Deletes all faces, images and persons found for a given user. + * + * @param string $user ID of user to execute resetting for + * @param Int $modelId model to clean + */ + public function resetModelForUser(string $userId, $modelId) { + $this->personMapper->deleteUserModel($userId, $modelId); + $this->faceMapper->deleteUserModel($userId, $modelId); + $this->imageMapper->deleteUserModel($userId, $modelId); + + $this->settingsService->setUserFullScanDone(false, $userId); + } + /** * Reset error in images in order to re-analyze again. * If no user is given, resetting is executed for all users.