feat(web/server): Face thumbnail selection (#3081)

* add migration * verify running migration populate new value * implemented service * generate api * FE works * FR Works * fix test * fix test fixture * fix test * fix test * consolidate api * fix test * added test * pr feedback * refactor * click ont humbnail to show feature selection as well
immich-app · Jul 2, 2023 · 7947f4d · 7947f4d
1 parent 1df068b
commit 7947f4d
Show file tree

Hide file tree

Showing 21 changed files with 395 additions and 58 deletions.
diff --git a/mobile/openapi/doc/PersonUpdateDto.md b/mobile/openapi/doc/PersonUpdateDto.md
diff --git a/mobile/openapi/lib/model/person_update_dto.dart b/mobile/openapi/lib/model/person_update_dto.dart
diff --git a/mobile/openapi/test/person_update_dto_test.dart b/mobile/openapi/test/person_update_dto_test.dart
diff --git a/server/immich-openapi-specs.json b/server/immich-openapi-specs.json
@@ -5816,12 +5816,14 @@
         "type": "object",
         "properties": {
           "name": {
-            "type": "string"
+            "type": "string",
+            "description": "Person name."
+          },
+          "featureFaceAssetId": {
+            "type": "string",
+            "description": "Asset is used to get the feature face thumbnail."
           }
-        },
-        "required": [
-          "name"
-        ]
+        }
       },
       "QueueStatusDto": {
         "type": "object",

diff --git a/server/src/domain/facial-recognition/facial-recognition.service.spec.ts b/server/src/domain/facial-recognition/facial-recognition.service.spec.ts
@@ -191,6 +191,12 @@ describe(FacialRecognitionService.name, () => {
         personId: 'person-1',
         assetId: 'asset-id',
         embedding: [1, 2, 3, 4],
+        boundingBoxX1: 100,
+        boundingBoxY1: 100,
+        boundingBoxX2: 200,
+        boundingBoxY2: 200,
+        imageHeight: 500,
+        imageWidth: 400,
       });
     });
 
@@ -207,6 +213,12 @@ describe(FacialRecognitionService.name, () => {
         personId: 'person-1',
         assetId: 'asset-id',
         embedding: [1, 2, 3, 4],
+        boundingBoxX1: 100,
+        boundingBoxY1: 100,
+        boundingBoxX2: 200,
+        boundingBoxY2: 200,
+        imageHeight: 500,
+        imageWidth: 400,
       });
       expect(jobMock.queue.mock.calls).toEqual([
         [

diff --git a/server/src/domain/facial-recognition/facial-recognition.services.ts b/server/src/domain/facial-recognition/facial-recognition.services.ts
@@ -83,7 +83,16 @@ export class FacialRecognitionService {
 
       const faceId: AssetFaceId = { assetId: asset.id, personId };
 
-      await this.faceRepository.create({ ...faceId, embedding });
+      await this.faceRepository.create({
+        ...faceId,
+        embedding,
+        imageHeight: rest.imageHeight,
+        imageWidth: rest.imageWidth,
+        boundingBoxX1: rest.boundingBox.x1,
+        boundingBoxX2: rest.boundingBox.x2,
+        boundingBoxY1: rest.boundingBox.y1,
+        boundingBoxY2: rest.boundingBox.y2,
+      });
       await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_FACE, data: faceId });
     }
 

diff --git a/server/src/domain/person/person.dto.ts b/server/src/domain/person/person.dto.ts
@@ -1,10 +1,20 @@
 import { AssetFaceEntity, PersonEntity } from '@app/infra/entities';
-import { IsNotEmpty, IsString } from 'class-validator';
+import { IsOptional, IsString } from 'class-validator';
 
 export class PersonUpdateDto {
-  @IsNotEmpty()
+  /**
+   * Person name.
+   */
+  @IsOptional()
   @IsString()
-  name!: string;
+  name?: string;
+
+  /**
+   * Asset is used to get the feature face thumbnail.
+   */
+  @IsOptional()
+  @IsString()
+  featureFaceAssetId?: string;
 }
 
 export class PersonResponseDto {

diff --git a/server/src/domain/person/person.repository.ts b/server/src/domain/person/person.repository.ts
@@ -1,5 +1,5 @@
-import { AssetEntity, PersonEntity } from '@app/infra/entities';
-
+import { AssetFaceId } from '@app/domain';
+import { AssetEntity, AssetFaceEntity, PersonEntity } from '@app/infra/entities';
 export const IPersonRepository = 'IPersonRepository';
 
 export interface PersonSearchOptions {
@@ -16,4 +16,6 @@ export interface IPersonRepository {
   update(entity: Partial<PersonEntity>): Promise<PersonEntity>;
   delete(entity: PersonEntity): Promise<PersonEntity | null>;
   deleteAll(): Promise<number>;
+
+  getFaceById(payload: AssetFaceId): Promise<AssetFaceEntity | null>;
 }
diff --git a/server/src/domain/person/person.service.spec.ts b/server/src/domain/person/person.service.spec.ts
@@ -2,6 +2,7 @@ import { BadRequestException, NotFoundException } from '@nestjs/common';
 import {
   assetEntityStub,
   authStub,
+  faceStub,
   newJobRepositoryMock,
   newPersonRepositoryMock,
   newStorageRepositoryMock,
@@ -108,6 +109,36 @@ describe(PersonService.name, () => {
         data: { ids: [assetEntityStub.image.id] },
       });
     });
+
+    it("should update a person's thumbnailPath", async () => {
+      personMock.getById.mockResolvedValue(personStub.withName);
+      personMock.getFaceById.mockResolvedValue(faceStub.face1);
+
+      await expect(
+        sut.update(authStub.admin, 'person-1', { featureFaceAssetId: faceStub.face1.assetId }),
+      ).resolves.toEqual(responseDto);
+
+      expect(personMock.getById).toHaveBeenCalledWith('admin_id', 'person-1');
+      expect(personMock.getFaceById).toHaveBeenCalledWith({
+        assetId: faceStub.face1.assetId,
+        personId: 'person-1',
+      });
+      expect(jobMock.queue).toHaveBeenCalledWith({
+        name: JobName.GENERATE_FACE_THUMBNAIL,
+        data: {
+          assetId: faceStub.face1.assetId,
+          personId: 'person-1',
+          boundingBox: {
+            x1: faceStub.face1.boundingBoxX1,
+            x2: faceStub.face1.boundingBoxX2,
+            y1: faceStub.face1.boundingBoxY1,
+            y2: faceStub.face1.boundingBoxY2,
+          },
+          imageHeight: faceStub.face1.imageHeight,
+          imageWidth: faceStub.face1.imageWidth,
+        },
+      });
+    });
   });
 
   describe('handlePersonCleanup', () => {

diff --git a/server/src/domain/person/person.service.ts b/server/src/domain/person/person.service.ts
@@ -1,3 +1,4 @@
+import { PersonEntity } from '@app/infra/entities';
 import { BadRequestException, Inject, Injectable, Logger, NotFoundException } from '@nestjs/common';
 import { AssetResponseDto, mapAsset } from '../asset';
 import { AuthUserDto } from '../auth';
@@ -52,18 +53,54 @@ export class PersonService {
   }
 
   async update(authUser: AuthUserDto, personId: string, dto: PersonUpdateDto): Promise<PersonResponseDto> {
-    const exists = await this.repository.getById(authUser.id, personId);
-    if (!exists) {
+    let person = await this.repository.getById(authUser.id, personId);
+    if (!person) {
       throw new BadRequestException();
     }
 
-    const person = await this.repository.update({ id: personId, name: dto.name });
+    if (dto.name) {
+      person = await this.updateName(authUser, personId, dto.name);
+    }
+
+    if (dto.featureFaceAssetId) {
+      await this.updateFaceThumbnail(personId, dto.featureFaceAssetId);
+    }
+
+    return mapPerson(person);
+  }
+
+  private async updateName(authUser: AuthUserDto, personId: string, name: string): Promise<PersonEntity> {
+    const person = await this.repository.update({ id: personId, name });
 
     const relatedAsset = await this.getAssets(authUser, personId);
     const assetIds = relatedAsset.map((asset) => asset.id);
     await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_ASSET, data: { ids: assetIds } });
 
-    return mapPerson(person);
+    return person;
+  }
+
+  private async updateFaceThumbnail(personId: string, assetId: string): Promise<void> {
+    const face = await this.repository.getFaceById({ assetId, personId });
+
+    if (!face) {
+      throw new BadRequestException();
+    }
+
+    return await this.jobRepository.queue({
+      name: JobName.GENERATE_FACE_THUMBNAIL,
+      data: {
+        assetId: assetId,
+        personId,
+        boundingBox: {
+          x1: face.boundingBoxX1,
+          x2: face.boundingBoxX2,
+          y1: face.boundingBoxY1,
+          y2: face.boundingBoxY2,
+        },
+        imageHeight: face.imageHeight,
+        imageWidth: face.imageWidth,
+      },
+    });
   }
 
   async handlePersonCleanup() {

diff --git a/server/src/infra/entities/asset-face.entity.ts b/server/src/infra/entities/asset-face.entity.ts
@@ -17,6 +17,24 @@ export class AssetFaceEntity {
   })
   embedding!: number[] | null;
 
+  @Column({ default: 0, type: 'int' })
+  imageWidth!: number;
+
+  @Column({ default: 0, type: 'int' })
+  imageHeight!: number;
+
+  @Column({ default: 0, type: 'int' })
+  boundingBoxX1!: number;
+
+  @Column({ default: 0, type: 'int' })
+  boundingBoxY1!: number;
+
+  @Column({ default: 0, type: 'int' })
+  boundingBoxX2!: number;
+
+  @Column({ default: 0, type: 'int' })
+  boundingBoxY2!: number;
+
   @ManyToOne(() => AssetEntity, (asset) => asset.faces, { onDelete: 'CASCADE', onUpdate: 'CASCADE' })
   asset!: AssetEntity;
 

diff --git a/server/src/infra/migrations/1688241394489-AddDetectFaceResultInfo.ts b/server/src/infra/migrations/1688241394489-AddDetectFaceResultInfo.ts
@@ -0,0 +1,23 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class AddDetectFaceResultInfo1688241394489 implements MigrationInterface {
+  name = 'AddDetectFaceResultInfo1688241394489';
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`ALTER TABLE "asset_faces" ADD "imageWidth" integer NOT NULL DEFAULT '0'`);
+    await queryRunner.query(`ALTER TABLE "asset_faces" ADD "imageHeight" integer NOT NULL DEFAULT '0'`);
+    await queryRunner.query(`ALTER TABLE "asset_faces" ADD "boundingBoxX1" integer NOT NULL DEFAULT '0'`);
+    await queryRunner.query(`ALTER TABLE "asset_faces" ADD "boundingBoxY1" integer NOT NULL DEFAULT '0'`);
+    await queryRunner.query(`ALTER TABLE "asset_faces" ADD "boundingBoxX2" integer NOT NULL DEFAULT '0'`);
+    await queryRunner.query(`ALTER TABLE "asset_faces" ADD "boundingBoxY2" integer NOT NULL DEFAULT '0'`);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`ALTER TABLE "asset_faces" DROP COLUMN "boundingBoxY2"`);
+    await queryRunner.query(`ALTER TABLE "asset_faces" DROP COLUMN "boundingBoxX2"`);
+    await queryRunner.query(`ALTER TABLE "asset_faces" DROP COLUMN "boundingBoxY1"`);
+    await queryRunner.query(`ALTER TABLE "asset_faces" DROP COLUMN "boundingBoxX1"`);
+    await queryRunner.query(`ALTER TABLE "asset_faces" DROP COLUMN "imageHeight"`);
+    await queryRunner.query(`ALTER TABLE "asset_faces" DROP COLUMN "imageWidth"`);
+  }
+}
diff --git a/server/src/infra/repositories/person.repository.ts b/server/src/infra/repositories/person.repository.ts
@@ -1,4 +1,4 @@
-import { IPersonRepository, PersonSearchOptions } from '@app/domain';
+import { AssetFaceId, IPersonRepository, PersonSearchOptions } from '@app/domain';
 import { InjectRepository } from '@nestjs/typeorm';
 import { Repository } from 'typeorm';
 import { AssetEntity, AssetFaceEntity, PersonEntity } from '../entities';
@@ -77,4 +77,8 @@ export class PersonRepository implements IPersonRepository {
     const { id } = await this.personRepository.save(entity);
     return this.personRepository.findOneByOrFail({ id });
   }
+
+  async getFaceById({ personId, assetId }: AssetFaceId): Promise<AssetFaceEntity | null> {
+    return this.assetFaceRepository.findOneBy({ assetId, personId });
+  }
 }