From ea044648f300b4fecdfb5238d5a22370391f99c3 Mon Sep 17 00:00:00 2001 From: mertalev <101130780+mertalev@users.noreply.github.com> Date: Thu, 9 May 2024 17:52:37 -0400 Subject: [PATCH 1/7] wip --- server/src/entities/asset-face.entity.ts | 8 ++-- server/src/entities/face-search.entity.ts | 21 ++++++++++ ...715217873291-SeparateFaceSearchRelation.ts | 39 +++++++++++++++++++ .../src/repositories/database.repository.ts | 2 +- server/src/repositories/person.repository.ts | 7 +--- server/src/repositories/search.repository.ts | 5 ++- server/src/services/person.service.spec.ts | 5 ++- server/src/services/person.service.ts | 19 ++++++--- server/test/fixtures/face.stub.ts | 18 ++++----- 9 files changed, 96 insertions(+), 28 deletions(-) create mode 100644 server/src/entities/face-search.entity.ts create mode 100644 server/src/migrations/1715217873291-SeparateFaceSearchRelation.ts diff --git a/server/src/entities/asset-face.entity.ts b/server/src/entities/asset-face.entity.ts index 38fcd460636d4..265faf4bf529d 100644 --- a/server/src/entities/asset-face.entity.ts +++ b/server/src/entities/asset-face.entity.ts @@ -1,6 +1,7 @@ import { AssetEntity } from 'src/entities/asset.entity'; import { PersonEntity } from 'src/entities/person.entity'; -import { Column, Entity, Index, ManyToOne, PrimaryGeneratedColumn } from 'typeorm'; +import { Column, Entity, Index, ManyToOne, OneToOne, PrimaryGeneratedColumn } from 'typeorm'; +import { FaceSearchEntity } from 'src/entities/face-search.entity'; @Entity('asset_faces', { synchronize: false }) @Index('IDX_asset_faces_assetId_personId', ['assetId', 'personId']) @@ -15,9 +16,8 @@ export class AssetFaceEntity { @Column({ nullable: true, type: 'uuid' }) personId!: string | null; - @Index('face_index', { synchronize: false }) - @Column({ type: 'float4', array: true, select: false, transformer: { from: (v) => JSON.parse(v), to: (v) => v } }) - embedding!: number[]; + @OneToOne(() => FaceSearchEntity, (faceSearchEntity) => faceSearchEntity.face) + faceSearch?: FaceSearchEntity; @Column({ default: 0, type: 'int' }) imageWidth!: number; diff --git a/server/src/entities/face-search.entity.ts b/server/src/entities/face-search.entity.ts new file mode 100644 index 0000000000000..e06de4cd7736d --- /dev/null +++ b/server/src/entities/face-search.entity.ts @@ -0,0 +1,21 @@ +import { AssetFaceEntity } from 'src/entities/asset-face.entity'; +import { asVector } from 'src/utils/database'; +import { Column, Entity, Index, JoinColumn, OneToOne, PrimaryColumn } from 'typeorm'; + +@Entity('face_search', { synchronize: false }) +export class FaceSearchEntity { + @OneToOne(() => AssetFaceEntity, { onDelete: 'CASCADE', nullable: true }) + @JoinColumn({ name: 'faceId', referencedColumnName: 'id' }) + face?: AssetFaceEntity; + + @PrimaryColumn() + faceId!: string; + + @Index('face_index', { synchronize: false }) + @Column({ + type: 'float4', + array: true, + transformer: { from: (v) => JSON.parse(v), to: (v) => asVector(v, true) }, + }) + embedding!: number[]; +} diff --git a/server/src/migrations/1715217873291-SeparateFaceSearchRelation.ts b/server/src/migrations/1715217873291-SeparateFaceSearchRelation.ts new file mode 100644 index 0000000000000..bd122b9e337ca --- /dev/null +++ b/server/src/migrations/1715217873291-SeparateFaceSearchRelation.ts @@ -0,0 +1,39 @@ +import { vectorExt } from 'src/database.config'; +import { DatabaseExtension } from 'src/interfaces/database.interface'; +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class SeparateFaceSearchRelation1715217873291 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + CREATE TABLE face_search ( + "faceId" uuid PRIMARY KEY REFERENCES asset_faces(id) ON DELETE CASCADE, + embedding vector(512) NOT NULL )`); + + await queryRunner.query(` + INSERT INTO face_search("faceId", embedding) + SELECT id, embedding + FROM asset_faces faces`); + + if (vectorExt === DatabaseExtension.VECTORS) { + await queryRunner.query(`SET vectors.pgvector_compatibility=on`); + } + await queryRunner.query(`SET search_path TO "$user", public, vectors`); + + await queryRunner.query(`ALTER TABLE asset_faces DROP COLUMN "embedding"`); + + await queryRunner.query(` + CREATE INDEX face_index ON face_search + USING hnsw (embedding vector_cosine_ops) + WITH (ef_construction = 300, m = 16)`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE asset_faces ADD COLUMN IF NOT EXISTS "embedding" TYPE vector(512)`); + await queryRunner.query(` + INSERT INTO asset_faces(id, embedding) + SELECT fs."faceId", fs.embedding + FROM face_search fs + ON CONFLICT (fs."faceId") DO UPDATE SET embedding = fs.embedding`); + await queryRunner.query(`DROP TABLE face_search`); + } +} diff --git a/server/src/repositories/database.repository.ts b/server/src/repositories/database.repository.ts index dc442e701740b..fc9e76b0aa50b 100644 --- a/server/src/repositories/database.repository.ts +++ b/server/src/repositories/database.repository.ts @@ -98,7 +98,7 @@ export class DatabaseRepository implements IDatabaseRepository { } catch (error) { if (getVectorExtension() === DatabaseExtension.VECTORS) { this.logger.warn(`Could not reindex index ${index}. Attempting to auto-fix.`); - const table = index === VectorIndex.CLIP ? 'smart_search' : 'asset_faces'; + const table = index === VectorIndex.CLIP ? 'smart_search' : 'face_search'; const dimSize = await this.getDimSize(table); await this.dataSource.manager.transaction(async (manager) => { await this.setSearchPath(manager); diff --git a/server/src/repositories/person.repository.ts b/server/src/repositories/person.repository.ts index 225a2edeca1b5..36d742f8dcfe1 100644 --- a/server/src/repositories/person.repository.ts +++ b/server/src/repositories/person.repository.ts @@ -14,7 +14,6 @@ import { PersonStatistics, UpdateFacesData, } from 'src/interfaces/person.interface'; -import { asVector } from 'src/utils/database'; import { Instrumentation } from 'src/utils/instrumentation'; import { Paginated, PaginationOptions, paginate } from 'src/utils/pagination'; import { FindManyOptions, FindOptionsRelations, FindOptionsSelect, In, Repository } from 'typeorm'; @@ -249,10 +248,8 @@ export class PersonRepository implements IPersonRepository { } async createFaces(entities: AssetFaceEntity[]): Promise { - const res = await this.assetFaceRepository.insert( - entities.map((entity) => ({ ...entity, embedding: () => asVector(entity.embedding, true) })), - ); - return res.identifiers.map((row) => row.id); + const res = await this.assetFaceRepository.save(entities); + return res.map((row) => row.id); } async update(entity: Partial): Promise { diff --git a/server/src/repositories/search.repository.ts b/server/src/repositories/search.repository.ts index f0c5dcb3642ca..439ccd099c129 100644 --- a/server/src/repositories/search.repository.ts +++ b/server/src/repositories/search.repository.ts @@ -218,10 +218,11 @@ export class SearchRepository implements ISearchRepository { await this.assetRepository.manager.transaction(async (manager) => { const cte = manager .createQueryBuilder(AssetFaceEntity, 'faces') - .select('faces.embedding <=> :embedding', 'distance') + .select('search.embedding <=> :embedding', 'distance') .innerJoin('faces.asset', 'asset') + .innerJoin('faces.faceSearch', 'search') .where('asset.ownerId IN (:...userIds )') - .orderBy('faces.embedding <=> :embedding') + .orderBy('search.embedding <=> :embedding') .setParameters({ userIds, embedding: asVector(embedding) }); cte.limit(numResults); diff --git a/server/src/services/person.service.spec.ts b/server/src/services/person.service.spec.ts index bf6fc8207eb2c..69faf976a1907 100644 --- a/server/src/services/person.service.spec.ts +++ b/server/src/services/person.service.spec.ts @@ -668,15 +668,18 @@ describe(PersonService.name, () => { machineLearningMock.detectFaces.mockResolvedValue(detectFaceMock); searchMock.searchFaces.mockResolvedValue([{ face: faceStub.face1, distance: 0.7 }]); assetMock.getByIds.mockResolvedValue([assetStub.image]); + const faceId = 'face-id'; + cryptoMock.randomUUID.mockReturnValue(faceId); const face = { + faceId, assetId: 'asset-id', - embedding: [1, 2, 3, 4], boundingBoxX1: 100, boundingBoxY1: 100, boundingBoxX2: 200, boundingBoxY2: 200, imageHeight: 500, imageWidth: 400, + faceSearch: { faceId, embedding: [1, 2, 3, 4] }, }; await sut.handleDetectFaces({ id: assetStub.image.id }); diff --git a/server/src/services/person.service.ts b/server/src/services/person.service.ts index 57940f3113854..7aa08805d93a9 100644 --- a/server/src/services/person.service.ts +++ b/server/src/services/person.service.ts @@ -70,7 +70,7 @@ export class PersonService { @Inject(IStorageRepository) private storageRepository: IStorageRepository, @Inject(IJobRepository) private jobRepository: IJobRepository, @Inject(ISearchRepository) private smartInfoRepository: ISearchRepository, - @Inject(ICryptoRepository) cryptoRepository: ICryptoRepository, + @Inject(ICryptoRepository) private cryptoRepository: ICryptoRepository, @Inject(ILoggerRepository) private logger: ILoggerRepository, ) { this.access = AccessCore.create(accessRepository); @@ -347,15 +347,17 @@ export class PersonService { if (faces.length > 0) { await this.jobRepository.queue({ name: JobName.QUEUE_FACIAL_RECOGNITION, data: { force: false } }); + const faceId = this.cryptoRepository.randomUUID(); const mappedFaces = faces.map((face) => ({ + faceId, assetId: asset.id, - embedding: face.embedding, imageHeight, imageWidth, boundingBoxX1: face.boundingBox.x1, boundingBoxY1: face.boundingBox.y1, boundingBoxX2: face.boundingBox.x2, boundingBoxY2: face.boundingBox.y2, + faceSearch: { faceId, embedding: face.embedding }, })); const faceIds = await this.repository.createFaces(mappedFaces); @@ -409,14 +411,19 @@ export class PersonService { const face = await this.repository.getFaceByIdWithAssets( id, - { person: true, asset: true }, - { id: true, personId: true, embedding: true }, + { person: true, asset: true, faceSearch: true }, + { id: true, personId: true, faceSearch: { embedding: true } }, ); if (!face || !face.asset) { this.logger.warn(`Face ${id} not found`); return JobStatus.FAILED; } + if (!face.faceSearch?.embedding) { + this.logger.warn(`Face ${id} does not have an embedding`); + return JobStatus.FAILED; + } + if (face.personId) { this.logger.debug(`Face ${id} already has a person assigned`); return JobStatus.SKIPPED; @@ -424,7 +431,7 @@ export class PersonService { const matches = await this.smartInfoRepository.searchFaces({ userIds: [face.asset.ownerId], - embedding: face.embedding, + embedding: face.faceSearch.embedding, maxDistance: machineLearning.facialRecognition.maxDistance, numResults: machineLearning.facialRecognition.minFaces, }); @@ -448,7 +455,7 @@ export class PersonService { if (!personId) { const matchWithPerson = await this.smartInfoRepository.searchFaces({ userIds: [face.asset.ownerId], - embedding: face.embedding, + embedding: face.faceSearch.embedding, maxDistance: machineLearning.facialRecognition.maxDistance, numResults: 1, hasPerson: true, diff --git a/server/test/fixtures/face.stub.ts b/server/test/fixtures/face.stub.ts index 5ecb5701ced7e..82935dd345658 100644 --- a/server/test/fixtures/face.stub.ts +++ b/server/test/fixtures/face.stub.ts @@ -11,13 +11,13 @@ export const faceStub = { asset: assetStub.image, personId: personStub.withName.id, person: personStub.withName, - embedding: [1, 2, 3, 4], boundingBoxX1: 0, boundingBoxY1: 0, boundingBoxX2: 1, boundingBoxY2: 1, imageHeight: 1024, imageWidth: 1024, + faceSearch: { faceId: 'assetFaceId1', embedding: [1, 2, 3, 4] }, }), primaryFace1: Object.freeze>({ id: 'assetFaceId2', @@ -25,13 +25,13 @@ export const faceStub = { asset: assetStub.image, personId: personStub.primaryPerson.id, person: personStub.primaryPerson, - embedding: [1, 2, 3, 4], boundingBoxX1: 0, boundingBoxY1: 0, boundingBoxX2: 1, boundingBoxY2: 1, imageHeight: 1024, imageWidth: 1024, + faceSearch: { faceId: 'assetFaceId2', embedding: [1, 2, 3, 4] }, }), mergeFace1: Object.freeze>({ id: 'assetFaceId3', @@ -39,13 +39,13 @@ export const faceStub = { asset: assetStub.image, personId: personStub.mergePerson.id, person: personStub.mergePerson, - embedding: [1, 2, 3, 4], boundingBoxX1: 0, boundingBoxY1: 0, boundingBoxX2: 1, boundingBoxY2: 1, imageHeight: 1024, imageWidth: 1024, + faceSearch: { faceId: 'assetFaceId3', embedding: [1, 2, 3, 4] }, }), mergeFace2: Object.freeze>({ id: 'assetFaceId4', @@ -53,13 +53,13 @@ export const faceStub = { asset: assetStub.image1, personId: personStub.mergePerson.id, person: personStub.mergePerson, - embedding: [1, 2, 3, 4], boundingBoxX1: 0, boundingBoxY1: 0, boundingBoxX2: 1, boundingBoxY2: 1, imageHeight: 1024, imageWidth: 1024, + faceSearch: { faceId: 'assetFaceId4', embedding: [1, 2, 3, 4] }, }), start: Object.freeze>({ id: 'assetFaceId5', @@ -67,13 +67,13 @@ export const faceStub = { asset: assetStub.image, personId: personStub.newThumbnail.id, person: personStub.newThumbnail, - embedding: [1, 2, 3, 4], boundingBoxX1: 5, boundingBoxY1: 5, boundingBoxX2: 505, boundingBoxY2: 505, imageHeight: 2880, imageWidth: 2160, + faceSearch: { faceId: 'assetFaceId5', embedding: [1, 2, 3, 4] }, }), middle: Object.freeze>({ id: 'assetFaceId6', @@ -81,13 +81,13 @@ export const faceStub = { asset: assetStub.image, personId: personStub.newThumbnail.id, person: personStub.newThumbnail, - embedding: [1, 2, 3, 4], boundingBoxX1: 100, boundingBoxY1: 100, boundingBoxX2: 200, boundingBoxY2: 200, imageHeight: 500, imageWidth: 400, + faceSearch: { faceId: 'assetFaceId6', embedding: [1, 2, 3, 4] }, }), end: Object.freeze>({ id: 'assetFaceId7', @@ -95,13 +95,13 @@ export const faceStub = { asset: assetStub.image, personId: personStub.newThumbnail.id, person: personStub.newThumbnail, - embedding: [1, 2, 3, 4], boundingBoxX1: 300, boundingBoxY1: 300, boundingBoxX2: 495, boundingBoxY2: 495, imageHeight: 500, imageWidth: 500, + faceSearch: { faceId: 'assetFaceId7', embedding: [1, 2, 3, 4] }, }), noPerson1: Object.freeze({ id: 'assetFaceId8', @@ -109,13 +109,13 @@ export const faceStub = { asset: assetStub.image, personId: null, person: null, - embedding: [1, 2, 3, 4], boundingBoxX1: 0, boundingBoxY1: 0, boundingBoxX2: 1, boundingBoxY2: 1, imageHeight: 1024, imageWidth: 1024, + faceSearch: { faceId: 'assetFaceId8', embedding: [1, 2, 3, 4] }, }), noPerson2: Object.freeze({ id: 'assetFaceId9', @@ -123,12 +123,12 @@ export const faceStub = { asset: assetStub.image, personId: null, person: null, - embedding: [1, 2, 3, 4], boundingBoxX1: 0, boundingBoxY1: 0, boundingBoxX2: 1, boundingBoxY2: 1, imageHeight: 1024, imageWidth: 1024, + faceSearch: { faceId: 'assetFaceId9', embedding: [1, 2, 3, 4] }, }), }; From 3b18f4b31bf597499052faa3049462f11e0f592a Mon Sep 17 00:00:00 2001 From: mertalev <101130780+mertalev@users.noreply.github.com> Date: Sat, 15 Jun 2024 16:54:16 -0400 Subject: [PATCH 2/7] various fixes --- server/src/entities/asset-face.entity.ts | 2 +- server/src/entities/face-search.entity.ts | 2 +- ...715217873291-SeparateFaceSearchRelation.ts | 32 ++++++++++++------- server/src/services/person.service.ts | 28 +++++++++------- 4 files changed, 39 insertions(+), 25 deletions(-) diff --git a/server/src/entities/asset-face.entity.ts b/server/src/entities/asset-face.entity.ts index 265faf4bf529d..049767d9a608c 100644 --- a/server/src/entities/asset-face.entity.ts +++ b/server/src/entities/asset-face.entity.ts @@ -16,7 +16,7 @@ export class AssetFaceEntity { @Column({ nullable: true, type: 'uuid' }) personId!: string | null; - @OneToOne(() => FaceSearchEntity, (faceSearchEntity) => faceSearchEntity.face) + @OneToOne(() => FaceSearchEntity, (faceSearchEntity) => faceSearchEntity.face, { cascade: ['insert'] }) faceSearch?: FaceSearchEntity; @Column({ default: 0, type: 'int' }) diff --git a/server/src/entities/face-search.entity.ts b/server/src/entities/face-search.entity.ts index e06de4cd7736d..3fd3c65f28e07 100644 --- a/server/src/entities/face-search.entity.ts +++ b/server/src/entities/face-search.entity.ts @@ -15,7 +15,7 @@ export class FaceSearchEntity { @Column({ type: 'float4', array: true, - transformer: { from: (v) => JSON.parse(v), to: (v) => asVector(v, true) }, + transformer: { from: (v) => JSON.parse(v), to: (v) => asVector(v) }, }) embedding!: number[]; } diff --git a/server/src/migrations/1715217873291-SeparateFaceSearchRelation.ts b/server/src/migrations/1715217873291-SeparateFaceSearchRelation.ts index bd122b9e337ca..26566e0f4ac9b 100644 --- a/server/src/migrations/1715217873291-SeparateFaceSearchRelation.ts +++ b/server/src/migrations/1715217873291-SeparateFaceSearchRelation.ts @@ -1,9 +1,14 @@ -import { vectorExt } from 'src/database.config'; +import { getVectorExtension } from 'src/database.config'; import { DatabaseExtension } from 'src/interfaces/database.interface'; import { MigrationInterface, QueryRunner } from 'typeorm'; export class SeparateFaceSearchRelation1715217873291 implements MigrationInterface { public async up(queryRunner: QueryRunner): Promise { + if (getVectorExtension() === DatabaseExtension.VECTORS) { + await queryRunner.query(`SET search_path TO "$user", public, vectors`); + await queryRunner.query(`SET vectors.pgvector_compatibility=on`); + } + await queryRunner.query(` CREATE TABLE face_search ( "faceId" uuid PRIMARY KEY REFERENCES asset_faces(id) ON DELETE CASCADE, @@ -14,11 +19,6 @@ export class SeparateFaceSearchRelation1715217873291 implements MigrationInterfa SELECT id, embedding FROM asset_faces faces`); - if (vectorExt === DatabaseExtension.VECTORS) { - await queryRunner.query(`SET vectors.pgvector_compatibility=on`); - } - await queryRunner.query(`SET search_path TO "$user", public, vectors`); - await queryRunner.query(`ALTER TABLE asset_faces DROP COLUMN "embedding"`); await queryRunner.query(` @@ -28,12 +28,22 @@ export class SeparateFaceSearchRelation1715217873291 implements MigrationInterfa } public async down(queryRunner: QueryRunner): Promise { - await queryRunner.query(`ALTER TABLE asset_faces ADD COLUMN IF NOT EXISTS "embedding" TYPE vector(512)`); + if (getVectorExtension() === DatabaseExtension.VECTORS) { + await queryRunner.query(`SET search_path TO "$user", public, vectors`); + await queryRunner.query(`SET vectors.pgvector_compatibility=on`); + } + + await queryRunner.query(`ALTER TABLE asset_faces ADD COLUMN "embedding" vector(512)`); await queryRunner.query(` - INSERT INTO asset_faces(id, embedding) - SELECT fs."faceId", fs.embedding - FROM face_search fs - ON CONFLICT (fs."faceId") DO UPDATE SET embedding = fs.embedding`); + UPDATE asset_faces + SET embedding = fs.embedding + FROM face_search fs + WHERE id = fs."faceId"`); await queryRunner.query(`DROP TABLE face_search`); + + await queryRunner.query(` + CREATE INDEX face_index ON asset_faces + USING hnsw (embedding vector_cosine_ops) + WITH (ef_construction = 300, m = 16)`); } } diff --git a/server/src/services/person.service.ts b/server/src/services/person.service.ts index 7aa08805d93a9..05034dc6f97bf 100644 --- a/server/src/services/person.service.ts +++ b/server/src/services/person.service.ts @@ -22,6 +22,7 @@ import { mapFaces, mapPerson, } from 'src/dtos/person.dto'; +import { AssetFaceEntity } from 'src/entities/asset-face.entity'; import { AssetEntity, AssetType } from 'src/entities/asset.entity'; import { PersonPathType } from 'src/entities/move.entity'; import { PersonEntity } from 'src/entities/person.entity'; @@ -347,18 +348,21 @@ export class PersonService { if (faces.length > 0) { await this.jobRepository.queue({ name: JobName.QUEUE_FACIAL_RECOGNITION, data: { force: false } }); - const faceId = this.cryptoRepository.randomUUID(); - const mappedFaces = faces.map((face) => ({ - faceId, - assetId: asset.id, - imageHeight, - imageWidth, - boundingBoxX1: face.boundingBox.x1, - boundingBoxY1: face.boundingBox.y1, - boundingBoxX2: face.boundingBox.x2, - boundingBoxY2: face.boundingBox.y2, - faceSearch: { faceId, embedding: face.embedding }, - })); + const mappedFaces: Partial[] = []; + for (const face of faces) { + const faceId = this.cryptoRepository.randomUUID(); + mappedFaces.push({ + id: faceId, + assetId: asset.id, + imageHeight, + imageWidth, + boundingBoxX1: face.boundingBox.x1, + boundingBoxY1: face.boundingBox.y1, + boundingBoxX2: face.boundingBox.x2, + boundingBoxY2: face.boundingBox.y2, + faceSearch: { faceId, embedding: face.embedding }, + }); + } const faceIds = await this.repository.createFaces(mappedFaces); await this.jobRepository.queueAll(faceIds.map((id) => ({ name: JobName.FACIAL_RECOGNITION, data: { id } }))); From 760750ebb07a29d0cb567287004c9a5fecba1a62 Mon Sep 17 00:00:00 2001 From: mertalev <101130780+mertalev@users.noreply.github.com> Date: Sat, 15 Jun 2024 17:34:51 -0400 Subject: [PATCH 3/7] new migration --- ...=> 1718486162779-AddFaceSearchRelation.ts} | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) rename server/src/migrations/{1715217873291-SeparateFaceSearchRelation.ts => 1718486162779-AddFaceSearchRelation.ts} (58%) diff --git a/server/src/migrations/1715217873291-SeparateFaceSearchRelation.ts b/server/src/migrations/1718486162779-AddFaceSearchRelation.ts similarity index 58% rename from server/src/migrations/1715217873291-SeparateFaceSearchRelation.ts rename to server/src/migrations/1718486162779-AddFaceSearchRelation.ts index 26566e0f4ac9b..4dce07b6c4370 100644 --- a/server/src/migrations/1715217873291-SeparateFaceSearchRelation.ts +++ b/server/src/migrations/1718486162779-AddFaceSearchRelation.ts @@ -2,7 +2,7 @@ import { getVectorExtension } from 'src/database.config'; import { DatabaseExtension } from 'src/interfaces/database.interface'; import { MigrationInterface, QueryRunner } from 'typeorm'; -export class SeparateFaceSearchRelation1715217873291 implements MigrationInterface { +export class AddFaceSearchRelation1718486162779 implements MigrationInterface { public async up(queryRunner: QueryRunner): Promise { if (getVectorExtension() === DatabaseExtension.VECTORS) { await queryRunner.query(`SET search_path TO "$user", public, vectors`); @@ -10,21 +10,21 @@ export class SeparateFaceSearchRelation1715217873291 implements MigrationInterfa } await queryRunner.query(` - CREATE TABLE face_search ( - "faceId" uuid PRIMARY KEY REFERENCES asset_faces(id) ON DELETE CASCADE, - embedding vector(512) NOT NULL )`); + CREATE TABLE face_search ( + "faceId" uuid PRIMARY KEY REFERENCES asset_faces(id) ON DELETE CASCADE, + embedding vector(512) NOT NULL )`); await queryRunner.query(` - INSERT INTO face_search("faceId", embedding) - SELECT id, embedding - FROM asset_faces faces`); + INSERT INTO face_search("faceId", embedding) + SELECT id, embedding + FROM asset_faces faces`); await queryRunner.query(`ALTER TABLE asset_faces DROP COLUMN "embedding"`); await queryRunner.query(` - CREATE INDEX face_index ON face_search - USING hnsw (embedding vector_cosine_ops) - WITH (ef_construction = 300, m = 16)`); + CREATE INDEX face_index ON face_search + USING hnsw (embedding vector_cosine_ops) + WITH (ef_construction = 300, m = 16)`); } public async down(queryRunner: QueryRunner): Promise { @@ -35,15 +35,15 @@ export class SeparateFaceSearchRelation1715217873291 implements MigrationInterfa await queryRunner.query(`ALTER TABLE asset_faces ADD COLUMN "embedding" vector(512)`); await queryRunner.query(` - UPDATE asset_faces - SET embedding = fs.embedding - FROM face_search fs - WHERE id = fs."faceId"`); + UPDATE asset_faces + SET embedding = fs.embedding + FROM face_search fs + WHERE id = fs."faceId"`); await queryRunner.query(`DROP TABLE face_search`); await queryRunner.query(` - CREATE INDEX face_index ON asset_faces - USING hnsw (embedding vector_cosine_ops) - WITH (ef_construction = 300, m = 16)`); + CREATE INDEX face_index ON asset_faces + USING hnsw (embedding vector_cosine_ops) + WITH (ef_construction = 300, m = 16)`); } } From 8d342d54d08abb364a1289019e865bad0ebb6574 Mon Sep 17 00:00:00 2001 From: mertalev <101130780+mertalev@users.noreply.github.com> Date: Sat, 15 Jun 2024 17:34:56 -0400 Subject: [PATCH 4/7] fix test --- server/src/services/person.service.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/services/person.service.spec.ts b/server/src/services/person.service.spec.ts index 69faf976a1907..eb0e3ad1e9863 100644 --- a/server/src/services/person.service.spec.ts +++ b/server/src/services/person.service.spec.ts @@ -671,7 +671,7 @@ describe(PersonService.name, () => { const faceId = 'face-id'; cryptoMock.randomUUID.mockReturnValue(faceId); const face = { - faceId, + id: faceId, assetId: 'asset-id', boundingBoxX1: 100, boundingBoxY1: 100, From 9a7ebfef0b5c3f58477e7fc11a0795885901a46f Mon Sep 17 00:00:00 2001 From: mertalev <101130780+mertalev@users.noreply.github.com> Date: Sat, 15 Jun 2024 19:46:56 -0400 Subject: [PATCH 5/7] add face search entity, update sql --- server/src/entities/asset-face.entity.ts | 2 +- server/src/entities/index.ts | 2 ++ server/src/queries/search.repository.sql | 5 +++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/server/src/entities/asset-face.entity.ts b/server/src/entities/asset-face.entity.ts index 049767d9a608c..c21aacfcd1a4d 100644 --- a/server/src/entities/asset-face.entity.ts +++ b/server/src/entities/asset-face.entity.ts @@ -1,7 +1,7 @@ import { AssetEntity } from 'src/entities/asset.entity'; +import { FaceSearchEntity } from 'src/entities/face-search.entity'; import { PersonEntity } from 'src/entities/person.entity'; import { Column, Entity, Index, ManyToOne, OneToOne, PrimaryGeneratedColumn } from 'typeorm'; -import { FaceSearchEntity } from 'src/entities/face-search.entity'; @Entity('asset_faces', { synchronize: false }) @Index('IDX_asset_faces_assetId_personId', ['assetId', 'personId']) diff --git a/server/src/entities/index.ts b/server/src/entities/index.ts index 313f2dc2691e0..cd3d74724b8e4 100644 --- a/server/src/entities/index.ts +++ b/server/src/entities/index.ts @@ -8,6 +8,7 @@ import { AssetStackEntity } from 'src/entities/asset-stack.entity'; import { AssetEntity } from 'src/entities/asset.entity'; import { AuditEntity } from 'src/entities/audit.entity'; import { ExifEntity } from 'src/entities/exif.entity'; +import { FaceSearchEntity } from 'src/entities/face-search.entity'; import { GeodataPlacesEntity } from 'src/entities/geodata-places.entity'; import { LibraryEntity } from 'src/entities/library.entity'; import { MemoryEntity } from 'src/entities/memory.entity'; @@ -34,6 +35,7 @@ export const entities = [ AssetJobStatusEntity, AuditEntity, ExifEntity, + FaceSearchEntity, GeodataPlacesEntity, MemoryEntity, MoveEntity, diff --git a/server/src/queries/search.repository.sql b/server/src/queries/search.repository.sql index 9efeae6248f77..987828a86054a 100644 --- a/server/src/queries/search.repository.sql +++ b/server/src/queries/search.repository.sql @@ -241,15 +241,16 @@ WITH "faces"."boundingBoxY1" AS "boundingBoxY1", "faces"."boundingBoxX2" AS "boundingBoxX2", "faces"."boundingBoxY2" AS "boundingBoxY2", - "faces"."embedding" <= > $1 AS "distance" + "search"."embedding" <= > $1 AS "distance" FROM "asset_faces" "faces" INNER JOIN "assets" "asset" ON "asset"."id" = "faces"."assetId" AND ("asset"."deletedAt" IS NULL) + INNER JOIN "face_search" "search" ON "search"."faceId" = "faces"."id" WHERE "asset"."ownerId" IN ($2) ORDER BY - "faces"."embedding" <= > $1 ASC + "search"."embedding" <= > $1 ASC LIMIT 100 ) From 1a2cfb556851459acf831d43c2d27ba04d4f0113 Mon Sep 17 00:00:00 2001 From: mertalev <101130780+mertalev@users.noreply.github.com> Date: Sat, 15 Jun 2024 19:48:20 -0400 Subject: [PATCH 6/7] update e2e --- e2e/src/utils.ts | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/e2e/src/utils.ts b/e2e/src/utils.ts index 72b3480299044..f3afdddf8b9bb 100644 --- a/e2e/src/utils.ts +++ b/e2e/src/utils.ts @@ -398,14 +398,7 @@ export const utils = { return; } - const vector = Array.from({ length: 512 }, Math.random); - const embedding = `[${vector.join(',')}]`; - - await client.query('INSERT INTO asset_faces ("assetId", "personId", "embedding") VALUES ($1, $2, $3)', [ - assetId, - personId, - embedding, - ]); + await client.query('INSERT INTO asset_faces ("assetId", "personId") VALUES ($1, $2)', [assetId, personId]); }, setPersonThumbnail: async (personId: string) => { From 2adf3cf5fd885ba864742b594b33b97b982934fa Mon Sep 17 00:00:00 2001 From: mertalev <101130780+mertalev@users.noreply.github.com> Date: Sat, 15 Jun 2024 19:54:11 -0400 Subject: [PATCH 7/7] set storage to external --- server/src/migrations/1718486162779-AddFaceSearchRelation.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server/src/migrations/1718486162779-AddFaceSearchRelation.ts b/server/src/migrations/1718486162779-AddFaceSearchRelation.ts index 4dce07b6c4370..5bf3fcd97b1ba 100644 --- a/server/src/migrations/1718486162779-AddFaceSearchRelation.ts +++ b/server/src/migrations/1718486162779-AddFaceSearchRelation.ts @@ -14,6 +14,9 @@ export class AddFaceSearchRelation1718486162779 implements MigrationInterface { "faceId" uuid PRIMARY KEY REFERENCES asset_faces(id) ON DELETE CASCADE, embedding vector(512) NOT NULL )`); + await queryRunner.query(`ALTER TABLE face_search ALTER COLUMN embedding SET STORAGE EXTERNAL`); + await queryRunner.query(`ALTER TABLE smart_search ALTER COLUMN embedding SET STORAGE EXTERNAL`); + await queryRunner.query(` INSERT INTO face_search("faceId", embedding) SELECT id, embedding @@ -34,6 +37,8 @@ export class AddFaceSearchRelation1718486162779 implements MigrationInterface { } await queryRunner.query(`ALTER TABLE asset_faces ADD COLUMN "embedding" vector(512)`); + await queryRunner.query(`ALTER TABLE face_search ALTER COLUMN embedding SET STORAGE DEFAULT`); + await queryRunner.query(`ALTER TABLE smart_search ALTER COLUMN embedding SET STORAGE DEFAULT`); await queryRunner.query(` UPDATE asset_faces SET embedding = fs.embedding