diff --git a/mobile/lib/shared/services/asset.service.dart b/mobile/lib/shared/services/asset.service.dart index 48f8c635245c5..64a0f28ab7d3f 100644 --- a/mobile/lib/shared/services/asset.service.dart +++ b/mobile/lib/shared/services/asset.service.dart @@ -63,7 +63,7 @@ class AssetService { /// Returns `null` if the server state did not change, else list of assets Future?> _getRemoteAssets(User user) async { - const int chunkSize = 5000; + const int chunkSize = 10000; try { final DateTime now = DateTime.now().toUtc(); final List allAssets = []; diff --git a/server/src/domain/repositories/asset.repository.ts b/server/src/domain/repositories/asset.repository.ts index a8433c7680212..7a2941ab9d74e 100644 --- a/server/src/domain/repositories/asset.repository.ts +++ b/server/src/domain/repositories/asset.repository.ts @@ -1,4 +1,4 @@ -import { AssetSearchOptions, SearchExploreItem } from '@app/domain'; +import { AssetSearchOneToOneRelationOptions, AssetSearchOptions, SearchExploreItem } from '@app/domain'; import { AssetEntity, AssetJobStatusEntity, AssetType, ExifEntity } from '@app/infra/entities'; import { FindOptionsRelations, FindOptionsSelect } from 'typeorm'; import { Paginated, PaginationOptions } from '../domain.util'; @@ -133,6 +133,10 @@ export interface IAssetRepository { getByLibraryIdAndOriginalPath(libraryId: string, originalPath: string): Promise; deleteAll(ownerId: string): Promise; getAll(pagination: PaginationOptions, options?: AssetSearchOptions): Paginated; + getAllByFileCreationDate( + pagination: PaginationOptions, + options?: AssetSearchOneToOneRelationOptions, + ): Paginated; getAllByDeviceId(userId: string, deviceId: string): Promise; updateAll(ids: string[], options: Partial): Promise; save(asset: Pick & Partial): Promise; diff --git a/server/src/domain/repositories/search.repository.ts b/server/src/domain/repositories/search.repository.ts index 35ead53dbc3e4..7183e9e3fe269 100644 --- a/server/src/domain/repositories/search.repository.ts +++ b/server/src/domain/repositories/search.repository.ts @@ -69,7 +69,6 @@ export interface SearchAssetIDOptions { export interface SearchUserIdOptions { deviceId?: string; libraryId?: string; - ownerId?: string; userIds?: string[]; } @@ -147,16 +146,19 @@ export interface SearchPaginationOptions { size: number; } -export type AssetSearchOptions = SearchDateOptions & +type BaseAssetSearchOptions = SearchDateOptions & SearchIdOptions & SearchExifOptions & SearchOrderOptions & SearchPathOptions & - SearchRelationOptions & SearchStatusOptions & SearchUserIdOptions & SearchPeopleOptions; +export type AssetSearchOptions = BaseAssetSearchOptions & SearchRelationOptions; + +export type AssetSearchOneToOneRelationOptions = BaseAssetSearchOptions & SearchOneToOneRelationOptions; + export type AssetSearchBuilderOptions = Omit; export type SmartSearchOptions = SearchDateOptions & diff --git a/server/src/immich/api-v1/asset/asset-repository.ts b/server/src/immich/api-v1/asset/asset-repository.ts index 7d55fa790e385..2f54db27d0cf1 100644 --- a/server/src/immich/api-v1/asset/asset-repository.ts +++ b/server/src/immich/api-v1/asset/asset-repository.ts @@ -1,10 +1,8 @@ -import { AssetEntity, ExifEntity } from '@app/infra/entities'; -import { OptionalBetween } from '@app/infra/infra.utils'; +import { AssetEntity } from '@app/infra/entities'; import { Injectable } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; import { In } from 'typeorm/find-options/operator/In.js'; import { Repository } from 'typeorm/repository/Repository.js'; -import { AssetSearchDto } from './dto/asset-search.dto'; import { CheckExistingAssetsDto } from './dto/check-existing-assets.dto'; import { SearchPropertiesDto } from './dto/search-properties.dto'; import { CuratedLocationsResponseDto } from './response-dto/curated-locations-response.dto'; @@ -21,7 +19,6 @@ export interface AssetOwnerCheck extends AssetCheck { export interface IAssetRepositoryV1 { get(id: string): Promise; - getAllByUserId(userId: string, dto: AssetSearchDto): Promise; getLocationsByUserId(userId: string): Promise; getDetectedObjectsByUserId(userId: string): Promise; getSearchPropertiesByUserId(userId: string): Promise; @@ -34,10 +31,7 @@ export const IAssetRepositoryV1 = 'IAssetRepositoryV1'; @Injectable() export class AssetRepositoryV1 implements IAssetRepositoryV1 { - constructor( - @InjectRepository(AssetEntity) private assetRepository: Repository, - @InjectRepository(ExifEntity) private exifRepository: Repository, - ) {} + constructor(@InjectRepository(AssetEntity) private assetRepository: Repository) {} getSearchPropertiesByUserId(userId: string): Promise { return this.assetRepository @@ -89,33 +83,6 @@ export class AssetRepositoryV1 implements IAssetRepositoryV1 { ); } - /** - * Get all assets belong to the user on the database - * @param ownerId - */ - getAllByUserId(ownerId: string, dto: AssetSearchDto): Promise { - return this.assetRepository.find({ - where: { - ownerId, - isVisible: true, - isFavorite: dto.isFavorite, - isArchived: dto.isArchived, - updatedAt: OptionalBetween(dto.updatedAfter, dto.updatedBefore), - }, - relations: { - exifInfo: true, - tags: true, - stack: { assets: true }, - }, - skip: dto.skip || 0, - take: dto.take, - order: { - fileCreatedAt: 'DESC', - }, - withDeleted: true, - }); - } - get(id: string): Promise { return this.assetRepository.findOne({ where: { id }, diff --git a/server/src/immich/api-v1/asset/asset.service.spec.ts b/server/src/immich/api-v1/asset/asset.service.spec.ts index 9f0aa371e8635..48354d440ec60 100644 --- a/server/src/immich/api-v1/asset/asset.service.spec.ts +++ b/server/src/immich/api-v1/asset/asset.service.spec.ts @@ -77,7 +77,6 @@ describe('AssetService', () => { beforeEach(() => { assetRepositoryMockV1 = { get: jest.fn(), - getAllByUserId: jest.fn(), getDetectedObjectsByUserId: jest.fn(), getLocationsByUserId: jest.fn(), getSearchPropertiesByUserId: jest.fn(), diff --git a/server/src/immich/api-v1/asset/asset.service.ts b/server/src/immich/api-v1/asset/asset.service.ts index f438e55e9c30e..58f800f08cce8 100644 --- a/server/src/immich/api-v1/asset/asset.service.ts +++ b/server/src/immich/api-v1/asset/asset.service.ts @@ -114,8 +114,11 @@ export class AssetService { public async getAllAssets(auth: AuthDto, dto: AssetSearchDto): Promise { const userId = dto.userId || auth.user.id; await this.access.requirePermission(auth, Permission.TIMELINE_READ, userId); - const assets = await this.assetRepositoryV1.getAllByUserId(userId, dto); - return assets.map((asset) => mapAsset(asset, { withStack: true })); + const assets = await this.assetRepository.getAllByFileCreationDate( + { take: dto.take ?? 1000, skip: dto.skip }, + { ...dto, userIds: [userId], withDeleted: true, orderDirection: 'DESC', withExif: true }, + ); + return assets.items.map((asset) => mapAsset(asset)); } async serveThumbnail(auth: AuthDto, assetId: string, dto: GetAssetThumbnailDto): Promise { diff --git a/server/src/infra/entities/asset.entity.ts b/server/src/infra/entities/asset.entity.ts index 10973be74eea1..373271158177c 100644 --- a/server/src/infra/entities/asset.entity.ts +++ b/server/src/infra/entities/asset.entity.ts @@ -85,6 +85,7 @@ export class AssetEntity { @DeleteDateColumn({ type: 'timestamptz', nullable: true }) deletedAt!: Date | null; + @Index('idx_asset_file_created_at') @Column({ type: 'timestamptz' }) fileCreatedAt!: Date; diff --git a/server/src/infra/migrations/1708227417898-AddFileCreatedAtIndex.ts b/server/src/infra/migrations/1708227417898-AddFileCreatedAtIndex.ts new file mode 100644 index 0000000000000..f7ca40cd4633a --- /dev/null +++ b/server/src/infra/migrations/1708227417898-AddFileCreatedAtIndex.ts @@ -0,0 +1,12 @@ +import { MigrationInterface, QueryRunner } from "typeorm"; + +export class AddFileCreatedAtIndex1708227417898 implements MigrationInterface { + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`CREATE INDEX idx_asset_file_created_at ON assets ("fileCreatedAt")`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP INDEX idx_asset_file_created_at`); + } +} diff --git a/server/src/infra/repositories/asset.repository.ts b/server/src/infra/repositories/asset.repository.ts index b60f7285716ec..a31ee2ad44539 100644 --- a/server/src/infra/repositories/asset.repository.ts +++ b/server/src/infra/repositories/asset.repository.ts @@ -2,6 +2,7 @@ import { AssetBuilderOptions, AssetCreate, AssetExploreFieldOptions, + AssetSearchOneToOneRelationOptions, AssetSearchOptions, AssetStats, AssetStatsOptions, @@ -175,8 +176,12 @@ export class AssetRepository implements IAssetRepository { }); } - getByUserId(pagination: PaginationOptions, userId: string, options: AssetSearchOptions = {}): Paginated { - return this.getAll(pagination, { ...options, ownerId: userId }); + getByUserId( + pagination: PaginationOptions, + userId: string, + options: Omit = {}, + ): Paginated { + return this.getAll(pagination, { ...options, userIds: [userId] }); } @GenerateSql({ params: [[DummyValue.UUID]] }) @@ -205,6 +210,29 @@ export class AssetRepository implements IAssetRepository { }); } + @GenerateSql({ + params: [ + { skip: 20_000, take: 10_000 }, + { + takenBefore: DummyValue.DATE, + userIds: [DummyValue.UUID], + }, + ], + }) + getAllByFileCreationDate( + pagination: PaginationOptions, + options: AssetSearchOneToOneRelationOptions = {}, + ): Paginated { + let builder = this.repository.createQueryBuilder('asset'); + builder = searchAssetBuilder(builder, options); + builder.orderBy('asset.fileCreatedAt', options.orderDirection ?? 'DESC'); + return paginatedBuilder(builder, { + mode: PaginationMode.LIMIT_OFFSET, + skip: pagination.skip, + take: pagination.take, + }); + } + /** * Get assets by device's Id on the database * @param ownerId diff --git a/server/src/infra/sql/asset.repository.sql b/server/src/infra/sql/asset.repository.sql index 88f17eb9205f6..d971129e75106 100644 --- a/server/src/infra/sql/asset.repository.sql +++ b/server/src/infra/sql/asset.repository.sql @@ -395,6 +395,55 @@ ORDER BY LIMIT 1 +-- AssetRepository.getAllByFileCreationDate +SELECT + "asset"."id" AS "asset_id", + "asset"."deviceAssetId" AS "asset_deviceAssetId", + "asset"."ownerId" AS "asset_ownerId", + "asset"."libraryId" AS "asset_libraryId", + "asset"."deviceId" AS "asset_deviceId", + "asset"."type" AS "asset_type", + "asset"."originalPath" AS "asset_originalPath", + "asset"."resizePath" AS "asset_resizePath", + "asset"."webpPath" AS "asset_webpPath", + "asset"."thumbhash" AS "asset_thumbhash", + "asset"."encodedVideoPath" AS "asset_encodedVideoPath", + "asset"."createdAt" AS "asset_createdAt", + "asset"."updatedAt" AS "asset_updatedAt", + "asset"."deletedAt" AS "asset_deletedAt", + "asset"."fileCreatedAt" AS "asset_fileCreatedAt", + "asset"."localDateTime" AS "asset_localDateTime", + "asset"."fileModifiedAt" AS "asset_fileModifiedAt", + "asset"."isFavorite" AS "asset_isFavorite", + "asset"."isArchived" AS "asset_isArchived", + "asset"."isExternal" AS "asset_isExternal", + "asset"."isReadOnly" AS "asset_isReadOnly", + "asset"."isOffline" AS "asset_isOffline", + "asset"."checksum" AS "asset_checksum", + "asset"."duration" AS "asset_duration", + "asset"."isVisible" AS "asset_isVisible", + "asset"."livePhotoVideoId" AS "asset_livePhotoVideoId", + "asset"."originalFileName" AS "asset_originalFileName", + "asset"."sidecarPath" AS "asset_sidecarPath", + "asset"."stackId" AS "asset_stackId" +FROM + "assets" "asset" +WHERE + ( + "asset"."fileCreatedAt" <= $1 + AND 1 = 1 + AND "asset"."ownerId" IN ($2) + AND 1 = 1 + AND 1 = 1 + ) + AND ("asset"."deletedAt" IS NULL) +ORDER BY + "asset"."fileCreatedAt" DESC +LIMIT + 10001 +OFFSET + 20000 + -- AssetRepository.getAllByDeviceId SELECT "AssetEntity"."deviceAssetId" AS "AssetEntity_deviceAssetId", diff --git a/server/test/repositories/asset.repository.mock.ts b/server/test/repositories/asset.repository.mock.ts index 1c98b78c9edd7..0be384b3aeff1 100644 --- a/server/test/repositories/asset.repository.mock.ts +++ b/server/test/repositories/asset.repository.mock.ts @@ -18,6 +18,7 @@ export const newAssetRepositoryMock = (): jest.Mocked => { getFirstAssetForAlbumId: jest.fn(), getLastUpdatedAssetForAlbumId: jest.fn(), getAll: jest.fn().mockResolvedValue({ items: [], hasNextPage: false }), + getAllByFileCreationDate: jest.fn(), getAllByDeviceId: jest.fn(), updateAll: jest.fn(), getByLibraryId: jest.fn(),