feat(web): deduplication UI

open-api/immich-openapi-specs.json

@@ -6957,6 +6957,10 @@
           "dateTimeOriginal": {
             "type": "string"
           },
+          "duplicateId": {
+            "nullable": true,
+            "type": "string"
+          },
           "ids": {
             "items": {
               "format": "uuid",

open-api/typescript-sdk/src/fetch-client.ts

@@ -238,6 +238,7 @@ export type AssetBulkDeleteDto = {
 };
 export type AssetBulkUpdateDto = {
     dateTimeOriginal?: string;
+    duplicateId?: string | null;
     ids: string[];
     isArchived?: boolean;
     isFavorite?: boolean;

server/src/dtos/asset.dto.ts

@@ -57,6 +57,9 @@ export class AssetBulkUpdateDto extends UpdateAssetBase {
 
   @ValidateBoolean({ optional: true })
   removeParent?: boolean;
+
+  @Optional()
+  duplicateId?: string | null;
 }
 
 export class UpdateAssetDto extends UpdateAssetBase {

server/src/dtos/duplicate.dto.ts

@@ -1,3 +1,4 @@
+import { IsNotEmpty } from 'class-validator';
 import { groupBy } from 'lodash';
 import { AssetResponseDto } from 'src/dtos/asset-response.dto';
 
@@ -6,6 +7,11 @@ export class DuplicateResponseDto {
   assets!: AssetResponseDto[];
 }
 
+export class ResolveDuplicatesDto {
+  @IsNotEmpty()
+  assetIds!: string[];
+}
+
 export function mapDuplicateResponse(assets: AssetResponseDto[]): DuplicateResponseDto[] {
   const result = [];
 

web/src/lib/components/admin-page/jobs/jobs-panel.svelte

@@ -8,6 +8,7 @@
   import { handleError } from '$lib/utils/handle-error';
   import { JobCommand, JobName, sendJobCommand, type AllJobStatusResponseDto, type JobCommandDto } from '@immich/sdk';
   import {
+    mdiContentDuplicate,
     mdiFaceRecognition,
     mdiFileJpgBox,
     mdiFileXmlBox,
@@ -88,6 +89,12 @@
       subtitle: 'Run machine learning on assets to support smart search',
       disabled: !$featureFlags.smartSearch,
     },
+    [JobName.DuplicateDetection]: {
+      icon: mdiContentDuplicate,
+      title: getJobName(JobName.DuplicateDetection),
+      subtitle: 'Run machine learning on assets to detect similar images. Relies on Smart Search',
+      disabled: !$featureFlags.duplicateDetection,
+    },
     [JobName.FaceDetection]: {
       icon: mdiFaceRecognition,
       title: getJobName(JobName.FaceDetection),

web/src/lib/components/admin-page/settings/machine-learning-settings/machine-learning-settings.svelte

@@ -77,6 +77,40 @@
         </div>
       </SettingAccordion>
 
+      <SettingAccordion
+        key="duplicate-detection"
+        title="Duplicate Detection"
+        subtitle="Use CLIP embeddings to find likely duplicates"
+      >
+        <div class="ml-4 mt-4 flex flex-col gap-4">
+          <SettingSwitch
+            id="enable-duplicate-detection"
+            title="ENABLED"
+            subtitle="If disabled, exactly identical assets will still be de-duplicated."
+            bind:checked={config.machineLearning.duplicateDetection.enabled}
+            disabled={disabled || !config.machineLearning.enabled || !config.machineLearning.clip.enabled}
+          />
+
+          <hr />
+
+          <SettingInputField
+            inputType={SettingInputFieldType.NUMBER}
+            label="MAX DETECTION DISTANCE"
+            bind:value={config.machineLearning.duplicateDetection.maxDistance}
+            step="0.01"
+            min={0.001}
+            max={0.1}
+            desc="Maximum distance between two images to consider them duplicates, ranging from 0.001-0.1. Higher values will detect more duplicates, but may result in false positives."
+            disabled={disabled ||
+              !config.machineLearning.enabled ||
+              !config.machineLearning.clip.enabled ||
+              !config.machineLearning.duplicateDetection.enabled}
+            isEdited={config.machineLearning.duplicateDetection.maxDistance !==
+              savedConfig.machineLearning.duplicateDetection.maxDistance}
+          />
+        </div>
+      </SettingAccordion>
+
       <SettingAccordion
         key="facial-recognition"
         title="Facial Recognition"

web/src/lib/components/elements/buttons/button.svelte

@@ -43,7 +43,7 @@
       'text-immich-primary dark:text-immich-dark-primary enabled:dark:hover:bg-immich-dark-primary/10 enabled:hover:bg-immich-primary/10',
     'light-red': 'bg-[#F9DEDC] text-[#410E0B] enabled:hover:bg-red-50',
     red: 'bg-red-500 text-white enabled:hover:bg-red-400',
-    green: 'bg-green-500 text-gray-800 enabled:hover:bg-green-400/90',
+    green: 'bg-green-400 text-gray-800 enabled:hover:bg-green-400/90',
     gray: 'bg-gray-500 dark:bg-gray-200 enabled:hover:bg-gray-500/75 enabled:dark:hover:bg-gray-200/80 text-white dark:text-immich-dark-gray',
     'transparent-gray':
       'dark:text-immich-dark-fg enabled:hover:bg-immich-primary/5 enabled:hover:text-gray-700 enabled:hover:dark:text-immich-dark-fg enabled:dark:hover:bg-immich-dark-primary/25',

web/src/lib/components/shared-components/side-bar/side-bar.svelte

@@ -19,6 +19,8 @@
     mdiMapOutline,
     mdiTrashCan,
     mdiTrashCanOutline,
+    mdiToolbox,
+    mdiToolboxOutline,
   } from '@mdi/js';
   import LoadingSpinner from '../loading-spinner.svelte';
   import StatusBox from '../status-box.svelte';
@@ -42,6 +44,7 @@
   let isPhotosSelected: boolean;
   let isSharingSelected: boolean;
   let isTrashSelected: boolean;
+  let isUtilitiesSelected: boolean;
 </script>
 
 <SideBarSection>
@@ -136,6 +139,13 @@
       </svelte:fragment>
     </SideBarLink>
 
+    <SideBarLink
+      title="Utilities"
+      routeId="/(user)/utilities"
+      bind:isSelected={isUtilitiesSelected}
+      icon={isUtilitiesSelected ? mdiToolbox : mdiToolboxOutline}
+    ></SideBarLink>
+
     <SideBarLink
       title="Archive"
       routeId="/(user)/archive"

web/src/lib/components/utilities-page/duplicates/duplicates-compare-control.svelte

@@ -0,0 +1,133 @@
+<script lang="ts">
+  import Button from '$lib/components/elements/buttons/button.svelte';
+  import Icon from '$lib/components/elements/icon.svelte';
+  import { getAssetThumbnailUrl } from '$lib/utils';
+  import { ThumbnailFormat, type AssetResponseDto, type DuplicateResponseDto, getAllAlbums } from '@immich/sdk';
+  import { mdiCheck, mdiTrashCanOutline } from '@mdi/js';
+  import { onMount } from 'svelte';
+  import { s } from '$lib/utils';
+  import { getAssetResolution, getFileSize } from '$lib/utils/asset-utils';
+  import { sortBy } from 'lodash-es';
+
+  export let duplicate: DuplicateResponseDto;
+  export let onResolve: (duplicateAssetIds: string[], trashIds: string[]) => void;
+
+  let selectedAssetIds = new Set<string>();
+
+  $: trashCount = duplicate.assets.length - selectedAssetIds.size;
+
+  onMount(() => {
+    const suggestedAsset = sortBy(duplicate.assets, (asset) => asset.exifInfo?.fileSizeInByte).pop();
+
+    if (!suggestedAsset) {
+      selectedAssetIds = new Set(duplicate.assets[0].id);
+      return;
+    }
+
+    selectedAssetIds.add(suggestedAsset.id);
+    selectedAssetIds = selectedAssetIds;
+  });
+
+  const onSelectAsset = (asset: AssetResponseDto) => {
+    if (selectedAssetIds.has(asset.id)) {
+      selectedAssetIds.delete(asset.id);
+    } else {
+      selectedAssetIds.add(asset.id);
+    }
+
+    selectedAssetIds = selectedAssetIds;
+  };
+
+  const handleResolve = () => {
+    const trashIds = duplicate.assets.map((asset) => asset.id).filter((id) => !selectedAssetIds.has(id));
+    const duplicateAssetIds = duplicate.assets.map((asset) => asset.id);
+    onResolve(duplicateAssetIds, trashIds);
+  };
+</script>
+
+<div class="pt-4 rounded-3xl border dark:border-2 border-gray-300 dark:border-gray-700 max-w-[900px] m-auto mb-16">
+  <div class="flex flex-wrap gap-1 place-items-center place-content-center px-4 pt-4">
+    {#each duplicate.assets as asset, index (index)}
+      {@const isSelected = selectedAssetIds.has(asset.id)}
+      {@const isFromExternalLibrary = !!asset.libraryId}
+      {@const assetData = JSON.stringify(asset, null, 2)}
+
+      <div class="relative">
+        <button on:click={() => onSelectAsset(asset)} class="block relative">
+          <!-- THUMBNAIL-->
+          <img
+            src={getAssetThumbnailUrl(asset.id, ThumbnailFormat.Webp)}
+            alt={asset.id}
+            title={`${assetData}`}
+            class={`w-[250px] h-[250px] object-cover rounded-t-xl border-t-[4px] border-l-[4px] border-r-[4px] border-gray-300 ${isSelected ? 'border-immich-primary dark:border-immich-dark-primary' : 'dark:border-gray-800'} transition-all`}
+            draggable="false"
+          />
+
+          <!-- OVERLAY CHIP -->
+          <div
+            class={`absolute bottom-2 right-3 ${isSelected ? 'bg-green-400/90' : 'bg-red-300/90'} px-4 py-1 rounded-xl text-xs font-semibold`}
+          >
+            {isSelected ? 'Keep' : 'Trash'}
+          </div>
+
+          <!-- EXTERNAL LIBRARY CHIP-->
+          {#if isFromExternalLibrary}
+            <div
+              class="absolute top-2 right-3 bg-immich-primary/90 px-4 py-1 rounded-xl text-xs font-semibold text-white"
+            >
+              External
+            </div>
+          {/if}
+        </button>
+
+        <!-- ASSET INFO-->
+        <table
+          class={`text-xs w-full rounded-b-xl font-semibold ${isSelected ? 'bg-immich-primary text-white dark:bg-immich-dark-primary dark:text-black' : 'bg-gray-200 dark:bg-gray-800 dark:text-white'} mt-0 transition-all`}
+        >
+          <tr
+            class={`h-[32px] ${isSelected ? 'border-immich-primary rounded-xl dark:border-immich-dark-primary' : 'border-gray-300'} text-center `}
+          >
+            <td>{asset.originalFileName}</td>
+          </tr>
+
+          <tr
+            class={`h-[32px] ${isSelected ? 'border-immich-primary rounded-xl dark:border-immich-dark-primary' : 'border-gray-300'} text-center`}
+          >
+            <td>{getAssetResolution(asset)} - {getFileSize(asset)}</td>
+          </tr>
+
+          <tr
+            class={`h-[32px] ${isSelected ? 'border-immich-primary rounded-xl dark:border-immich-dark-primary' : 'border-gray-300'} text-center `}
+          >
+            <td>
+              {#await getAllAlbums({ assetId: asset.id })}
+                Scanning for album...
+              {:then albums}
+                {#if albums.length === 0}
+                  Not in any album
+                {:else}
+                  In {albums.length} album{s(albums.length)}
+                {/if}
+              {/await}
+            </td>
+          </tr>
+        </table>
+      </div>
+    {/each}
+  </div>
+
+  <!-- CONFIRM BUTTONS -->
+  <div class="flex gap-4 my-4 border-transparent w-full justify-end p-4 h-[85px]">
+    {#if trashCount === 0}
+      <Button size="sm" color="primary" class="flex place-items-center gap-2" on:click={handleResolve}
+        ><Icon path={mdiCheck} size="20" />Keep All
+      </Button>
+    {:else}
+      <Button size="sm" color="red" class="flex place-items-center gap-2" on:click={handleResolve}
+        ><Icon path={mdiTrashCanOutline} size="20" />{trashCount === duplicate.assets.length
+          ? 'Trash All'
+          : `Trash ${trashCount}`}
+      </Button>
+    {/if}
+  </div>
+</div>

web/src/lib/components/utilities-page/utilities-menu.svelte

@@ -0,0 +1,18 @@
+<script lang="ts">
+  import { mdiContentDuplicate } from '@mdi/js';
+  import Icon from '$lib/components/elements/icon.svelte';
+  import { AppRoute } from '$lib/constants';
+</script>
+
+<a href={AppRoute.DUPLICATES}>
+  <div class="border border-gray-300 dark:border-immich-dark-gray rounded-3xl pt-1 pb-6 dark:text-white">
+    <p class="text-xs font-medium p-4">ORGANIZE YOUR LIBRARY</p>
+
+    <button class="w-full hover:bg-gray-100 dark:hover:bg-immich-dark-gray flex gap-4 p-4">
+      <span
+        ><Icon path={mdiContentDuplicate} class="text-immich-primary dark:text-immich-dark-primary" size="24" />
+      </span>
+      Review duplicates
+    </button>
+  </div>
+</a>

web/src/lib/constants.ts

@@ -39,6 +39,9 @@ export enum AppRoute {
   AUTH_REGISTER = '/auth/register',
   AUTH_CHANGE_PASSWORD = '/auth/change-password',
   AUTH_ONBOARDING = '/auth/onboarding',
+
+  UTILITIES = '/utilities',
+  DUPLICATES = '/utilities/duplicates',
 }
 
 export enum ProjectionType {