Add AV1 transcoding support (#8491)

* Add AV1 transcoding support

- AV1 encoding on CPU via SVT-AV1 (libsvtav1 in ffmpeg)
- Supports CRF and optionally capped CRF (max bitrate)
- Tested playback successfully in Chrome Win+Android, Firefox Win+Linux, Android app

* AV1: Add support for encoding threads option

* Revert previous commit; specifying params multiple times is bad

We need to specify all svtav1-params at once, so putting the thread option into getThreadOptions is not possible.

* AV1: Override VAAPI getSupportedCodecs as it does not yet support AV1 unlike nvenc, qsv, amf

* Change BaseHWConfig supported codecs to only H264/HEVC

Configs that support VP9 and/or AV1 need to override getSupportedCodecs()

* Set SVT-AV1 threads with svtav1-params, remove duplicate block in NVENCConfig

* AV1Config: Fix empty svtav1-params array being added to options

* add tests

* update api

* allow crf-based two-pass mode

* formatting

* suggest 35

---------

Co-authored-by: mertalev <101130780+mertalev@users.noreply.github.com>

open-api/immich-openapi-specs.json

@@ -11231,7 +11231,8 @@
         "enum": [
           "h264",
           "hevc",
-          "vp9"
+          "vp9",
+          "av1"
         ],
         "type": "string"
       }

open-api/typescript-sdk/src/fetch-client.ts

@@ -2984,7 +2984,8 @@ export enum AudioCodec {
 export enum VideoCodec {
     H264 = "h264",
     Hevc = "hevc",
-    Vp9 = "vp9"
+    Vp9 = "vp9",
+    Av1 = "av1"
 }
 export enum CQMode {
     Auto = "auto",

server/src/entities/system-config.entity.ts

@@ -153,6 +153,7 @@ export enum VideoCodec {
   H264 = 'h264',
   HEVC = 'hevc',
   VP9 = 'vp9',
+  AV1 = 'av1',
 }
 
 export enum AudioCodec {

server/src/services/media.service.spec.ts

@@ -1268,6 +1268,157 @@ describe(MediaService.name, () => {
       );
     });
 
+    it('should use av1 if specified', async () => {
+      mediaMock.probe.mockResolvedValue(probeStub.videoStreamVp9);
+      configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: VideoCodec.AV1 }]);
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+      await sut.handleVideoConversion({ id: assetStub.video.id });
+      expect(mediaMock.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
+        {
+          inputOptions: [],
+          outputOptions: [
+            '-c:v av1',
+            '-c:a copy',
+            '-movflags faststart',
+            '-fps_mode passthrough',
+            '-map 0:0',
+            '-map 0:1',
+            '-v verbose',
+            '-vf scale=-2:720,format=yuv420p',
+            '-preset 12',
+            '-crf 23',
+          ],
+          twoPass: false,
+        },
+      );
+    });
+
+    it('should map `veryslow` preset to 4 for av1', async () => {
+      mediaMock.probe.mockResolvedValue(probeStub.videoStreamVp9);
+      configMock.load.mockResolvedValue([
+        { key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: VideoCodec.AV1 },
+        { key: SystemConfigKey.FFMPEG_PRESET, value: 'veryslow' },
+      ]);
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+      await sut.handleVideoConversion({ id: assetStub.video.id });
+      expect(mediaMock.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
+        {
+          inputOptions: [],
+          outputOptions: [
+            '-c:v av1',
+            '-c:a copy',
+            '-movflags faststart',
+            '-fps_mode passthrough',
+            '-map 0:0',
+            '-map 0:1',
+            '-v verbose',
+            '-vf scale=-2:720,format=yuv420p',
+            '-preset 4',
+            '-crf 23',
+          ],
+          twoPass: false,
+        },
+      );
+    });
+
+    it('should set max bitrate for av1 if specified', async () => {
+      mediaMock.probe.mockResolvedValue(probeStub.videoStreamVp9);
+      configMock.load.mockResolvedValue([
+        { key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: VideoCodec.AV1 },
+        { key: SystemConfigKey.FFMPEG_MAX_BITRATE, value: '2M' },
+      ]);
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+      await sut.handleVideoConversion({ id: assetStub.video.id });
+      expect(mediaMock.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
+        {
+          inputOptions: [],
+          outputOptions: [
+            '-c:v av1',
+            '-c:a copy',
+            '-movflags faststart',
+            '-fps_mode passthrough',
+            '-map 0:0',
+            '-map 0:1',
+            '-v verbose',
+            '-vf scale=-2:720,format=yuv420p',
+            '-preset 12',
+            '-crf 23',
+            '-svtav1-params mbr=2M',
+          ],
+          twoPass: false,
+        },
+      );
+    });
+
+    it('should set threads for av1 if specified', async () => {
+      mediaMock.probe.mockResolvedValue(probeStub.videoStreamVp9);
+      configMock.load.mockResolvedValue([
+        { key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: VideoCodec.AV1 },
+        { key: SystemConfigKey.FFMPEG_THREADS, value: 4 },
+      ]);
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+      await sut.handleVideoConversion({ id: assetStub.video.id });
+      expect(mediaMock.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
+        {
+          inputOptions: [],
+          outputOptions: [
+            '-c:v av1',
+            '-c:a copy',
+            '-movflags faststart',
+            '-fps_mode passthrough',
+            '-map 0:0',
+            '-map 0:1',
+            '-v verbose',
+            '-vf scale=-2:720,format=yuv420p',
+            '-preset 12',
+            '-crf 23',
+            '-svtav1-params lp=4',
+          ],
+          twoPass: false,
+        },
+      );
+    });
+
+    it('should set both bitrate and threads for av1 if specified', async () => {
+      mediaMock.probe.mockResolvedValue(probeStub.videoStreamVp9);
+      configMock.load.mockResolvedValue([
+        { key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: VideoCodec.AV1 },
+        { key: SystemConfigKey.FFMPEG_THREADS, value: 4 },
+        { key: SystemConfigKey.FFMPEG_MAX_BITRATE, value: '2M' },
+      ]);
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+      await sut.handleVideoConversion({ id: assetStub.video.id });
+      expect(mediaMock.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
+        {
+          inputOptions: [],
+          outputOptions: [
+            '-c:v av1',
+            '-c:a copy',
+            '-movflags faststart',
+            '-fps_mode passthrough',
+            '-map 0:0',
+            '-map 0:1',
+            '-v verbose',
+            '-vf scale=-2:720,format=yuv420p',
+            '-preset 12',
+            '-crf 23',
+            '-svtav1-params lp=4:mbr=2M',
+          ],
+          twoPass: false,
+        },
+      );
+    });
+
     it('should skip transcoding for audioless videos with optimal policy if video codec is correct', async () => {
       mediaMock.probe.mockResolvedValue(probeStub.noAudioStreams);
       configMock.load.mockResolvedValue([

server/src/services/media.service.ts

@@ -32,6 +32,7 @@ import { IStorageRepository } from 'src/interfaces/storage.interface';
 import { ISystemConfigRepository } from 'src/interfaces/system-config.interface';
 import { ImmichLogger } from 'src/utils/logger';
 import {
+  AV1Config,
   H264Config,
   HEVCConfig,
   NVENCConfig,
@@ -439,6 +440,9 @@ export class MediaService {
       case VideoCodec.VP9: {
         return new VP9Config(config);
       }
+      case VideoCodec.AV1: {
+        return new AV1Config(config);
+      }
       default: {
         throw new UnsupportedMediaTypeException(`Codec '${config.targetVideoCodec}' is unsupported`);
       }

server/src/utils/media.ts

@@ -124,7 +124,7 @@ class BaseConfig implements VideoCodecSWConfig {
       return false;
     }
 
-    return this.isBitrateConstrained() || this.config.targetVideoCodec === VideoCodec.VP9;
+    return this.isBitrateConstrained();
   }
 
   getBitrateDistribution() {
@@ -265,7 +265,7 @@ export class BaseHWConfig extends BaseConfig implements VideoCodecHWConfig {
   }
 
   getSupportedCodecs() {
-    return [VideoCodec.H264, VideoCodec.HEVC, VideoCodec.VP9];
+    return [VideoCodec.H264, VideoCodec.HEVC];
   }
 
   validateDevices(devices: string[]) {
@@ -394,6 +394,44 @@ export class VP9Config extends BaseConfig {
   getThreadOptions() {
     return ['-row-mt 1', ...super.getThreadOptions()];
   }
+
+  eligibleForTwoPass() {
+    return this.config.twoPass;
+  }
+}
+
+export class AV1Config extends BaseConfig {
+  getPresetOptions() {
+    const speed = this.getPresetIndex() + 4; // Use 4 as slowest, giving us an effective range of 4-12 which is far more useful than 0-8
+    if (speed >= 0) {
+      return [`-preset ${speed}`];
+    }
+    return [];
+  }
+
+  getBitrateOptions() {
+    const options = [`-crf ${this.config.crf}`];
+    const bitrates = this.getBitrateDistribution();
+    const svtparams = [];
+    if (this.config.threads > 0) {
+      svtparams.push(`lp=${this.config.threads}`);
+    }
+    if (bitrates.max > 0) {
+      svtparams.push(`mbr=${bitrates.max}${bitrates.unit}`);
+    }
+    if (svtparams.length > 0) {
+      options.push(`-svtav1-params ${svtparams.join(':')}`);
+    }
+    return options;
+  }
+
+  getThreadOptions() {
+    return []; // Already set above with svtav1-params
+  }
+
+  eligibleForTwoPass() {
+    return this.config.twoPass;
+  }
 }
 
 export class NVENCConfig extends BaseHWConfig {
@@ -527,6 +565,10 @@ export class QSVConfig extends BaseHWConfig {
     return options;
   }
 
+  getSupportedCodecs() {
+    return [VideoCodec.H264, VideoCodec.HEVC, VideoCodec.VP9];
+  }
+
   // recommended from https://github.com/intel/media-delivery/blob/master/doc/benchmarks/intel-iris-xe-max-graphics/intel-iris-xe-max-graphics.md
   getBFrames() {
     if (this.config.bframes < 0) {
@@ -605,6 +647,10 @@ export class VAAPIConfig extends BaseHWConfig {
     return options;
   }
 
+  getSupportedCodecs() {
+    return [VideoCodec.H264, VideoCodec.HEVC, VideoCodec.VP9];
+  }
+
   useCQP() {
     return this.config.cqMode !== CQMode.ICQ || this.config.targetVideoCodec === VideoCodec.VP9;
   }

web/src/lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte

@@ -54,7 +54,7 @@
           inputType={SettingInputFieldType.NUMBER}
           {disabled}
           label="CONSTANT RATE FACTOR (-crf)"
-          desc="Video quality level. Typical values are 23 for H.264, 28 for HEVC, and 31 for VP9. Lower is better, but takes longer to encode and produces larger files."
+          desc="Video quality level. Typical values are 23 for H.264, 28 for HEVC, 31 for VP9, and 35 for AV1. Lower is better, but produces larger files."
           bind:value={config.ffmpeg.crf}
           required={true}
           isEdited={config.ffmpeg.crf !== savedConfig.ffmpeg.crf}
@@ -115,12 +115,13 @@
         <SettingSelect
           label="VIDEO CODEC"
           {disabled}
-          desc="VP9 has high efficiency and web compatibility, but takes longer to transcode. HEVC performs similarly, but has lower web compatibility. H.264 is widely compatible and quick to transcode, but produces much larger files."
+          desc="VP9 has high efficiency and web compatibility, but takes longer to transcode. HEVC performs similarly, but has lower web compatibility. H.264 is widely compatible and quick to transcode, but produces much larger files. AV1 is the most efficient codec but lacks support on older devices."
           bind:value={config.ffmpeg.targetVideoCodec}
           options={[
             { value: VideoCodec.H264, text: 'h264' },
             { value: VideoCodec.Hevc, text: 'hevc' },
             { value: VideoCodec.Vp9, text: 'vp9' },
+            { value: VideoCodec.Av1, text: 'av1' },
           ]}
           name="vcodec"
           isEdited={config.ffmpeg.targetVideoCodec !== savedConfig.ffmpeg.targetVideoCodec}
@@ -137,6 +138,7 @@
             { value: VideoCodec.H264, text: 'H.264' },
             { value: VideoCodec.Hevc, text: 'HEVC' },
             { value: VideoCodec.Vp9, text: 'VP9' },
+            { value: VideoCodec.Av1, text: 'AV1' },
           ]}
           isEdited={!isEqual(sortBy(config.ffmpeg.acceptedVideoCodecs), sortBy(savedConfig.ffmpeg.acceptedVideoCodecs))}
         />