Skip to content

Commit f1d8ab8

Browse files
mertalevmertalev
authored
feat(server): track video metadata (#28023)
* track video metadata * earlier duration check * revert colorspace change * duplicate constant * formatting * linting * add comments * redundant variable * simplify tests * use totalDuration instead of format.duration * medium tests * install ffmpeg * install noble * update test-assets commit * make timeBase non-nullable * linting * use proper smallint * add ffmpeg to mise * simplify duration * regenerate migration
1 parent c0898b9 commit f1d8ab8

31 files changed

Lines changed: 1693 additions & 385 deletions

‎.github/workflows/test.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -392,6 +392,8 @@ jobs:
392392
node-version-file: './server/.nvmrc'
393393
cache: 'pnpm'
394394
cache-dependency-path: '**/pnpm-lock.yaml'
395+
- name: Setup Mise
396+
uses: immich-app/devtools/actions/use-mise@035e80a7d4355d5f087ffb95db9e4a0944c04e56 # use-mise-action-v1.1.3
395397
- name: Run pnpm install
396398
run: SHARP_IGNORE_GLOBAL_LIBVIPS=true pnpm install --frozen-lockfile
397399
- name: Run medium tests

‎e2e/test-assets‎

‎mise.toml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,15 @@ version = "1.35.1"
2626
bin = "dcm"
2727
postinstall = "chmod +x $MISE_TOOL_INSTALL_PATH/dcm"
2828

29+
[tools."github:jellyfin/jellyfin-ffmpeg"]
30+
version = "7.1.3-6"
31+
32+
[tools."github:jellyfin/jellyfin-ffmpeg".platforms]
33+
linux-x64 = { asset_pattern = "jellyfin-ffmpeg_*_portable_linux64-gpl.tar.xz" }
34+
linux-arm64 = { asset_pattern = "jellyfin-ffmpeg_*_portable_linuxarm64-gpl.tar.xz" }
35+
macos-x64 = { asset_pattern = "jellyfin-ffmpeg_*_portable_mac64-gpl.tar.xz" }
36+
macos-arm64 = { asset_pattern = "jellyfin-ffmpeg_*_portable_macarm64-gpl.tar.xz" }
37+
2938
[settings]
3039
experimental = true
3140
pin = true

‎server/eslint.config.mjs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ export default typescriptEslint.config([
4848
'unicorn/import-style': 'off',
4949
'unicorn/prefer-structured-clone': 'off',
5050
'unicorn/no-for-loop': 'off',
51+
'unicorn/no-array-sort': 'off',
5152
'@typescript-eslint/await-thenable': 'error',
5253
'@typescript-eslint/no-misused-promises': 'error',
5354
'require-await': 'off',

‎server/src/enum.ts‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -603,6 +603,133 @@ export enum ExifOrientation {
603603
Rotate270CW = 8,
604604
}
605605

606+
/** ITU-T H.273 colour primaries codes. */
607+
export enum ColorPrimaries {
608+
Reserved = 0,
609+
Bt709 = 1,
610+
Unknown = 2,
611+
Bt470M = 4,
612+
Bt470Bg = 5,
613+
Smpte170M = 6,
614+
Smpte240M = 7,
615+
Film = 8,
616+
Bt2020 = 9,
617+
Smpte428 = 10,
618+
Smpte431 = 11,
619+
Smpte432 = 12,
620+
Ebu3213 = 22,
621+
}
622+
623+
/** ITU-T H.273 transfer characteristics codes. */
624+
export enum ColorTransfer {
625+
Reserved = 0,
626+
Bt709 = 1,
627+
Unknown = 2,
628+
Bt470M = 4,
629+
Bt470Bg = 5,
630+
Smpte170M = 6,
631+
Smpte240M = 7,
632+
Linear = 8,
633+
Log100 = 9,
634+
Log316 = 10,
635+
Iec6196624 = 11,
636+
Bt1361E = 12,
637+
Iec6196621 = 13,
638+
Bt202010 = 14,
639+
Bt202012 = 15,
640+
Smpte2084 = 16,
641+
Smpte428 = 17,
642+
AribStdB67 = 18,
643+
}
644+
645+
/** ITU-T H.273 matrix coefficients codes. */
646+
export enum ColorMatrix {
647+
Gbr = 0,
648+
Bt709 = 1,
649+
Unknown = 2,
650+
Reserved = 3,
651+
Fcc = 4,
652+
Bt470Bg = 5,
653+
Smpte170M = 6,
654+
Smpte240M = 7,
655+
Ycgco = 8,
656+
Bt2020Nc = 9,
657+
Bt2020C = 10,
658+
Smpte2085 = 11,
659+
ChromaDerivedNc = 12,
660+
ChromaDerivedC = 13,
661+
Ictcp = 14,
662+
}
663+
664+
/** H.264 `profile_idc` values. */
665+
// H.264 has a few profiles that have the same value but different names, included so lookup by name works
666+
export enum H264Profile {
667+
ConstrainedBaseline = 66,
668+
// eslint-disable-next-line @typescript-eslint/no-duplicate-enum-values
669+
Baseline = 66,
670+
Main = 77,
671+
Extended = 88,
672+
ConstrainedHigh = 100,
673+
// eslint-disable-next-line @typescript-eslint/no-duplicate-enum-values
674+
ProgressiveHigh = 100,
675+
// eslint-disable-next-line @typescript-eslint/no-duplicate-enum-values
676+
High = 100,
677+
High10 = 110,
678+
High422 = 122,
679+
High444Predictive = 244,
680+
}
681+
682+
/** HEVC `profile_idc` values. */
683+
export enum HevcProfile {
684+
Main = 1,
685+
Main10 = 2,
686+
MainStillPicture = 3,
687+
Rext = 4,
688+
}
689+
690+
/** AV1 `seq_profile` values. */
691+
export enum Av1Profile {
692+
Main = 0,
693+
High = 1,
694+
Professional = 2,
695+
}
696+
697+
/** MPEG-4 Audio Object Type values for AAC. */
698+
export enum AacProfile {
699+
Main = 1,
700+
Lc = 2,
701+
Ssr = 3,
702+
Ltp = 4,
703+
HeAac = 5,
704+
Ld = 23,
705+
HeAacv2 = 29,
706+
Eld = 39,
707+
XheAac = 42,
708+
}
709+
710+
/** Dolby Vision bitstream profile numbers from the DOVI configuration record. */
711+
export enum DvProfile {
712+
Dvhe03 = 3,
713+
Dvhe04 = 4,
714+
Dvhe05 = 5,
715+
Dvhe07 = 7,
716+
Dvhe08 = 8,
717+
Dvav09 = 9,
718+
Dav110 = 10,
719+
}
720+
721+
/**
722+
* Dolby Vision base-layer signal-compatibility ID from the DOVI configuration record.
723+
* Identifies what the base HEVC/AVC layer renders as on a non-DV decoder.
724+
*/
725+
export enum DvSignalCompatibility {
726+
None = 0,
727+
Hdr10 = 1,
728+
Sdr709 = 2,
729+
Hlg = 4,
730+
Sdr2020 = 6,
731+
}
732+
606733
export enum DatabaseExtension {
607734
Cube = 'cube',
608735
EarthDistance = 'earthdistance',

‎server/src/queries/asset.job.repository.sql‎

Lines changed: 139 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -239,10 +239,68 @@ select
239239
"asset_edit"."assetId" = "asset"."id"
240240
) as agg
241241
) as "edits",
242-
to_json("asset_exif") as "exifInfo"
242+
to_json("asset_exif") as "exifInfo",
243+
(
244+
select
245+
to_json(obj)
246+
from
247+
(
248+
select
249+
"asset_video"."index",
250+
"asset_video"."codecName",
251+
"asset_video"."profile",
252+
"asset_video"."level",
253+
"asset_video"."bitrate",
254+
"asset_exif"."exifImageWidth" as "width",
255+
"asset_exif"."exifImageHeight" as "height",
256+
"asset_video"."pixelFormat",
257+
"asset_video"."frameCount",
258+
"asset_exif"."fps" as "frameRate",
259+
"asset_video"."timeBase",
260+
case
261+
when "asset_exif"."orientation" = '6' then -90
262+
when "asset_exif"."orientation" = '8' then 90
263+
when "asset_exif"."orientation" = '3' then 180
264+
else 0
265+
end as "rotation",
266+
"asset_video"."colorPrimaries",
267+
"asset_video"."colorMatrix",
268+
"asset_video"."colorTransfer",
269+
"asset_video"."dvProfile",
270+
"asset_video"."dvLevel",
271+
"asset_video"."dvBlSignalCompatibilityId"
272+
from
273+
(
274+
select
275+
1
276+
) as "dummy"
277+
where
278+
"asset_video"."assetId" is not null
279+
) as obj
280+
) as "videoStream",
281+
(
282+
select
283+
to_json(obj)
284+
from
285+
(
286+
select
287+
"asset_video"."formatName",
288+
"asset_video"."formatLongName",
289+
"asset"."duration",
290+
"asset_video"."bitrate"
291+
from
292+
(
293+
select
294+
1
295+
) as "dummy"
296+
where
297+
"asset_video"."assetId" is not null
298+
) as obj
299+
) as "format"
243300
from
244301
"asset"
245302
inner join "asset_exif" on "asset"."id" = "asset_exif"."assetId"
303+
left join "asset_video" on "asset_video"."assetId" = "asset"."id"
246304
where
247305
"asset"."id" = $4
248306

@@ -554,9 +612,88 @@ select
554612
where
555613
"asset_file"."assetId" = "asset"."id"
556614
) as agg
557-
) as "files"
615+
) as "files",
616+
(
617+
select
618+
to_json(obj)
619+
from
620+
(
621+
select
622+
"asset_audio"."index",
623+
"asset_audio"."codecName",
624+
"asset_audio"."profile",
625+
"asset_audio"."bitrate"
626+
from
627+
(
628+
select
629+
1
630+
) as "dummy"
631+
where
632+
"asset_audio"."assetId" is not null
633+
) as obj
634+
) as "audioStream",
635+
(
636+
select
637+
to_json(obj)
638+
from
639+
(
640+
select
641+
"asset_video"."index",
642+
"asset_video"."codecName",
643+
"asset_video"."profile",
644+
"asset_video"."level",
645+
"asset_video"."bitrate",
646+
"asset_exif"."exifImageWidth" as "width",
647+
"asset_exif"."exifImageHeight" as "height",
648+
"asset_video"."pixelFormat",
649+
"asset_video"."frameCount",
650+
"asset_exif"."fps" as "frameRate",
651+
"asset_video"."timeBase",
652+
case
653+
when "asset_exif"."orientation" = '6' then -90
654+
when "asset_exif"."orientation" = '8' then 90
655+
when "asset_exif"."orientation" = '3' then 180
656+
else 0
657+
end as "rotation",
658+
"asset_video"."colorPrimaries",
659+
"asset_video"."colorMatrix",
660+
"asset_video"."colorTransfer",
661+
"asset_video"."dvProfile",
662+
"asset_video"."dvLevel",
663+
"asset_video"."dvBlSignalCompatibilityId"
664+
from
665+
(
666+
select
667+
1
668+
) as "dummy"
669+
where
670+
"asset_video"."assetId" is not null
671+
) as obj
672+
) as "videoStream",
673+
(
674+
select
675+
to_json(obj)
676+
from
677+
(
678+
select
679+
"asset_video"."formatName",
680+
"asset_video"."formatLongName",
681+
"asset"."duration",
682+
"asset_video"."bitrate"
683+
from
684+
(
685+
select
686+
1
687+
) as "dummy"
688+
where
689+
"asset_video"."assetId" is not null
690+
) as obj
691+
) as "format"
558692
from
559693
"asset"
694+
inner join "asset_exif" on "asset"."id" = "asset_exif"."assetId"
695+
inner join "asset_video" on "asset_video"."assetId" = "asset"."id"
696+
left join "asset_audio" on "asset_audio"."assetId" = "asset"."id"
560697
where
561698
"asset"."id" = $1
562699
and "asset"."type" = 'VIDEO'

‎server/src/repositories/asset-job.repository.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,16 @@ import { DB } from 'src/schema';
99
import {
1010
anyUuid,
1111
asUuid,
12+
withAudioStream,
1213
withDefaultVisibility,
1314
withEdits,
1415
withExif,
1516
withExifInner,
1617
withFaces,
1718
withFilePath,
1819
withFiles,
20+
withVideoFormat,
21+
withVideoStream,
1922
} from 'src/utils/database';
2023
import { mimeTypes } from 'src/utils/mime-types';
2124

@@ -134,6 +137,9 @@ export class AssetJobRepository {
134137
)
135138
.select(withEdits)
136139
.$call(withExifInner)
140+
.leftJoin('asset_video', 'asset_video.assetId', 'asset.id')
141+
.select((eb) => withVideoStream(eb).as('videoStream'))
142+
.select((eb) => withVideoFormat(eb).as('format'))
137143
.where('asset.id', '=', id)
138144
.executeTakeFirst();
139145
}
@@ -333,8 +339,14 @@ export class AssetJobRepository {
333339
getForVideoConversion(id: string) {
334340
return this.db
335341
.selectFrom('asset')
342+
.innerJoin('asset_exif', 'asset.id', 'asset_exif.assetId')
343+
.innerJoin('asset_video', 'asset_video.assetId', 'asset.id')
344+
.leftJoin('asset_audio', 'asset_audio.assetId', 'asset.id')
336345
.select(['asset.id', 'asset.ownerId', 'asset.originalPath'])
337346
.select(withFiles)
347+
.select((eb) => withAudioStream(eb).as('audioStream'))
348+
.select((eb) => withVideoStream(eb).$notNull().as('videoStream'))
349+
.select((eb) => withVideoFormat(eb).$notNull().as('format'))
338350
.where('asset.id', '=', id)
339351
.where('asset.type', '=', sql.lit(AssetType.Video))
340352
.executeTakeFirst();

0 commit comments

Comments
 (0)