diff --git a/e2e/src/api/specs/library.e2e-spec.ts b/e2e/src/api/specs/library.e2e-spec.ts index 4994ed070dfd8..dde2cf79eb928 100644 --- a/e2e/src/api/specs/library.e2e-spec.ts +++ b/e2e/src/api/specs/library.e2e-spec.ts @@ -1,5 +1,5 @@ import { LibraryResponseDto, LoginResponseDto, getAllLibraries, scanLibrary } from '@immich/sdk'; -import { cpSync, existsSync, unlinkSync } from 'node:fs'; +import { cpSync, existsSync, rmSync, unlinkSync } from 'node:fs'; import { Socket } from 'socket.io-client'; import { userDto, uuidDto } from 'src/fixtures'; import { errorDto } from 'src/responses'; @@ -406,65 +406,93 @@ describe('/libraries', () => { it('should reimport a modified file', async () => { const library = await utils.createLibrary(admin.accessToken, { ownerId: admin.userId, - importPaths: [`${testAssetDirInternal}/temp`], + importPaths: [`${testAssetDirInternal}/temp/reimport`], }); - utils.createImageFile(`${testAssetDir}/temp/directoryA/assetB.jpg`); - await utimes(`${testAssetDir}/temp/directoryA/assetB.jpg`, 447_775_200_000); + utils.createImageFile(`${testAssetDir}/temp/reimport/asset.jpg`); + await utimes(`${testAssetDir}/temp/reimport/asset.jpg`, 447_775_200_000); await scan(admin.accessToken, library.id); await utils.waitForQueueFinish(admin.accessToken, 'library'); + await utils.waitForQueueFinish(admin.accessToken, 'sidecar'); + await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction'); - cpSync(`${testAssetDir}/albums/nature/tanners_ridge.jpg`, `${testAssetDir}/temp/directoryA/assetB.jpg`); - await utimes(`${testAssetDir}/temp/directoryA/assetB.jpg`, 447_775_200_001); + cpSync(`${testAssetDir}/albums/nature/tanners_ridge.jpg`, `${testAssetDir}/temp/reimport/asset.jpg`); + await utimes(`${testAssetDir}/temp/reimport/asset.jpg`, 447_775_200_001); const { status } = await request(app) .post(`/libraries/${library.id}/scan`) .set('Authorization', `Bearer ${admin.accessToken}`) - .send({ refreshModifiedFiles: true }); + .send(); expect(status).toBe(204); await utils.waitForQueueFinish(admin.accessToken, 'library'); + await utils.waitForQueueFinish(admin.accessToken, 'sidecar'); await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction'); - utils.removeImageFile(`${testAssetDir}/temp/directoryA/assetB.jpg`); const { assets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id, - model: 'NIKON D750', }); - expect(assets.count).toBe(1); + + expect(assets.count).toEqual(1); + + const asset = await utils.getAssetInfo(admin.accessToken, assets.items[0].id); + + expect(asset).toEqual( + expect.objectContaining({ + originalFileName: 'asset.jpg', + exifInfo: expect.objectContaining({ + model: 'NIKON D750', + }), + }), + ); + + utils.removeImageFile(`${testAssetDir}/temp/reimport/asset.jpg`); }); it('should not reimport unmodified files', async () => { const library = await utils.createLibrary(admin.accessToken, { ownerId: admin.userId, - importPaths: [`${testAssetDirInternal}/temp`], + importPaths: [`${testAssetDirInternal}/temp/reimport`], }); - utils.createImageFile(`${testAssetDir}/temp/directoryA/assetB.jpg`); - await utimes(`${testAssetDir}/temp/directoryA/assetB.jpg`, 447_775_200_000); + utils.createImageFile(`${testAssetDir}/temp/reimport/asset.jpg`); + await utimes(`${testAssetDir}/temp/reimport/asset.jpg`, 447_775_200_000); await scan(admin.accessToken, library.id); await utils.waitForQueueFinish(admin.accessToken, 'library'); - cpSync(`${testAssetDir}/albums/nature/tanners_ridge.jpg`, `${testAssetDir}/temp/directoryA/assetB.jpg`); - await utimes(`${testAssetDir}/temp/directoryA/assetB.jpg`, 447_775_200_000); + cpSync(`${testAssetDir}/albums/nature/tanners_ridge.jpg`, `${testAssetDir}/temp/reimport/asset.jpg`); + await utimes(`${testAssetDir}/temp/reimport/asset.jpg`, 447_775_200_000); const { status } = await request(app) .post(`/libraries/${library.id}/scan`) .set('Authorization', `Bearer ${admin.accessToken}`) - .send({ refreshModifiedFiles: true }); + .send(); expect(status).toBe(204); await utils.waitForQueueFinish(admin.accessToken, 'library'); + await utils.waitForQueueFinish(admin.accessToken, 'sidecar'); await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction'); - utils.removeImageFile(`${testAssetDir}/temp/directoryA/assetB.jpg`); const { assets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id, - model: 'NIKON D750', }); - expect(assets.count).toBe(0); + + expect(assets.count).toEqual(1); + + const asset = await utils.getAssetInfo(admin.accessToken, assets.items[0].id); + + expect(asset).toEqual( + expect.objectContaining({ + originalFileName: 'asset.jpg', + exifInfo: expect.not.objectContaining({ + model: 'NIKON D750', + }), + }), + ); + + utils.removeImageFile(`${testAssetDir}/temp/reimport/asset.jpg`); }); it('should set an asset offline if its file is missing', async () => { @@ -615,6 +643,7 @@ describe('/libraries', () => { await scan(admin.accessToken, library.id); await utils.waitForQueueFinish(admin.accessToken, 'library'); + await utils.waitForQueueFinish(admin.accessToken, 'sidecar'); await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction'); const { assets: newAssets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id }); @@ -622,12 +651,11 @@ describe('/libraries', () => { expect(newAssets.items).toEqual([ expect.objectContaining({ originalFileName: 'glarus.nef', - fileCreatedAt: '2000-09-27T12:35:33.000Z', // This time comes from the xmp file, not from the image + fileCreatedAt: '2000-09-27T12:35:33.000Z', }), ]); - unlinkSync(`${testAssetDir}/temp/xmp/glarus.xmp`); - unlinkSync(`${testAssetDir}/temp/xmp/glarus.nef`); + rmSync(`${testAssetDir}/temp/xmp`, { recursive: true, force: true }); }); it('should import metadata from file.ext.xmp', async () => { @@ -641,6 +669,7 @@ describe('/libraries', () => { await scan(admin.accessToken, library.id); await utils.waitForQueueFinish(admin.accessToken, 'library'); + await utils.waitForQueueFinish(admin.accessToken, 'sidecar'); await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction'); const { assets: newAssets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id }); @@ -648,12 +677,11 @@ describe('/libraries', () => { expect(newAssets.items).toEqual([ expect.objectContaining({ originalFileName: 'glarus.nef', - fileCreatedAt: '2000-09-27T12:35:33.000Z', // This time comes from the xmp file, not from the image + fileCreatedAt: '2000-09-27T12:35:33.000Z', }), ]); - unlinkSync(`${testAssetDir}/temp/xmp/glarus.nef.xmp`); - unlinkSync(`${testAssetDir}/temp/xmp/glarus.nef`); + rmSync(`${testAssetDir}/temp/xmp`, { recursive: true, force: true }); }); it('should import metadata in file.ext.xmp before file.xmp if both exist', async () => { @@ -668,6 +696,111 @@ describe('/libraries', () => { await scan(admin.accessToken, library.id); await utils.waitForQueueFinish(admin.accessToken, 'library'); + await utils.waitForQueueFinish(admin.accessToken, 'sidecar'); + await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction'); + + const { assets: newAssets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id }); + + expect(newAssets.items).toEqual([ + expect.objectContaining({ + originalFileName: 'glarus.nef', + fileCreatedAt: '2000-09-27T12:35:33.000Z', + }), + ]); + + rmSync(`${testAssetDir}/temp/xmp`, { recursive: true, force: true }); + }); + + it('should switch from using file.xmp to file.ext.xmp when asset refreshes', async () => { + const library = await utils.createLibrary(admin.accessToken, { + ownerId: admin.userId, + importPaths: [`${testAssetDirInternal}/temp/xmp`], + }); + + cpSync(`${testAssetDir}/metadata/xmp/dates/2000.xmp`, `${testAssetDir}/temp/xmp/glarus.xmp`); + cpSync(`${testAssetDir}/formats/raw/Nikon/D80/glarus.nef`, `${testAssetDir}/temp/xmp/glarus.nef`); + await utimes(`${testAssetDir}/temp/xmp/glarus.nef`, 447_775_200_000); + + await scan(admin.accessToken, library.id); + await utils.waitForQueueFinish(admin.accessToken, 'library'); + await utils.waitForQueueFinish(admin.accessToken, 'sidecar'); + await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction'); + + cpSync(`${testAssetDir}/metadata/xmp/dates/2010.xmp`, `${testAssetDir}/temp/xmp/glarus.nef.xmp`); + unlinkSync(`${testAssetDir}/temp/xmp/glarus.xmp`); + await utimes(`${testAssetDir}/temp/xmp/glarus.nef`, 447_775_200_001); + + await scan(admin.accessToken, library.id); + await utils.waitForQueueFinish(admin.accessToken, 'library'); + await utils.waitForQueueFinish(admin.accessToken, 'sidecar'); + await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction'); + + const { assets: newAssets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id }); + + expect(newAssets.items).toEqual([ + expect.objectContaining({ + originalFileName: 'glarus.nef', + fileCreatedAt: '2010-09-27T12:35:33.000Z', + }), + ]); + + rmSync(`${testAssetDir}/temp/xmp`, { recursive: true, force: true }); + }); + + it('should switch from using file metadata to file.xmp metadata when asset refreshes', async () => { + const library = await utils.createLibrary(admin.accessToken, { + ownerId: admin.userId, + importPaths: [`${testAssetDirInternal}/temp/xmp`], + }); + + cpSync(`${testAssetDir}/formats/raw/Nikon/D80/glarus.nef`, `${testAssetDir}/temp/xmp/glarus.nef`); + await utimes(`${testAssetDir}/temp/xmp/glarus.nef`, 447_775_200_000); + + await scan(admin.accessToken, library.id); + await utils.waitForQueueFinish(admin.accessToken, 'library'); + await utils.waitForQueueFinish(admin.accessToken, 'sidecar'); + await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction'); + + cpSync(`${testAssetDir}/metadata/xmp/dates/2000.xmp`, `${testAssetDir}/temp/xmp/glarus.xmp`); + await utimes(`${testAssetDir}/temp/xmp/glarus.nef`, 447_775_200_001); + + await scan(admin.accessToken, library.id); + await utils.waitForQueueFinish(admin.accessToken, 'library'); + await utils.waitForQueueFinish(admin.accessToken, 'sidecar'); + await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction'); + + const { assets: newAssets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id }); + + expect(newAssets.items).toEqual([ + expect.objectContaining({ + originalFileName: 'glarus.nef', + fileCreatedAt: '2000-09-27T12:35:33.000Z', + }), + ]); + + rmSync(`${testAssetDir}/temp/xmp`, { recursive: true, force: true }); + }); + + it('should switch from using file metadata to file.xmp metadata when asset refreshes', async () => { + const library = await utils.createLibrary(admin.accessToken, { + ownerId: admin.userId, + importPaths: [`${testAssetDirInternal}/temp/xmp`], + }); + + cpSync(`${testAssetDir}/formats/raw/Nikon/D80/glarus.nef`, `${testAssetDir}/temp/xmp/glarus.nef`); + await utimes(`${testAssetDir}/temp/xmp/glarus.nef`, 447_775_200_000); + + await scan(admin.accessToken, library.id); + await utils.waitForQueueFinish(admin.accessToken, 'library'); + await utils.waitForQueueFinish(admin.accessToken, 'sidecar'); + await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction'); + + cpSync(`${testAssetDir}/metadata/xmp/dates/2000.xmp`, `${testAssetDir}/temp/xmp/glarus.nef.xmp`); + await utimes(`${testAssetDir}/temp/xmp/glarus.nef`, 447_775_200_001); + + await scan(admin.accessToken, library.id); + await utils.waitForQueueFinish(admin.accessToken, 'library'); + await utils.waitForQueueFinish(admin.accessToken, 'sidecar'); await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction'); const { assets: newAssets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id }); @@ -675,12 +808,117 @@ describe('/libraries', () => { expect(newAssets.items).toEqual([ expect.objectContaining({ originalFileName: 'glarus.nef', - fileCreatedAt: '2000-09-27T12:35:33.000Z', // This time comes from the xmp file, not from the image + fileCreatedAt: '2000-09-27T12:35:33.000Z', }), ]); + rmSync(`${testAssetDir}/temp/xmp`, { recursive: true, force: true }); + }); + + it('should switch from using file.ext.xmp to file.xmp when asset refreshes', async () => { + const library = await utils.createLibrary(admin.accessToken, { + ownerId: admin.userId, + importPaths: [`${testAssetDirInternal}/temp/xmp`], + }); + + cpSync(`${testAssetDir}/metadata/xmp/dates/2000.xmp`, `${testAssetDir}/temp/xmp/glarus.nef.xmp`); + cpSync(`${testAssetDir}/formats/raw/Nikon/D80/glarus.nef`, `${testAssetDir}/temp/xmp/glarus.nef`); + await utimes(`${testAssetDir}/temp/xmp/glarus.nef`, 447_775_200_000); + + await scan(admin.accessToken, library.id); + await utils.waitForQueueFinish(admin.accessToken, 'library'); + await utils.waitForQueueFinish(admin.accessToken, 'sidecar'); + await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction'); + + cpSync(`${testAssetDir}/metadata/xmp/dates/2010.xmp`, `${testAssetDir}/temp/xmp/glarus.xmp`); unlinkSync(`${testAssetDir}/temp/xmp/glarus.nef.xmp`); - unlinkSync(`${testAssetDir}/temp/xmp/glarus.nef`); + await utimes(`${testAssetDir}/temp/xmp/glarus.nef`, 447_775_200_001); + + await scan(admin.accessToken, library.id); + await utils.waitForQueueFinish(admin.accessToken, 'library'); + await utils.waitForQueueFinish(admin.accessToken, 'sidecar'); + await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction'); + + const { assets: newAssets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id }); + + expect(newAssets.items).toEqual([ + expect.objectContaining({ + originalFileName: 'glarus.nef', + fileCreatedAt: '2010-09-27T12:35:33.000Z', + }), + ]); + + rmSync(`${testAssetDir}/temp/xmp`, { recursive: true, force: true }); + }); + + it('should switch from using file.ext.xmp to file metadata', async () => { + const library = await utils.createLibrary(admin.accessToken, { + ownerId: admin.userId, + importPaths: [`${testAssetDirInternal}/temp/xmp`], + }); + + cpSync(`${testAssetDir}/metadata/xmp/dates/2000.xmp`, `${testAssetDir}/temp/xmp/glarus.nef.xmp`); + cpSync(`${testAssetDir}/formats/raw/Nikon/D80/glarus.nef`, `${testAssetDir}/temp/xmp/glarus.nef`); + await utimes(`${testAssetDir}/temp/xmp/glarus.nef`, 447_775_200_000); + + await scan(admin.accessToken, library.id); + await utils.waitForQueueFinish(admin.accessToken, 'library'); + await utils.waitForQueueFinish(admin.accessToken, 'sidecar'); + await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction'); + + unlinkSync(`${testAssetDir}/temp/xmp/glarus.nef.xmp`); + await utimes(`${testAssetDir}/temp/xmp/glarus.nef`, 447_775_200_001); + + await scan(admin.accessToken, library.id); + await utils.waitForQueueFinish(admin.accessToken, 'library'); + await utils.waitForQueueFinish(admin.accessToken, 'sidecar'); + await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction'); + + const { assets: newAssets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id }); + + expect(newAssets.items).toEqual([ + expect.objectContaining({ + originalFileName: 'glarus.nef', + fileCreatedAt: '2010-07-20T17:27:12.000Z', + }), + ]); + + rmSync(`${testAssetDir}/temp/xmp`, { recursive: true, force: true }); + }); + + it('should switch from using file.xmp to file metadata', async () => { + const library = await utils.createLibrary(admin.accessToken, { + ownerId: admin.userId, + importPaths: [`${testAssetDirInternal}/temp/xmp`], + }); + + cpSync(`${testAssetDir}/metadata/xmp/dates/2000.xmp`, `${testAssetDir}/temp/xmp/glarus.xmp`); + cpSync(`${testAssetDir}/formats/raw/Nikon/D80/glarus.nef`, `${testAssetDir}/temp/xmp/glarus.nef`); + await utimes(`${testAssetDir}/temp/xmp/glarus.nef`, 447_775_200_000); + + await scan(admin.accessToken, library.id); + await utils.waitForQueueFinish(admin.accessToken, 'library'); + await utils.waitForQueueFinish(admin.accessToken, 'sidecar'); + await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction'); + + unlinkSync(`${testAssetDir}/temp/xmp/glarus.xmp`); + await utimes(`${testAssetDir}/temp/xmp/glarus.nef`, 447_775_200_001); + + await scan(admin.accessToken, library.id); + await utils.waitForQueueFinish(admin.accessToken, 'library'); + await utils.waitForQueueFinish(admin.accessToken, 'sidecar'); + await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction'); + + const { assets: newAssets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id }); + + expect(newAssets.items).toEqual([ + expect.objectContaining({ + originalFileName: 'glarus.nef', + fileCreatedAt: '2010-07-20T17:27:12.000Z', + }), + ]); + + rmSync(`${testAssetDir}/temp/xmp`, { recursive: true, force: true }); }); }); }); diff --git a/e2e/test-assets b/e2e/test-assets index 99544a200412d..9e3b964b080dc 160000 --- a/e2e/test-assets +++ b/e2e/test-assets @@ -1 +1 @@ -Subproject commit 99544a200412d553103cc7b8f1a28f339c7cffd9 +Subproject commit 9e3b964b080dca6f035b29b86e66454ae8aeda78 diff --git a/server/src/interfaces/job.interface.ts b/server/src/interfaces/job.interface.ts index 270f282b829e1..7976f813022ff 100644 --- a/server/src/interfaces/job.interface.ts +++ b/server/src/interfaces/job.interface.ts @@ -135,7 +135,7 @@ export interface IDelayedJob extends IBaseJob { export interface IEntityJob extends IBaseJob { id: string; - source?: 'upload' | 'library-import' | 'sidecar-write' | 'copy'; + source?: 'upload' | 'sidecar-write' | 'copy'; notify?: boolean; } diff --git a/server/src/services/job.service.ts b/server/src/services/job.service.ts index 021179338287f..2faed0a51666a 100644 --- a/server/src/services/job.service.ts +++ b/server/src/services/job.service.ts @@ -250,7 +250,7 @@ export class JobService extends BaseService { } case JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE: { - if (item.data.source === 'upload' || item.data.source === 'copy' || item.data.source === 'library-import') { + if (item.data.source === 'upload' || item.data.source === 'copy') { await this.jobRepository.queue({ name: JobName.GENERATE_THUMBNAILS, data: item.data }); } break; @@ -266,7 +266,7 @@ export class JobService extends BaseService { } case JobName.GENERATE_THUMBNAILS: { - if (!item.data.notify && item.data.source !== 'upload' && item.data.source === 'library-import') { + if (!item.data.notify && item.data.source !== 'upload') { break; } diff --git a/server/src/services/library.service.spec.ts b/server/src/services/library.service.spec.ts index 8e21378ce67b1..9b944045ab738 100644 --- a/server/src/services/library.service.spec.ts +++ b/server/src/services/library.service.spec.ts @@ -422,10 +422,9 @@ describe(LibraryService.name, () => { expect(jobMock.queue.mock.calls).toEqual([ [ { - name: JobName.METADATA_EXTRACTION, + name: JobName.SIDECAR_DISCOVERY, data: { id: assetStub.image.id, - source: 'upload', }, }, ], @@ -467,10 +466,9 @@ describe(LibraryService.name, () => { expect(jobMock.queue.mock.calls).toEqual([ [ { - name: JobName.METADATA_EXTRACTION, + name: JobName.SIDECAR_DISCOVERY, data: { id: assetStub.image.id, - source: 'upload', }, }, ], diff --git a/server/src/services/library.service.ts b/server/src/services/library.service.ts index 435c11508def1..0deddc894195b 100644 --- a/server/src/services/library.service.ts +++ b/server/src/services/library.service.ts @@ -423,9 +423,10 @@ export class LibraryService extends BaseService { async queuePostSyncJobs(asset: AssetEntity) { this.logger.debug(`Queueing metadata extraction for: ${asset.originalPath}`); + // We queue a sidecar discovery which, in turn, queues metadata extraction await this.jobRepository.queue({ - name: JobName.METADATA_EXTRACTION, - data: { id: asset.id, source: 'library-import' }, + name: JobName.SIDECAR_DISCOVERY, + data: { id: asset.id }, }); } diff --git a/server/src/services/metadata.service.ts b/server/src/services/metadata.service.ts index b9e6af67d1837..e0566c84b72d2 100644 --- a/server/src/services/metadata.service.ts +++ b/server/src/services/metadata.service.ts @@ -148,23 +148,13 @@ export class MetadataService extends BaseService { } @OnJob({ name: JobName.METADATA_EXTRACTION, queue: QueueName.METADATA_EXTRACTION }) - async handleMetadataExtraction({ id, source }: JobOf): Promise { - this.logger.verbose(`Extracting metadata for asset ${id}`); - + async handleMetadataExtraction({ id }: JobOf): Promise { const { metadata, reverseGeocoding } = await this.getConfig({ withCache: true }); - - if (source === 'library-import') { - await this.processSidecar(id, false); - } - const [asset] = await this.assetRepository.getByIds([id], { faces: { person: false } }); - if (!asset) { return JobStatus.FAILED; } - this.logger.verbose(`Sidecar path: ${asset.sidecarPath}`); - const stats = await this.storageRepository.stat(asset.originalPath); const exifTags = await this.getExifTags(asset); @@ -708,7 +698,7 @@ export class MetadataService extends BaseService { return JobStatus.FAILED; } - if (!isSync && (!asset.isVisible || asset.sidecarPath)) { + if (!isSync && (!asset.isVisible || asset.sidecarPath) && !asset.isExternal) { return JobStatus.FAILED; } @@ -730,9 +720,15 @@ export class MetadataService extends BaseService { sidecarPath = sidecarPathWithoutExt; } + if (asset.isExternal) { + if (sidecarPath !== asset.sidecarPath) { + await this.assetRepository.update({ id: asset.id, sidecarPath }); + } + return JobStatus.SUCCESS; + } + if (sidecarPath) { await this.assetRepository.update({ id: asset.id, sidecarPath }); - this.logger.verbose(`Sidecar discovered at ${sidecarPath} for asset ${asset.id} at `); return JobStatus.SUCCESS; }