diff --git a/app/upload/run.go b/app/upload/run.go index d7996187..fff1e7ff 100644 --- a/app/upload/run.go +++ b/app/upload/run.go @@ -6,6 +6,7 @@ import ( "fmt" "strings" "sync" + "time" "github.com/gdamore/tcell/v2" "github.com/simulot/immich-go/adapters" @@ -54,12 +55,21 @@ func (uc *UpCmd) saveTags(ctx context.Context, tag assets.Tag, ids []string) (as uc.app.Log().Info("created tag", "tag", tag.Value) tag.ID = r[0].ID } - _, err := uc.client.Immich.TagAssets(ctx, tag.ID, ids) - if err != nil { - uc.app.Log().Error("failed to add assets to tag", "err", err, "tag", tag.Value, "assets", len(ids)) - return tag, err + const batchSize = 500 + total := len(ids) + var err error + for i := 0; i < total; i += batchSize { + end := i + batchSize + if end > total { + end = total + } + _, err = uc.client.Immich.TagAssets(ctx, tag.ID, ids[i:end]) + if err != nil { + uc.app.Log().Error("failed to add assets to tag", "err", err, "tag", tag.Value, "assets", len(ids[i:end])) + return tag, err + } } - uc.app.Log().Info("updated tag", "tag", tag.Value, "assets", len(ids)) + uc.app.Log().Info("updated tag", "tag", tag.Value, "assets", total) return tag, err } @@ -97,9 +107,26 @@ func (uc *UpCmd) finishing(ctx context.Context) error { return nil } defer func() { uc.finished = true }() - // do waiting operations - uc.albumsCache.Close() - uc.tagsCache.Close() + + if uc.DeferTags { + if uc.client.PauseImmichBackgroundJobs { + uc.app.Log().Info("Resuming metadata extraction...") + // Use a background context to ensure the command is sent even if the main context is cancelling + bgCtx := context.Background() + _, err := uc.client.AdminImmich.SendJobCommand(bgCtx, "metadataExtraction", "resume", true) + if err != nil { + uc.app.Log().Error("Failed to resume metadata extraction", "err", err) + } + } + + uc.app.Log().Info("Waiting for metadata extraction to complete...") + err := uc.waitForMetadataExtraction(ctx) + if err != nil { + uc.app.Log().Error("Failed to wait for metadata extraction", "err", err) + } + uc.app.Log().Info("Metadata extraction complete, applying tags...") + uc.tagsCache.Close() + } // Resume immich background jobs if requested err := uc.resumeJobs(ctx) @@ -107,6 +134,12 @@ func (uc *UpCmd) finishing(ctx context.Context) error { return err } + // do waiting operations + uc.albumsCache.Close() + if !uc.DeferTags { + uc.tagsCache.Close() + } + // Generate FileProcessor report if uc.app.FileProcessor() != nil { report := uc.app.FileProcessor().GenerateReport() @@ -141,7 +174,11 @@ func (uc *UpCmd) upload(ctx context.Context, adapter adapters.Reader) error { uc.albumsCache = cache.NewCollectionCache(50, func(album assets.Album, ids []string) (assets.Album, error) { return uc.saveAlbum(ctx, album, ids) }) - uc.tagsCache = cache.NewCollectionCache(50, func(tag assets.Tag, ids []string) (assets.Tag, error) { + tagCacheSize := 50 + if uc.DeferTags { + tagCacheSize = 1 << 30 + } + uc.tagsCache = cache.NewCollectionCache(tagCacheSize, func(tag assets.Tag, ids []string) (assets.Tag, error) { return uc.saveTags(ctx, tag, ids) }) @@ -596,3 +633,28 @@ func (upCmd *UpCmd) DeleteLocalAssets() error { return nil } */ + +func (uc *UpCmd) waitForMetadataExtraction(ctx context.Context) error { + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + jobs, err := uc.client.Immich.GetJobs(ctx) + if err != nil { + return err + } + job, ok := jobs["metadataExtraction"] + if !ok { + // Job not found, assume it's done or not running + return nil + } + if job.JobCounts.Active == 0 && job.JobCounts.Waiting == 0 { + return nil + } + } + } +} diff --git a/app/upload/upload.go b/app/upload/upload.go index aed3cec7..2cb43bb5 100644 --- a/app/upload/upload.go +++ b/app/upload/upload.go @@ -77,6 +77,7 @@ type UpCmd struct { tagsCache *cache.CollectionCache[assets.Tag] // List of tags present on the server finished bool // the finish task has been run infoCollector *filenames.InfoCollector // Collects information about the files being processed + DeferTags bool // Defer tagging until metadata extraction is complete } func (uc *UpCmd) RegisterFlags(flags *pflag.FlagSet) { @@ -85,6 +86,7 @@ func (uc *UpCmd) RegisterFlags(flags *pflag.FlagSet) { flags.BoolVar(&uc.Overwrite, "overwrite", false, "Always overwrite files on the server with local versions") flags.StringSliceVar(&uc.Tags, "tag", nil, "Add tags to the imported assets. Can be specified multiple times. Hierarchy is supported using a / separator (e.g. 'tag1/subtag1')") flags.BoolVar(&uc.SessionTag, "session-tag", false, "Tag uploaded photos with a tag \"{immich-go}/YYYY-MM-DD HH-MM-SS\"") + flags.BoolVar(&uc.DeferTags, "defer-tags", false, "Defer tagging until metadata extraction is complete") uc.StackOptions.RegisterFlags(flags) } diff --git a/docs/commands/upload.md b/docs/commands/upload.md index 62d26c33..5a98e114 100644 --- a/docs/commands/upload.md +++ b/docs/commands/upload.md @@ -41,11 +41,12 @@ All upload sub-commands require these connection parameters: ## Tagging and Organization -| Option | Default | Description | -| --------------- | ------------ | -------------------------------------------- | -| `--session-tag` | `false` | Tag with upload session timestamp | -| `--tag` | - | Add custom tags (can be used multiple times) | -| `--device-uuid` | `$LOCALHOST` | Set device identifier | +| Option | Default | Description | +| --------------- | ------------ | ------------------------------------------------------------ | +| `--session-tag` | `false` | Tag with upload session timestamp | +| `--defer-tags` | `false` | Apply tags only after metadata extraction has completed to keep embedded file tags/keywords intact | +| `--tag` | - | Add custom tags (can be used multiple times) | +| `--device-uuid` | `$LOCALHOST` | Set device identifier | ## User Interface @@ -270,4 +271,4 @@ immich-go upload from-immich \ - [Configuration Options](../configuration.md) - [Technical Details](../technical.md) -- [Best Practices](../best-practices.md) \ No newline at end of file +- [Best Practices](../best-practices.md) diff --git a/docs/configuration.md b/docs/configuration.md index 16f59be6..cfd1dcc9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -150,6 +150,7 @@ overwrite = false pause-immich-jobs = true server = 'https://immich.app' session-tag = false +defer-tags = false skip-verify-ssl = false time-zone = '' @@ -473,6 +474,7 @@ upload: pause-immich-jobs: true server: https://immich.app session-tag: false + defer-tags: false skip-verify-ssl: false tag: {} time-zone: "" @@ -710,6 +712,7 @@ upload: "pause-immich-jobs": true, "server": "https://immich.app", "session-tag": false, + "defer-tags": false, "skip-verify-ssl": false, "tag": {}, "time-zone": "" diff --git a/docs/environment.md b/docs/environment.md index 4b4a6fa4..737e331f 100644 --- a/docs/environment.md +++ b/docs/environment.md @@ -165,6 +165,7 @@ The following environment variables can be used to configure `immich-go`. | `IMMICH_GO_UPLOAD_PAUSE_IMMICH_JOBS` | `--pause-immich-jobs` | `true` | Pause Immich background jobs during upload operations | | `IMMICH_GO_UPLOAD_SERVER` | `--server` | | Immich server address (example http://your-ip:2283 or https://your-domain) | | `IMMICH_GO_UPLOAD_SESSION_TAG` | `--session-tag` | `false` | Tag uploaded photos with a tag "{immich-go}/YYYY-MM-DD HH-MM-SS" | +| `IMMICH_GO_UPLOAD_DEFER_TAGS` | `--defer-tags` | `false` | Apply tags only after metadata extraction has finished | | `IMMICH_GO_UPLOAD_SKIP_VERIFY_SSL` | `--skip-verify-ssl` | `false` | Skip SSL verification | | `IMMICH_GO_UPLOAD_TAG` | `--tag` | `[]` | Add tags to the imported assets. Can be specified multiple times. Hierarchy is supported using a / separator (e.g. 'tag1/subtag1') | | `IMMICH_GO_UPLOAD_TIME_ZONE` | `--time-zone` | | Override the system time zone | diff --git a/docs/upload-commands-overview.md b/docs/upload-commands-overview.md index 86212293..59cf6c6e 100644 --- a/docs/upload-commands-overview.md +++ b/docs/upload-commands-overview.md @@ -52,6 +52,7 @@ Tags can be assigned in two main ways: * **`--folder-as-tags`**: Uses the folder structure to create tags. For example, a file at `Holidays/Summer 2024/photo.jpg` will be tagged `Holidays/Summer 2024`. * **XMP Metadata**: Tags are also read from XMP sidecar files. +* **`--defer-tags`**: Postpones tag creation until Immich finishes metadata extraction so built-in file tags/keywords aren’t dropped. `immich-go` creates new tags on the server as needed and efficiently tags assets in batches.