From afbb3b1b509f3d6fd6479053be9c2447dbbdad63 Mon Sep 17 00:00:00 2001 From: Rob Elkin Date: Thu, 26 Mar 2026 15:07:02 +0000 Subject: [PATCH 1/5] Log message IDs when fetch returns nil during sync When GetMessagesRawBatch returns nil for a message (typically a 404 because the message was deleted between listing and fetching), the error count was incremented but no message ID was logged. This made it impossible to diagnose which messages failed to sync. Add Warn-level logging with the message ID in both incremental and full sync paths so failed fetches are visible in logs. Co-Authored-By: Claude Opus 4.6 (1M context) --- internal/sync/incremental.go | 1 + internal/sync/sync.go | 1 + 2 files changed, 2 insertions(+) diff --git a/internal/sync/incremental.go b/internal/sync/incremental.go index 2517f2d9..d85577bf 100644 --- a/internal/sync/incremental.go +++ b/internal/sync/incremental.go @@ -159,6 +159,7 @@ func (s *Syncer) Incremental(ctx context.Context, source *store.Source) (summary } else { for i, raw := range rawMessages { if raw == nil { + s.logger.Warn("failed to fetch message (nil response)", "id", newMsgIDs[i]) checkpoint.ErrorsCount++ continue } diff --git a/internal/sync/sync.go b/internal/sync/sync.go index 15cf307a..d46f7c51 100644 --- a/internal/sync/sync.go +++ b/internal/sync/sync.go @@ -191,6 +191,7 @@ func (s *Syncer) processBatch(ctx context.Context, sourceID int64, listResp *gma for i, raw := range rawMessages { if raw == nil { + s.logger.Warn("failed to fetch message (nil response)", "id", newIDs[i]) checkpoint.ErrorsCount++ continue } From db5a4adf43fb14303c7b50918b2a2ca4c258b5db Mon Sep 17 00:00:00 2001 From: Rob Elkin Date: Thu, 26 Mar 2026 16:26:43 +0000 Subject: [PATCH 2/5] Add --rebuild-cache flag to sync and sync-full commands The serve command rebuilds the Parquet analytics cache after each sync, but the standalone CLI commands did not. This caused stale search results when syncs were triggered via launchd or manual CLI. Rather than always rebuilding (which couples sync and caching), add an opt-in --rebuild-cache flag that users can enable in their launchd config or scripts. Co-Authored-By: Claude Opus 4.6 (1M context) --- cmd/msgvault/cmd/sync.go | 16 ++++++++++++++++ cmd/msgvault/cmd/syncfull.go | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/cmd/msgvault/cmd/sync.go b/cmd/msgvault/cmd/sync.go index cc65cfde..394e3c7d 100644 --- a/cmd/msgvault/cmd/sync.go +++ b/cmd/msgvault/cmd/sync.go @@ -174,6 +174,19 @@ Examples: } } + // Rebuild analytics cache if requested and stale. + if syncRebuildCache { + analyticsDir := cfg.AnalyticsDir() + if staleness := cacheNeedsBuild(dbPath, analyticsDir); staleness.NeedsBuild { + result, cacheErr := buildCache(dbPath, analyticsDir, staleness.FullRebuild) + if cacheErr != nil { + logger.Warn("cache build failed", "error", cacheErr) + } else if !result.Skipped { + logger.Info("cache build completed", "exported", result.ExportedCount) + } + } + } + if len(syncErrors) > 0 { fmt.Println() fmt.Println("Errors:") @@ -264,6 +277,9 @@ func runIncrementalSync(ctx context.Context, s *store.Store, getOAuthMgr func(st return nil } +var syncRebuildCache bool + func init() { + syncIncrementalCmd.Flags().BoolVar(&syncRebuildCache, "rebuild-cache", false, "Rebuild analytics cache after sync") rootCmd.AddCommand(syncIncrementalCmd) } diff --git a/cmd/msgvault/cmd/syncfull.go b/cmd/msgvault/cmd/syncfull.go index 15847e62..edbddcb2 100644 --- a/cmd/msgvault/cmd/syncfull.go +++ b/cmd/msgvault/cmd/syncfull.go @@ -166,6 +166,19 @@ Examples: } } + // Rebuild analytics cache if requested and stale. + if syncFullRebuildCache { + analyticsDir := cfg.AnalyticsDir() + if staleness := cacheNeedsBuild(dbPath, analyticsDir); staleness.NeedsBuild { + result, cacheErr := buildCache(dbPath, analyticsDir, staleness.FullRebuild) + if cacheErr != nil { + logger.Warn("cache build failed", "error", cacheErr) + } else if !result.Skipped { + logger.Info("cache build completed", "exported", result.ExportedCount) + } + } + } + if len(syncErrors) > 0 { fmt.Println() fmt.Println("Errors:") @@ -422,11 +435,14 @@ func formatDuration(d time.Duration) string { return fmt.Sprintf("%ds", s) } +var syncFullRebuildCache bool + func init() { syncFullCmd.Flags().StringVar(&syncQuery, "query", "", "Gmail search query") syncFullCmd.Flags().BoolVar(&syncNoResume, "noresume", false, "Force fresh sync (don't resume)") syncFullCmd.Flags().StringVar(&syncBefore, "before", "", "Only messages before this date (YYYY-MM-DD)") syncFullCmd.Flags().StringVar(&syncAfter, "after", "", "Only messages after this date (YYYY-MM-DD)") syncFullCmd.Flags().IntVar(&syncLimit, "limit", 0, "Limit number of messages (for testing)") + syncFullCmd.Flags().BoolVar(&syncFullRebuildCache, "rebuild-cache", false, "Rebuild analytics cache after sync") rootCmd.AddCommand(syncFullCmd) } From fb7de1050f6aa44989f1a404b11a215dfd91b9e3 Mon Sep 17 00:00:00 2001 From: Rob Elkin Date: Thu, 26 Mar 2026 17:10:03 +0000 Subject: [PATCH 3/5] Make --rebuild-cache bypass staleness check When the flag is set, always run buildCache rather than gating on cacheNeedsBuild. The staleness check is still consulted to decide whether a full rebuild is needed, but the cache build itself runs unconditionally so users can recover from bad-but-undetected cache states. Co-Authored-By: Claude Opus 4.6 (1M context) --- cmd/msgvault/cmd/sync.go | 18 ++++++++++-------- cmd/msgvault/cmd/syncfull.go | 18 ++++++++++-------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/cmd/msgvault/cmd/sync.go b/cmd/msgvault/cmd/sync.go index 394e3c7d..1d2f2f50 100644 --- a/cmd/msgvault/cmd/sync.go +++ b/cmd/msgvault/cmd/sync.go @@ -174,16 +174,18 @@ Examples: } } - // Rebuild analytics cache if requested and stale. + // Rebuild analytics cache if requested. if syncRebuildCache { analyticsDir := cfg.AnalyticsDir() - if staleness := cacheNeedsBuild(dbPath, analyticsDir); staleness.NeedsBuild { - result, cacheErr := buildCache(dbPath, analyticsDir, staleness.FullRebuild) - if cacheErr != nil { - logger.Warn("cache build failed", "error", cacheErr) - } else if !result.Skipped { - logger.Info("cache build completed", "exported", result.ExportedCount) - } + fullRebuild := false + if staleness := cacheNeedsBuild(dbPath, analyticsDir); staleness.FullRebuild { + fullRebuild = true + } + result, cacheErr := buildCache(dbPath, analyticsDir, fullRebuild) + if cacheErr != nil { + logger.Warn("cache build failed", "error", cacheErr) + } else if !result.Skipped { + logger.Info("cache build completed", "exported", result.ExportedCount) } } diff --git a/cmd/msgvault/cmd/syncfull.go b/cmd/msgvault/cmd/syncfull.go index edbddcb2..a9542d5c 100644 --- a/cmd/msgvault/cmd/syncfull.go +++ b/cmd/msgvault/cmd/syncfull.go @@ -166,16 +166,18 @@ Examples: } } - // Rebuild analytics cache if requested and stale. + // Rebuild analytics cache if requested. if syncFullRebuildCache { analyticsDir := cfg.AnalyticsDir() - if staleness := cacheNeedsBuild(dbPath, analyticsDir); staleness.NeedsBuild { - result, cacheErr := buildCache(dbPath, analyticsDir, staleness.FullRebuild) - if cacheErr != nil { - logger.Warn("cache build failed", "error", cacheErr) - } else if !result.Skipped { - logger.Info("cache build completed", "exported", result.ExportedCount) - } + fullRebuild := false + if staleness := cacheNeedsBuild(dbPath, analyticsDir); staleness.FullRebuild { + fullRebuild = true + } + result, cacheErr := buildCache(dbPath, analyticsDir, fullRebuild) + if cacheErr != nil { + logger.Warn("cache build failed", "error", cacheErr) + } else if !result.Skipped { + logger.Info("cache build completed", "exported", result.ExportedCount) } } From b4d510e117ac5190415659fdc8e8b20fc92362bf Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Thu, 2 Apr 2026 08:03:16 -0500 Subject: [PATCH 4/5] Propagate --rebuild-cache errors as command failures When the user explicitly requests cache rebuild via --rebuild-cache, failures should cause a non-zero exit so automation can detect them. Previously these were silently downgraded to log warnings. Co-Authored-By: Claude Opus 4.6 (1M context) --- cmd/msgvault/cmd/sync.go | 2 +- cmd/msgvault/cmd/syncfull.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/msgvault/cmd/sync.go b/cmd/msgvault/cmd/sync.go index 1d2f2f50..6becf9df 100644 --- a/cmd/msgvault/cmd/sync.go +++ b/cmd/msgvault/cmd/sync.go @@ -183,7 +183,7 @@ Examples: } result, cacheErr := buildCache(dbPath, analyticsDir, fullRebuild) if cacheErr != nil { - logger.Warn("cache build failed", "error", cacheErr) + syncErrors = append(syncErrors, fmt.Sprintf("cache rebuild: %v", cacheErr)) } else if !result.Skipped { logger.Info("cache build completed", "exported", result.ExportedCount) } diff --git a/cmd/msgvault/cmd/syncfull.go b/cmd/msgvault/cmd/syncfull.go index a9542d5c..04b96658 100644 --- a/cmd/msgvault/cmd/syncfull.go +++ b/cmd/msgvault/cmd/syncfull.go @@ -175,7 +175,7 @@ Examples: } result, cacheErr := buildCache(dbPath, analyticsDir, fullRebuild) if cacheErr != nil { - logger.Warn("cache build failed", "error", cacheErr) + syncErrors = append(syncErrors, fmt.Sprintf("cache rebuild: %v", cacheErr)) } else if !result.Skipped { logger.Info("cache build completed", "exported", result.ExportedCount) } From 07f61c21685e35de3be27e76fab929370f15e7d7 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Thu, 2 Apr 2026 10:21:30 -0500 Subject: [PATCH 5/5] Separate cache rebuild errors from sync errors Report cache failures independently rather than mixing them into the "account(s) failed to sync" message. Sync failures and cache failures are now distinct error paths with accurate messages. Co-Authored-By: Claude Opus 4.6 (1M context) --- cmd/msgvault/cmd/sync.go | 11 ++++++++--- cmd/msgvault/cmd/syncfull.go | 11 ++++++++--- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/cmd/msgvault/cmd/sync.go b/cmd/msgvault/cmd/sync.go index 6becf9df..e420811c 100644 --- a/cmd/msgvault/cmd/sync.go +++ b/cmd/msgvault/cmd/sync.go @@ -175,15 +175,17 @@ Examples: } // Rebuild analytics cache if requested. + var cacheErr error if syncRebuildCache { analyticsDir := cfg.AnalyticsDir() fullRebuild := false if staleness := cacheNeedsBuild(dbPath, analyticsDir); staleness.FullRebuild { fullRebuild = true } - result, cacheErr := buildCache(dbPath, analyticsDir, fullRebuild) - if cacheErr != nil { - syncErrors = append(syncErrors, fmt.Sprintf("cache rebuild: %v", cacheErr)) + result, err := buildCache(dbPath, analyticsDir, fullRebuild) + if err != nil { + cacheErr = err + fmt.Printf("\nCache rebuild failed: %v\n", err) } else if !result.Skipped { logger.Info("cache build completed", "exported", result.ExportedCount) } @@ -198,6 +200,9 @@ Examples: return fmt.Errorf("%d account(s) failed to sync: %s", len(syncErrors), strings.Join(syncErrors, "; ")) } + if cacheErr != nil { + return fmt.Errorf("cache rebuild failed: %w", cacheErr) + } return nil }, diff --git a/cmd/msgvault/cmd/syncfull.go b/cmd/msgvault/cmd/syncfull.go index 04b96658..95038946 100644 --- a/cmd/msgvault/cmd/syncfull.go +++ b/cmd/msgvault/cmd/syncfull.go @@ -167,15 +167,17 @@ Examples: } // Rebuild analytics cache if requested. + var cacheErr error if syncFullRebuildCache { analyticsDir := cfg.AnalyticsDir() fullRebuild := false if staleness := cacheNeedsBuild(dbPath, analyticsDir); staleness.FullRebuild { fullRebuild = true } - result, cacheErr := buildCache(dbPath, analyticsDir, fullRebuild) - if cacheErr != nil { - syncErrors = append(syncErrors, fmt.Sprintf("cache rebuild: %v", cacheErr)) + result, err := buildCache(dbPath, analyticsDir, fullRebuild) + if err != nil { + cacheErr = err + fmt.Printf("\nCache rebuild failed: %v\n", err) } else if !result.Skipped { logger.Info("cache build completed", "exported", result.ExportedCount) } @@ -190,6 +192,9 @@ Examples: return fmt.Errorf("%d account(s) failed to sync: %s", len(syncErrors), strings.Join(syncErrors, "; ")) } + if cacheErr != nil { + return fmt.Errorf("cache rebuild failed: %w", cacheErr) + } return nil },