diff --git a/api/v1alpha1/sourcecrawler_types.go b/api/v1alpha1/sourcecrawler_types.go index e61df118..4632e1eb 100644 --- a/api/v1alpha1/sourcecrawler_types.go +++ b/api/v1alpha1/sourcecrawler_types.go @@ -60,11 +60,18 @@ type SourceCrawlerSpec struct { SourceCrawlerConfig SourceCrawlerConfig `json:"sourceCrawlerConfig,omitempty"` } +type InaccessibleRootFolder struct { + FolderID string `json:"folderID"` + URL string `json:"url,omitempty"` + Error string `json:"error"` +} + // SourceCrawlerStatus defines the observed state of SourceCrawler. type SourceCrawlerStatus struct { - LastAppliedGeneration int64 `json:"lastAppliedGeneration,omitempty"` - Conditions []metav1.Condition `json:"conditions,omitempty"` - FilesProcessed int64 `json:"filesProcessed,omitempty"` + LastAppliedGeneration int64 `json:"lastAppliedGeneration,omitempty"` + Conditions []metav1.Condition `json:"conditions,omitempty"` + FilesProcessed int64 `json:"filesProcessed,omitempty"` + InaccessibleRootFolders []InaccessibleRootFolder `json:"inaccessibleRootFolders,omitempty"` } // +kubebuilder:object:root=true diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index a055fc17..9792b156 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -634,6 +634,21 @@ func (in *GoogleDriveFolders) DeepCopy() *GoogleDriveFolders { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *InaccessibleRootFolder) DeepCopyInto(out *InaccessibleRootFolder) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InaccessibleRootFolder. +func (in *InaccessibleRootFolder) DeepCopy() *InaccessibleRootFolder { + if in == nil { + return nil + } + out := new(InaccessibleRootFolder) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Job) DeepCopyInto(out *Job) { *out = *in @@ -961,6 +976,11 @@ func (in *SourceCrawlerStatus) DeepCopyInto(out *SourceCrawlerStatus) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.InaccessibleRootFolders != nil { + in, out := &in.InaccessibleRootFolders, &out.InaccessibleRootFolders + *out = make([]InaccessibleRootFolder, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SourceCrawlerStatus. diff --git a/config/crd/bases/operator.dataverse.redhat.com_sourcecrawlers.yaml b/config/crd/bases/operator.dataverse.redhat.com_sourcecrawlers.yaml index 1c13ac15..9d483bc4 100644 --- a/config/crd/bases/operator.dataverse.redhat.com_sourcecrawlers.yaml +++ b/config/crd/bases/operator.dataverse.redhat.com_sourcecrawlers.yaml @@ -176,6 +176,20 @@ spec: filesProcessed: format: int64 type: integer + inaccessibleRootFolders: + items: + properties: + error: + type: string + folderID: + type: string + url: + type: string + required: + - error + - folderID + type: object + type: array lastAppliedGeneration: format: int64 type: integer diff --git a/internal/controller/sourcecrawler_controller.go b/internal/controller/sourcecrawler_controller.go index 2feeb387..9c1e8c07 100644 --- a/internal/controller/sourcecrawler_controller.go +++ b/internal/controller/sourcecrawler_controller.go @@ -139,14 +139,29 @@ func (r *SourceCrawlerReconciler) Reconcile(ctx context.Context, req ctrl.Reques } storedFiles, err := source.SyncFilesToFilestore(ctx, r.fileStore) + + var failedRootFolders []operatorv1alpha1.InaccessibleRootFolder + if gds, ok := source.(*unstructured.GDriveSource); ok && len(gds.FailedRootFolders) > 0 { + failedRootFolders = buildInaccessibleRootFolders(gds.FailedRootFolders, sourceCrawlerConfig.GoogleDriveConfig) + } + if err != nil { + if err := controllerutils.StatusPatch(ctx, r.Client, sourceCrawlerCR, func() { + sourceCrawlerCR.Status.InaccessibleRootFolders = failedRootFolders + }); err != nil { + logger.Error(err, "failed to update SourceCrawler CR status with inaccessible root folders") + } return ctrl.Result{}, r.handleError(ctx, sourceCrawlerCR, fmt.Errorf("failed to store files to filestore: %w", err)) } logger.Info("successfully stored files to filestore", "count", len(storedFiles)) successMessage := fmt.Sprintf("successfully reconciled source crawler: %s", sourceCrawlerCR.Name) + if len(failedRootFolders) > 0 { + successMessage += fmt.Sprintf(", %d inaccessible root folders", len(failedRootFolders)) + } if err := controllerutils.StatusPatch(ctx, r.Client, sourceCrawlerCR, func() { sourceCrawlerCR.Status.FilesProcessed += int64(len(storedFiles)) + sourceCrawlerCR.Status.InaccessibleRootFolders = failedRootFolders sourceCrawlerCR.UpdateStatus(successMessage, nil) }); err != nil { logger.Error(err, "failed to update SourceCrawler CR status") @@ -295,6 +310,25 @@ func extractGDriveFolderID(rawURL string) (string, error) { return "", fmt.Errorf("could not extract folder ID from URL path: %s", u.Path) } +func buildInaccessibleRootFolders(failed []unstructured.FailedRootFolder, gdriveConfig *operatorv1alpha1.GoogleDriveConfig) []operatorv1alpha1.InaccessibleRootFolder { + folderIDToURL := make(map[string]string, len(gdriveConfig.Folders)) + for _, f := range gdriveConfig.Folders { + id, err := extractGDriveFolderID(f.URL) + if err == nil { + folderIDToURL[id] = f.URL + } + } + result := make([]operatorv1alpha1.InaccessibleRootFolder, 0, len(failed)) + for _, f := range failed { + result = append(result, operatorv1alpha1.InaccessibleRootFolder{ + FolderID: f.FolderID, + URL: folderIDToURL[f.FolderID], + Error: f.Error, + }) + } + return result +} + func (r *SourceCrawlerReconciler) handleError(ctx context.Context, sourceCrawlerCR *operatorv1alpha1.SourceCrawler, err error) error { logger := log.FromContext(ctx) logger.Error(err, "encountered error") diff --git a/pkg/unstructured/source.go b/pkg/unstructured/source.go index 2d31391e..61e33b50 100644 --- a/pkg/unstructured/source.go +++ b/pkg/unstructured/source.go @@ -236,6 +236,11 @@ func (s *S3BucketSource) s3Key(filestorePath string) string { return path.Join(s.Prefix, baseName) } +type FailedRootFolder struct { + FolderID string + Error string +} + // GDriveSource implements DataSource for Google Drive folders. type GDriveSource struct { GDriveClient *gdrive.Client @@ -245,6 +250,7 @@ type GDriveSource struct { ConcurrentFolders int ConcurrentDownloads int OutputDir string + FailedRootFolders []FailedRootFolder } // Close releases resources held by the underlying clients. @@ -285,6 +291,10 @@ func (g *GDriveSource) SyncFilesToFilestore(ctx context.Context, fs *filestore.F for i, r := range results { if r.err != nil { logger.Error(r.err, "folder crawl failed", "folderID", g.FolderIDs[i]) + g.FailedRootFolders = append(g.FailedRootFolders, FailedRootFolder{ + FolderID: g.FolderIDs[i], + Error: r.err.Error(), + }) continue } for _, record := range r.result.Records { @@ -307,6 +317,10 @@ func (g *GDriveSource) SyncFilesToFilestore(ctx context.Context, fs *filestore.F } } + if len(g.FailedRootFolders) == len(g.FolderIDs) { + return nil, errors.New("all configured root folders are inaccessible (service account may lack access)") + } + logger.Info("gdrive crawl complete, starting file download", "discoveredFiles", len(fileRecords), "concurrentDownloads", g.ConcurrentDownloads,