From cb17c76bf509bc6541a773a9e8151010fd73085d Mon Sep 17 00:00:00 2001 From: j-rafique Date: Mon, 9 Sep 2024 15:35:17 +0500 Subject: [PATCH] [PSL-1262] meta worker improvements for the migration --- p2p/kademlia/store/sqlite/meta_worker.go | 11 +++++------ .../metamigrator/migration_data_identifier.go | 3 ++- supernode/services/metamigrator/service.go | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/p2p/kademlia/store/sqlite/meta_worker.go b/p2p/kademlia/store/sqlite/meta_worker.go index 3c5368643..19d0e4acc 100644 --- a/p2p/kademlia/store/sqlite/meta_worker.go +++ b/p2p/kademlia/store/sqlite/meta_worker.go @@ -539,10 +539,10 @@ func (d *MigrationMetaStore) checkAndExecuteMigration(ctx context.Context) { log.WithContext(ctx).WithError(err).Error("migration worker: check disk space failed") } - //if !isLow { - // Disk space is sufficient, stop migration - //return - //} + if !isLow { + //Disk space is sufficient, stop migration + return + } log.WithContext(ctx).WithField("islow", isLow).Info("Starting data migration") // Step 1: Fetch pending migrations @@ -807,8 +807,7 @@ func (d *MigrationMetaStore) InsertMetaMigrationData(ctx context.Context, migrat now := time.Now() for _, key := range keys { if _, err := stmt.Exec(key, migrationID, now, now); err != nil { - tx.Rollback() - return fmt.Errorf("failed to insert meta migration data: %w", err) + continue } } diff --git a/supernode/services/metamigrator/migration_data_identifier.go b/supernode/services/metamigrator/migration_data_identifier.go index 1077a605c..d2e5086e4 100644 --- a/supernode/services/metamigrator/migration_data_identifier.go +++ b/supernode/services/metamigrator/migration_data_identifier.go @@ -45,7 +45,7 @@ func (task *MetaMigratorTask) IdentifyMigrationData(ctx context.Context) (err er log.WithContext(ctx).Info("no stale data found to migrate") return nil } - log.WithContext(ctx).WithField("total_keys", totalCount).Info("total-data that needs to migrate has been identified") + log.WithContext(ctx).WithField("total_keys", totalCount).Info("total-data that needs to be migrated has been identified") numOfBatches := getNumOfBatches(totalCount) log.WithContext(ctx).WithField("no_of_batches", numOfBatches).Info("batches required to store migration-meta has been calculated") @@ -65,6 +65,7 @@ func (task *MetaMigratorTask) IdentifyMigrationData(ctx context.Context) (err er log.WithContext(ctx).WithField("batch", batchNo).Debug("data added to migration-meta for migration") } + log.WithContext(ctx).Info("all data that needs to be migrated has been added to meta-migration") return nil } diff --git a/supernode/services/metamigrator/service.go b/supernode/services/metamigrator/service.go index f2e885651..3561ad6a5 100644 --- a/supernode/services/metamigrator/service.go +++ b/supernode/services/metamigrator/service.go @@ -12,7 +12,7 @@ import ( const ( logPrefix = "MetaMigrator" - defaultMetaMigratorDataIdentifier = 24 * time.Hour + defaultMetaMigratorDataIdentifier = 120 * time.Minute lowSpaceThresholdGB = 50 // in GB )