Merge branch 'master' of github.com:pingcap/tidb into variable-pipeli…

…ned-flush-concurrency
pingcap · Feb 8, 2025 · 12a4053 · 12a4053
2 parents c2dfed6 + 41c60da
commit 12a4053
Show file tree

Hide file tree

Showing 152 changed files with 13,319 additions and 11,928 deletions.
diff --git a/DEPS.bzl b/DEPS.bzl
@@ -2855,13 +2855,13 @@ def go_deps():
         name = "com_github_golang_glog",
         build_file_proto_mode = "disable_global",
         importpath = "github.com/golang/glog",
-        sha256 = "07688d418628ff30ffd40fde44956d1fb6bae4436003d7fcca40c85236b9484a",
-        strip_prefix = "github.com/golang/[email protected].0",
+        sha256 = "6636c4a48a010844df02886621c32706af2f5e707ad23acb52fe22510b60c822",
+        strip_prefix = "github.com/golang/[email protected].4",
         urls = [
-            "http://bazel-cache.pingcap.net:8080/gomod/github.com/golang/glog/com_github_golang_glog-v1.2.0.zip",
-            "http://ats.apps.svc/gomod/github.com/golang/glog/com_github_golang_glog-v1.2.0.zip",
-            "https://cache.hawkingrei.com/gomod/github.com/golang/glog/com_github_golang_glog-v1.2.0.zip",
-            "https://storage.googleapis.com/pingcapmirror/gomod/github.com/golang/glog/com_github_golang_glog-v1.2.0.zip",
+            "http://bazel-cache.pingcap.net:8080/gomod/github.com/golang/glog/com_github_golang_glog-v1.2.4.zip",
+            "http://ats.apps.svc/gomod/github.com/golang/glog/com_github_golang_glog-v1.2.4.zip",
+            "https://cache.hawkingrei.com/gomod/github.com/golang/glog/com_github_golang_glog-v1.2.4.zip",
+            "https://storage.googleapis.com/pingcapmirror/gomod/github.com/golang/glog/com_github_golang_glog-v1.2.4.zip",
         ],
     )
     go_repository(
@@ -5880,13 +5880,13 @@ def go_deps():
         name = "com_github_pingcap_kvproto",
         build_file_proto_mode = "disable_global",
         importpath = "github.com/pingcap/kvproto",
-        sha256 = "db08607b0c90f3909b66577e9c568d0cbd6b2825d287d7b5caab86ea6e4b60ad",
-        strip_prefix = "github.com/pingcap/[email protected]20250108041715-3b77f2c65c63",
+        sha256 = "35c9d49f6ba393693a0d98cf98323e57cde2f9a93ca58068db64e40d5274451f",
+        strip_prefix = "github.com/pingcap/[email protected]20250117122752-2b87602a94a1",
         urls = [
-            "http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20250108041715-3b77f2c65c63.zip",
-            "http://ats.apps.svc/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20250108041715-3b77f2c65c63.zip",
-            "https://cache.hawkingrei.com/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20250108041715-3b77f2c65c63.zip",
-            "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20250108041715-3b77f2c65c63.zip",
+            "http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20250117122752-2b87602a94a1.zip",
+            "http://ats.apps.svc/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20250117122752-2b87602a94a1.zip",
+            "https://cache.hawkingrei.com/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20250117122752-2b87602a94a1.zip",
+            "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20250117122752-2b87602a94a1.zip",
         ],
     )
     go_repository(
@@ -7050,26 +7050,26 @@ def go_deps():
         name = "com_github_tikv_client_go_v2",
         build_file_proto_mode = "disable_global",
         importpath = "github.com/tikv/client-go/v2",
-        sha256 = "e59695d9b1dd460126e9c7dbc60ab44224e56d6a9c6fea04c88127fb0b82694c",
-        strip_prefix = "github.com/tikv/client-go/[email protected].20250117034919-61e09c6539bd",
+        sha256 = "7ec4bc81c07a51d2c220127b61db15e4fdfb38ceef27de53004b06dde7b4775f",
+        strip_prefix = "github.com/tikv/client-go/[email protected].20250207065327-ec9ad0fd70cf",
         urls = [
-            "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20250117034919-61e09c6539bd.zip",
-            "http://ats.apps.svc/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20250117034919-61e09c6539bd.zip",
-            "https://cache.hawkingrei.com/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20250117034919-61e09c6539bd.zip",
-            "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20250117034919-61e09c6539bd.zip",
+            "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20250207065327-ec9ad0fd70cf.zip",
+            "http://ats.apps.svc/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20250207065327-ec9ad0fd70cf.zip",
+            "https://cache.hawkingrei.com/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20250207065327-ec9ad0fd70cf.zip",
+            "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20250207065327-ec9ad0fd70cf.zip",
         ],
     )
     go_repository(
         name = "com_github_tikv_pd_client",
         build_file_proto_mode = "disable_global",
         importpath = "github.com/tikv/pd/client",
-        sha256 = "a704b691889a87001a40bbcf40b69df8b0d801f063bf07bc15542088df15682a",
-        strip_prefix = "github.com/tikv/pd/[email protected]20250107032658-5c4ab57d68de",
+        sha256 = "a8228263f7b348a97c4fe3231e3d0242f121344f47b9c31a042ff2edc768a86d",
+        strip_prefix = "github.com/tikv/pd/[email protected]20250205134827-ac4e640fac03",
         urls = [
-            "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20250107032658-5c4ab57d68de.zip",
-            "http://ats.apps.svc/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20250107032658-5c4ab57d68de.zip",
-            "https://cache.hawkingrei.com/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20250107032658-5c4ab57d68de.zip",
-            "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20250107032658-5c4ab57d68de.zip",
+            "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20250205134827-ac4e640fac03.zip",
+            "http://ats.apps.svc/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20250205134827-ac4e640fac03.zip",
+            "https://cache.hawkingrei.com/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20250205134827-ac4e640fac03.zip",
+            "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20250205134827-ac4e640fac03.zip",
         ],
     )
     go_repository(
@@ -10222,13 +10222,13 @@ def go_deps():
         name = "org_golang_x_mod",
         build_file_proto_mode = "disable_global",
         importpath = "golang.org/x/mod",
-        sha256 = "0d0a96054b32dfa40cec2a2868bf87cdda31e3bb2dac0a5e2218efe3ef3ef3cc",
-        strip_prefix = "golang.org/x/mod@v0.22.0",
+        sha256 = "ba8f7a56ba1c44d39400237bb3a92fec7b62df9188a488ea0c2b2f29116706dc",
+        strip_prefix = "golang.org/x/mod@v0.23.0",
         urls = [
-            "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/mod/org_golang_x_mod-v0.22.0.zip",
-            "http://ats.apps.svc/gomod/golang.org/x/mod/org_golang_x_mod-v0.22.0.zip",
-            "https://cache.hawkingrei.com/gomod/golang.org/x/mod/org_golang_x_mod-v0.22.0.zip",
-            "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/mod/org_golang_x_mod-v0.22.0.zip",
+            "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/mod/org_golang_x_mod-v0.23.0.zip",
+            "http://ats.apps.svc/gomod/golang.org/x/mod/org_golang_x_mod-v0.23.0.zip",
+            "https://cache.hawkingrei.com/gomod/golang.org/x/mod/org_golang_x_mod-v0.23.0.zip",
+            "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/mod/org_golang_x_mod-v0.23.0.zip",
         ],
     )
     go_repository(
@@ -10248,13 +10248,13 @@ def go_deps():
         name = "org_golang_x_oauth2",
         build_file_proto_mode = "disable_global",
         importpath = "golang.org/x/oauth2",
-        sha256 = "fd0fa73afb9f7a1099bea44c12dc2b8078326f16d3b34ebebc174efdf035bce3",
-        strip_prefix = "golang.org/x/oauth2@v0.24.0",
+        sha256 = "09cda3779f2cab3558ad0da94f4c4ccfb56fe52a6f6cb54ac2edea94edf819c7",
+        strip_prefix = "golang.org/x/oauth2@v0.25.0",
         urls = [
-            "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/oauth2/org_golang_x_oauth2-v0.24.0.zip",
-            "http://ats.apps.svc/gomod/golang.org/x/oauth2/org_golang_x_oauth2-v0.24.0.zip",
-            "https://cache.hawkingrei.com/gomod/golang.org/x/oauth2/org_golang_x_oauth2-v0.24.0.zip",
-            "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/oauth2/org_golang_x_oauth2-v0.24.0.zip",
+            "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/oauth2/org_golang_x_oauth2-v0.25.0.zip",
+            "http://ats.apps.svc/gomod/golang.org/x/oauth2/org_golang_x_oauth2-v0.25.0.zip",
+            "https://cache.hawkingrei.com/gomod/golang.org/x/oauth2/org_golang_x_oauth2-v0.25.0.zip",
+            "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/oauth2/org_golang_x_oauth2-v0.25.0.zip",
         ],
     )
     go_repository(
@@ -10287,13 +10287,13 @@ def go_deps():
         name = "org_golang_x_sys",
         build_file_proto_mode = "disable_global",
         importpath = "golang.org/x/sys",
-        sha256 = "1d759c017d09328a4dacdd4a1a0170cf21641995c6e3ca1f96213369d73c6846",
-        strip_prefix = "golang.org/x/sys@v0.29.0",
+        sha256 = "f5214bfec98f00e5432da68bbece03e98600fa41594ed026556cbde8fcb6f7c9",
+        strip_prefix = "golang.org/x/sys@v0.30.0",
         urls = [
-            "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/sys/org_golang_x_sys-v0.29.0.zip",
-            "http://ats.apps.svc/gomod/golang.org/x/sys/org_golang_x_sys-v0.29.0.zip",
-            "https://cache.hawkingrei.com/gomod/golang.org/x/sys/org_golang_x_sys-v0.29.0.zip",
-            "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/sys/org_golang_x_sys-v0.29.0.zip",
+            "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/sys/org_golang_x_sys-v0.30.0.zip",
+            "http://ats.apps.svc/gomod/golang.org/x/sys/org_golang_x_sys-v0.30.0.zip",
+            "https://cache.hawkingrei.com/gomod/golang.org/x/sys/org_golang_x_sys-v0.30.0.zip",
+            "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/sys/org_golang_x_sys-v0.30.0.zip",
         ],
     )
     go_repository(
@@ -10339,13 +10339,13 @@ def go_deps():
         name = "org_golang_x_time",
         build_file_proto_mode = "disable_global",
         importpath = "golang.org/x/time",
-        sha256 = "ce9157f4961055bd942bbb02666ca5be9f92c92f8e64361aede9d1090df44464",
-        strip_prefix = "golang.org/x/time@v0.7.0",
+        sha256 = "e8a6b77acb93415e3819654d11f072c15eb1e72ba2bd37b6c732f05a359ff794",
+        strip_prefix = "golang.org/x/time@v0.9.0",
         urls = [
-            "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/time/org_golang_x_time-v0.7.0.zip",
-            "http://ats.apps.svc/gomod/golang.org/x/time/org_golang_x_time-v0.7.0.zip",
-            "https://cache.hawkingrei.com/gomod/golang.org/x/time/org_golang_x_time-v0.7.0.zip",
-            "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/time/org_golang_x_time-v0.7.0.zip",
+            "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/time/org_golang_x_time-v0.9.0.zip",
+            "http://ats.apps.svc/gomod/golang.org/x/time/org_golang_x_time-v0.9.0.zip",
+            "https://cache.hawkingrei.com/gomod/golang.org/x/time/org_golang_x_time-v0.9.0.zip",
+            "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/time/org_golang_x_time-v0.9.0.zip",
         ],
     )
     go_repository(

diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES
@@ -119,3 +119,5 @@ aliases:
     - qw4990
     - time-and-fate
     - winoros
+    - terry1purcell
+    - ghazalfamilyusa
diff --git a/WORKSPACE b/WORKSPACE
@@ -85,7 +85,7 @@ go_download_sdk(
         "https://mirrors.aliyun.com/golang/{}",
         "https://dl.google.com/go/{}",
     ],
-    version = "1.23.5",
+    version = "1.23.6",
 )
 
 go_register_toolchains(

diff --git a/br/cmd/br/operator.go b/br/cmd/br/operator.go
@@ -96,9 +96,10 @@ func newListMigrationsCommand() *cobra.Command {
 
 func newMigrateToCommand() *cobra.Command {
 	cmd := &cobra.Command{
-		Use:   "migrate-to",
-		Short: "migrate to a specific version",
-		Args:  cobra.NoArgs,
+		Use: "unsafe-migrate-to",
+		Short: "migrate to a specific version, use truncate will auto migrate to correct version, " +
+			"you should never use this command unless you know what you are doing",
+		Args: cobra.NoArgs,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			cfg := operator.MigrateToConfig{}
 			if err := cfg.ParseFromFlags(cmd.Flags()); err != nil {
@@ -109,6 +110,7 @@ func newMigrateToCommand() *cobra.Command {
 		},
 	}
 	operator.DefineFlagsForMigrateToConfig(cmd.Flags())
+	cmd.Hidden = true
 	return cmd
 }
 

diff --git a/br/pkg/restore/log_client/client.go b/br/pkg/restore/log_client/client.go
@@ -270,7 +270,7 @@ func (rc *LogClient) Close(ctx context.Context) {
 	log.Info("Restore client closed")
 }
 
-func rewriteRulesFor(sst SSTs, rules *restoreutils.RewriteRules) (*restoreutils.RewriteRules, error) {
+func (rc *LogClient) rewriteRulesFor(sst SSTs, rules *restoreutils.RewriteRules) (*restoreutils.RewriteRules, error) {
 	if r, ok := sst.(RewrittenSSTs); ok {
 		rewritten := r.RewrittenTo()
 		if rewritten != sst.TableID() {
@@ -287,6 +287,10 @@ func rewriteRulesFor(sst SSTs, rules *restoreutils.RewriteRules) (*restoreutils.
 			return rewriteRules, nil
 		}
 	}
+	// Need to set ts range for compacted sst to filter out irrelevant data.
+	if sst.Type() == CompactedSSTsType && !rules.HasSetTs() {
+		rules.SetTsRange(rc.shiftStartTS, rc.startTS, rc.restoreTS)
+	}
 	return rules, nil
 }
 
@@ -318,7 +322,7 @@ func (rc *LogClient) RestoreSSTFiles(
 			log.Warn("[Compacted SST Restore] Skipping excluded table during restore.", zap.Int64("table_id", i.TableID()))
 			continue
 		}
-		newRules, err := rewriteRulesFor(i, rewriteRules)
+		newRules, err := rc.rewriteRulesFor(i, rewriteRules)
 		if err != nil {
 			return err
 		}

diff --git a/br/pkg/restore/log_client/compacted_file_strategy.go b/br/pkg/restore/log_client/compacted_file_strategy.go
@@ -51,7 +51,7 @@ func (cs *CompactedFileSplitStrategy) inspect(ssts SSTs) sstIdentity {
 		}
 	}
 
-	rule := utils.GetRewriteRuleOfTable(ssts.TableID(), r.RewrittenTo(), 0, map[int64]int64{}, false)
+	rule := utils.GetRewriteRuleOfTable(ssts.TableID(), r.RewrittenTo(), map[int64]int64{}, false)
 
 	return sstIdentity{
 		EffectiveID:     r.RewrittenTo(),

diff --git a/br/pkg/restore/log_client/log_file_manager.go b/br/pkg/restore/log_client/log_file_manager.go
@@ -495,14 +495,19 @@ func (rc *LogFileManager) ReadAllEntries(
 	return kvEntries, nextKvEntries, nil
 }
 
-func Subcompactions(ctx context.Context, prefix string, s storage.ExternalStorage) SubCompactionIter {
+func Subcompactions(ctx context.Context, prefix string, s storage.ExternalStorage, shiftStartTS, restoredTS uint64) SubCompactionIter {
 	return iter.FlatMap(storage.UnmarshalDir(
 		ctx,
 		&storage.WalkOption{SubDir: prefix},
 		s,
 		func(t *backuppb.LogFileSubcompactions, name string, b []byte) error { return t.Unmarshal(b) },
 	), func(subcs *backuppb.LogFileSubcompactions) iter.TryNextor[*backuppb.LogFileSubcompaction] {
-		return iter.FromSlice(subcs.Subcompactions)
+		return iter.MapFilter(iter.FromSlice(subcs.Subcompactions), func(subc *backuppb.LogFileSubcompaction) (*backuppb.LogFileSubcompaction, bool) {
+			if subc.Meta.InputMaxTs < shiftStartTS || subc.Meta.InputMinTs > restoredTS {
+				return nil, true
+			}
+			return subc, false
+		})
 	})
 }
 

diff --git a/br/pkg/restore/log_client/migration.go b/br/pkg/restore/log_client/migration.go
@@ -170,6 +170,7 @@ func (builder *WithMigrationsBuilder) Build(migs []*backuppb.Migration) WithMigr
 		fullBackups:    fullBackups,
 		restoredTS:     builder.restoredTS,
 		startTS:        builder.startTS,
+		shiftStartTS:   builder.shiftStartTS,
 	}
 	return withMigrations
 }
@@ -222,8 +223,9 @@ type WithMigrations struct {
 	skipmap        metaSkipMap
 	compactionDirs []string
 	fullBackups    []string
-	restoredTS     uint64
+	shiftStartTS   uint64
 	startTS        uint64
+	restoredTS     uint64
 }
 
 func (wm *WithMigrations) Metas(metaNameIter MetaNameIter) MetaMigrationsIter {
@@ -249,7 +251,7 @@ func (wm *WithMigrations) Compactions(ctx context.Context, s storage.ExternalSto
 	compactionDirIter := iter.FromSlice(wm.compactionDirs)
 	return iter.FlatMap(compactionDirIter, func(name string) iter.TryNextor[*backuppb.LogFileSubcompaction] {
 		// name is the absolute path in external storage.
-		return Subcompactions(ctx, name, s)
+		return Subcompactions(ctx, name, s, wm.shiftStartTS, wm.restoredTS)
 	})
 }
 

diff --git a/br/pkg/restore/log_client/ssts.go b/br/pkg/restore/log_client/ssts.go
@@ -17,6 +17,11 @@ var (
 	_ RewrittenSSTs = &CopiedSST{}
 )
 
+const (
+	CompactedSSTsType = 1
+	CopiedSSTsType    = 2
+)
+
 // RewrittenSSTs is an extension to the `SSTs` that needs extra key rewriting.
 // This allows a SST being restored "as if" it in another table.
 //
@@ -37,6 +42,7 @@ type RewrittenSSTs interface {
 type SSTs interface {
 	fmt.Stringer
 
+	Type() int
 	// TableID returns the ID of the table associated with the SST files.
 	// This should be the same as the physical content's table ID.
 	TableID() int64
@@ -51,6 +57,10 @@ type CompactedSSTs struct {
 	*backuppb.LogFileSubcompaction
 }
 
+func (s *CompactedSSTs) Type() int {
+	return CompactedSSTsType
+}
+
 func (s *CompactedSSTs) String() string {
 	return fmt.Sprintf("CompactedSSTs: %s", s.Meta)
 }
@@ -75,7 +85,11 @@ type CopiedSST struct {
 }
 
 func (s *CopiedSST) String() string {
-	return fmt.Sprintf("AddedSSTs: %s", s.File)
+	return fmt.Sprintf("CopiedSSTs: %s", s.File)
+}
+
+func (s *CopiedSST) Type() int {
+	return CopiedSSTsType
 }
 
 func (s *CopiedSST) TableID() int64 {

diff --git a/br/pkg/restore/snap_client/import.go b/br/pkg/restore/snap_client/import.go
@@ -585,6 +585,10 @@ func (importer *SnapFileImporter) buildDownloadRequest(
 	regionInfo *split.RegionInfo,
 	cipher *backuppb.CipherInfo,
 ) (*import_sstpb.DownloadRequest, import_sstpb.SSTMeta, error) {
+	err := rewriteRules.SetTimeRangeFilter(file.Cf)
+	if err != nil {
+		return nil, import_sstpb.SSTMeta{}, err
+	}
 	// Get the rewrite rule for the file.
 	fileRule := restoreutils.FindMatchedRewriteRule(file, rewriteRules)
 	if fileRule == nil {