Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add table load conf on 2.5.x #2812

Merged
merged 8 commits into from
Mar 25, 2025
Merged

add table load conf on 2.5.x #2812

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
798bd57
add invalidate all store and region api
lizhenhuan Mar 18, 2025
847404b
change hashmap remove to iterator remove
lizhenhuan Mar 20, 2025
c65649b
add table load conf
lizhenhuan Mar 20, 2025
b656d9f
remove code
lizhenhuan Mar 22, 2025
19014be
remove code
lizhenhuan Mar 22, 2025
77ab6bb
remove code
lizhenhuan Mar 22, 2025
d70b883
remove log code
lizhenhuan Mar 22, 2025
2167c4c
fix code style
lizhenhuan Mar 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions core/src/main/scala/com/pingcap/tispark/TiConfigConst.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,4 +56,8 @@ object TiConfigConst {

val CHUNK_CODEC_FORMAT: String = "chunk"
val DEFAULT_CODEC_FORMAT: String = "chblock"

// cache load
val LOAD_TABLES: String = "spark.tispark.load_tables"
val DEFAULT_LOAD_TABLES: Boolean = true
}
3 changes: 3 additions & 0 deletions core/src/main/scala/com/pingcap/tispark/utils/TiUtil.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,9 @@ object TiUtil {
tiConf.setKvClientConcurrency(conf.get(TiConfigConst.KV_CLIENT_CONCURRENCY).toInt)
}

tiConf.setLoadTables(
conf.get(TiConfigConst.LOAD_TABLES, TiConfigConst.DEFAULT_LOAD_TABLES.toString).toBoolean)

tiConf
}

Expand Down
10 changes: 10 additions & 0 deletions tikv-client/src/main/java/com/pingcap/tikv/TiConfiguration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,8 @@ public class TiConfiguration implements Serializable {

private List<TiStoreType> isolationReadEngines = DEF_ISOLATION_READ_ENGINES;

private boolean loadTables = true;

public static TiConfiguration createDefault(String pdAddrsStr) {
Objects.requireNonNull(pdAddrsStr, "pdAddrsStr is null");
TiConfiguration conf = new TiConfiguration();
Expand Down Expand Up @@ -357,4 +359,12 @@ public int getKvClientConcurrency() {
public void setKvClientConcurrency(int kvClientConcurrency) {
this.kvClientConcurrency = kvClientConcurrency;
}

public void setLoadTables(boolean loadTables) {
this.loadTables = loadTables;
}

public boolean getLoadTables() {
return loadTables;
}
}
2 changes: 1 addition & 1 deletion tikv-client/src/main/java/com/pingcap/tikv/TiSession.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ public Catalog getCatalog() {
if (res == null) {
synchronized (this) {
if (catalog == null) {
catalog = new Catalog(this::createSnapshot, conf.ifShowRowId(), conf.getDBPrefix());
catalog = new Catalog(this::createSnapshot, conf.ifShowRowId(), conf.getDBPrefix(), conf.getLoadTables());
}
res = catalog;
}
Expand Down
13 changes: 8 additions & 5 deletions tikv-client/src/main/java/com/pingcap/tikv/catalog/Catalog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,13 +40,16 @@ public class Catalog implements AutoCloseable {
private final Supplier<Snapshot> snapshotProvider;
private CatalogCache metaCache;
private static final AtomicLong lastUpdateTime = new AtomicLong(0);
private final boolean loadTables;

public Catalog(Supplier<Snapshot> snapshotProvider, boolean showRowId, String dbPrefix) {
public Catalog(
Supplier<Snapshot> snapshotProvider, boolean showRowId, String dbPrefix, boolean loadTables) {
this.snapshotProvider = Objects.requireNonNull(snapshotProvider, "Snapshot Provider is null");
this.showRowId = showRowId;
this.dbPrefix = dbPrefix;
metaCache = new CatalogCache(new CatalogTransaction(snapshotProvider.get()), dbPrefix, false);
reloadCache(true);
this.loadTables = loadTables;
metaCache =
new CatalogCache(new CatalogTransaction(snapshotProvider.get()), dbPrefix, loadTables);
}

@Override
Expand Down Expand Up @@ -77,7 +80,7 @@ public List<TiDBInfo> listDatabases() {

public List<TiTableInfo> listTables(TiDBInfo database) {
Objects.requireNonNull(database, "database is null");
reloadCache(true);
reloadCache(loadTables);
if (showRowId) {
return metaCache
.listTables(database)
Expand Down Expand Up @@ -147,7 +150,7 @@ public TiTableInfo getTable(String dbName, String tableName) {
public TiTableInfo getTable(TiDBInfo database, String tableName) {
Objects.requireNonNull(database, "database is null");
Objects.requireNonNull(tableName, "tableName is null");
reloadCache(true);
reloadCache(loadTables);
TiTableInfo table = metaCache.getTable(database, tableName);
if (showRowId && table != null) {
return table.copyTableWithRowId();
Expand Down