sr_alter 批量修改表/物化视图副本数

fixpng · fixpng · commit c7b305589878 · 2025-03-27T22:00:14.000+08:00
diff --git a/02-data-processing/sr_alter_table_rn.py b/02-data-processing/sr_alter_table_rn.py
@@ -0,0 +1,102 @@
+"""
+starrocks 批量修改表副本数，可配置修改前和修改后的副本数。
+普通表及分区表。
+"""
+import pymysql
+import time
+
+
+# 副本数配置
+REPLICATION_NUM_BEFORE = "3"  # 修改前的副本数
+REPLICATION_NUM_AFTER = "1"   # 修改后的副本数
+
+# 数据库连接配置
+DB_CONFIG = {
+    "host": "127.0.0.1",
+    "user": "root",
+    "password": "root.COM2025*",
+    "database": "information_schema",
+    "port": 9030,
+}
+
+
+def fetch_tables_with_replication_num(connection):
+    """查询指定 replication_num 的普通表"""
+    query = f"""
+    SELECT TABLE_SCHEMA, TABLE_NAME 
+    FROM information_schema.tables_config 
+    WHERE TABLE_ENGINE='OLAP' AND PROPERTIES LIKE '%"replication_num":"{REPLICATION_NUM_BEFORE}"%'
+    """
+    with connection.cursor() as cursor:
+        cursor.execute(query)
+        return cursor.fetchall()
+
+
+def fetch_partition_tables_with_replication_num(connection):
+    """查询指定 replication_num 的分区表"""
+    query = f"""
+    SELECT TABLE_SCHEMA, TABLE_NAME 
+    FROM information_schema.tables_config 
+    WHERE TABLE_ENGINE='OLAP' AND PARTITION_KEY <> '' AND PROPERTIES LIKE '%"replication_num":"{REPLICATION_NUM_BEFORE}"%'
+    """
+    with connection.cursor() as cursor:
+        cursor.execute(query)
+        return cursor.fetchall()
+
+
+def alter_table_replication_num(connection, schema, table):
+    """修改普通表的 replication_num"""
+    alter_sql = (
+        f'ALTER TABLE `{schema}`.`{table}` SET ("default.replication_num" = "{REPLICATION_NUM_AFTER}")'
+    )
+    try:
+        with connection.cursor() as cursor:
+            cursor.execute(alter_sql)
+            print(f"[INFO] Successfully altered table: {schema}.{table}")
+    except Exception as e:
+        print(f"[ERROR] Failed to alter table {schema}.{table}: {str(e)}")
+
+
+def alter_partition_replication_num(connection, schema, table):
+    """修改分区表的 replication_num"""
+    alter_sql = f'ALTER TABLE `{schema}`.`{table}` MODIFY PARTITION (*) SET("replication_num"="{REPLICATION_NUM_AFTER}")'
+    try:
+        with connection.cursor() as cursor:
+            cursor.execute(alter_sql)
+            print(f"[INFO] Successfully altered partition table: {schema}.{table}")
+    except Exception as e:
+        print(f"[ERROR] Failed to alter partition table {schema}.{table}: {str(e)}")
+
+
+def main():
+    # 连接数据库
+    connection = pymysql.connect(**DB_CONFIG)
+    try:
+        # 获取需要修改的普通表
+        tables = fetch_tables_with_replication_num(connection)
+        if not tables:
+            print(f"[INFO] No normal tables found with replication_num={REPLICATION_NUM_BEFORE}.")
+        else:
+            # 遍历并修改普通表
+            for schema, table in tables:
+                alter_table_replication_num(connection, schema, table)
+                time.sleep(0.5)
+
+        # 获取需要修改的分区表
+        partition_tables = fetch_partition_tables_with_replication_num(connection)
+        if not partition_tables:
+            print(f"[INFO] No partition tables found with replication_num={REPLICATION_NUM_BEFORE}.")
+        else:
+            # 遍历并修改分区表
+            for schema, table in partition_tables:
+                alter_partition_replication_num(connection, schema, table)
+                time.sleep(0.5)
+
+        # 提交事务
+        connection.commit()
+    finally:
+        connection.close()
+
+
+if __name__ == "__main__":
+    main()
diff --git a/02-data-processing/sr_alter_view_rn.py b/02-data-processing/sr_alter_view_rn.py
@@ -0,0 +1,112 @@
+"""
+starrocks 批量修改物化视图副本数，可配置修改前和修改后的副本数。(删除重建)
+SR (v3.2.15)当前物化视图无法修改副本数，已反馈官方：
+MySQL [information_schema]> ALTER  MATERIALIZED VIEW  dsj_dwd.mv_zt_hhy_staff_user_company_info SET ("replication_num" = "1");
+ERROR 1064 (HY000): Unexpected exception: Getting analyzing error. Detail message: Modify failed because unknown properties: {replication_num=1}, please add `session.` prefix if you want add session variables for mv(eg, "session.query_timeout"="30000000").. 
+"""
+import pymysql
+import time
+
+# 副本数配置
+REPLICATION_NUM_BEFORE = "3"  # 修改前的副本数
+REPLICATION_NUM_AFTER = "1"   # 修改后的副本数
+
+# 数据库连接配置
+DB_CONFIG = {
+    "host": "127.0.0.1",
+    "user": "root",
+    "password": "root.COM2025*",
+    "database": "information_schema",
+    "port": 9030,
+}
+
+
+def fetch_materialized_views():
+    """查询所有指定 replication_num 的物化视图"""
+    query = f"""
+    SELECT TABLE_SCHEMA, TABLE_NAME
+    FROM information_schema.tables_config 
+    WHERE TABLE_ENGINE = 'MATERIALIZED_VIEW' 
+    AND PROPERTIES LIKE '%"replication_num":"{REPLICATION_NUM_BEFORE}"%';
+    """
+    connection = pymysql.connect(**DB_CONFIG)
+    try:
+        with connection.cursor() as cursor:
+            cursor.execute(query)
+            return cursor.fetchall()
+    finally:
+        connection.close()
+
+
+def get_create_view_statement(schema, table):
+    """获取物化视图的创建语句"""
+    query = f"SHOW CREATE TABLE `{schema}`.`{table}`;"
+    connection = pymysql.connect(**DB_CONFIG)
+    try:
+        with connection.cursor() as cursor:
+            cursor.execute(query)
+            result = cursor.fetchone()
+            return result[1]  # 返回创建语句
+    finally:
+        connection.close()
+
+
+def drop_view(schema, table):
+    """删除物化视图"""
+    query = f"DROP MATERIALIZED VIEW `{schema}`.`{table}`;"
+    connection = pymysql.connect(**DB_CONFIG)
+    try:
+        with connection.cursor() as cursor:
+            cursor.execute(query)
+        connection.commit()
+    finally:
+        connection.close()
+
+
+def create_view(create_statement):
+    """创建物化视图"""
+    print(create_statement)
+    connection = pymysql.connect(**DB_CONFIG)
+    try:
+        with connection.cursor() as cursor:
+            cursor.execute(create_statement)
+        connection.commit()
+    finally:
+        connection.close()
+
+
+def update_replication_num(schema, create_statement):
+    """修改创建语句中的 replication_num"""
+    return create_statement.replace(
+        f'"replication_num" = "{REPLICATION_NUM_BEFORE}"',
+        f'"replication_num" = "{REPLICATION_NUM_AFTER}"'
+    ).replace("CREATE MATERIALIZED VIEW ", f"CREATE MATERIALIZED VIEW `{schema}`.")
+
+
+def main():
+    # 获取所有指定 replication_num 的物化视图
+    views = fetch_materialized_views()
+    for schema, table in views:
+        print(f"Processing view: {schema}.{table}")
+
+        # 获取创建语句
+        create_statement = get_create_view_statement(schema, table)
+        print("Original create statement fetched.")
+
+        # 修改 replication_num
+        updated_statement = update_replication_num(schema, create_statement)
+        print(f"Create statement updated with replication_num={REPLICATION_NUM_AFTER}.")
+        time.sleep(0.5)
+
+        # 删除原视图
+        drop_view(schema, table)
+        print(f"Original view dropped: {schema}.{table}")
+        time.sleep(0.5)
+
+        # 重新创建视图
+        create_view(updated_statement)
+        print(f"View recreated with updated replication_num={REPLICATION_NUM_AFTER}.----------")
+
+
+if __name__ == "__main__":
+    main()
