diff --git a/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaTool.java b/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaTool.java index eef449e3dd11..ac053e8c1a74 100644 --- a/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaTool.java +++ b/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaTool.java @@ -22,6 +22,8 @@ import org.apache.commons.cli.OptionGroup; import org.apache.commons.io.output.NullOutputStream; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaException; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.tools.schematool.HiveSchemaHelper; @@ -29,14 +31,21 @@ import org.apache.hadoop.hive.metastore.tools.schematool.HiveSchemaHelper.MetaStoreConnectionInfo; import org.apache.hadoop.hive.metastore.tools.schematool.HiveSchemaHelper.NestedScriptParser; import org.apache.hive.beeline.BeeLine; +import org.apache.tez.dag.api.TezConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.File; +import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; import java.io.PrintStream; +import java.util.HashMap; +import java.util.Map; + +import static org.apache.hadoop.hive.metastore.utils.StringUtils.isEmpty; public class HiveSchemaTool extends MetastoreSchemaTool { private static final Logger LOG = LoggerFactory.getLogger(HiveSchemaTool.class.getName()); @@ -90,6 +99,7 @@ protected void execSql(String scriptDir, String scriptFile) @Override protected void execSql(String sqlScriptFile) throws IOException { + replaceLocationForProtoLogTables(sqlScriptFile); CommandBuilder builder = new HiveSchemaToolCommandBuilder(conf, url, driver, userName, passWord, sqlScriptFile); @@ -112,6 +122,57 @@ protected void execSql(String sqlScriptFile) throws IOException { } } + void replaceLocationForProtoLogTables(String sqlScriptFile) throws IOException { + TezConfiguration tezConf = new TezConfiguration(true); + boolean hiveProtoLoggingEnabled = true; + boolean tezProtoLoggingEnabled = true; + String hiveProtoBaseDir = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_PROTO_EVENTS_BASE_PATH); + String tezProtoBaseDir = tezConf.get(TezConfiguration.TEZ_HISTORY_LOGGING_PROTO_BASE_DIR); + String hiveLocation = "/tmp/query_data"; // if Hive protologging is not enabled, use dummy location for Hive protolog tables + String tezLocation = "/tmp"; // if Tez protologging is not enabled, use dummy location for Tez protolog tables + String line; + StringBuilder newLine = new StringBuilder(); + Map replacements = new HashMap<>(); + + if (isEmpty(hiveProtoBaseDir)) { + LOG.error("Hive conf variable hive.hook.proto.base-directory is not set for creating protologging tables"); + hiveProtoLoggingEnabled = false; + } + if (isEmpty(tezProtoBaseDir)) { + LOG.error("Tez conf variable tez.history.logging.proto-base-dir is not set for creating protologging tables"); + tezProtoLoggingEnabled = false; + } + + if (hiveProtoLoggingEnabled) { + String hiveProtoScheme = new Path(hiveProtoBaseDir).getFileSystem(conf).getScheme() + ":///"; + hiveLocation = new Path(hiveProtoBaseDir).getFileSystem(conf).getUri().isAbsolute() ? hiveProtoBaseDir : hiveProtoScheme + hiveProtoBaseDir; + } + if (tezProtoLoggingEnabled) { + String tezProtoScheme = new Path(tezProtoBaseDir).getFileSystem(tezConf).getScheme() + ":///"; + tezLocation = new Path(tezProtoBaseDir).getFileSystem(tezConf).getUri().isAbsolute() ? tezProtoBaseDir : tezProtoScheme + tezProtoBaseDir; + } + + replacements.put("_REPLACE_WITH_QUERY_DATA_LOCATION_", hiveLocation); + replacements.put("_REPLACE_WITH_APP_DATA_LOCATION_", tezLocation + "/app_data"); + replacements.put("_REPLACE_WITH_DAG_DATA_LOCATION_", tezLocation + "/dag_data"); + replacements.put("_REPLACE_WITH_DAG_META_LOCATION_", tezLocation + "/dag_meta"); + + try (BufferedReader reader = new BufferedReader(new FileReader(sqlScriptFile))) { + while ((line = reader.readLine()) != null) { + for (Map.Entry entry : replacements.entrySet()) { + if (line.contains(entry.getKey())) { + line = line.replace(entry.getKey(), entry.getValue()); + } + } + newLine.append(line).append("\n"); + } + } + + try (BufferedWriter writer = new BufferedWriter(new FileWriter(sqlScriptFile))) { + writer.write(newLine.toString()); + } + } + static class HiveSchemaToolCommandBuilder extends MetastoreSchemaTool.CommandBuilder { HiveSchemaToolCommandBuilder(Configuration conf, String url, String driver, String userName, diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index dd340b1bd59e..4581dce12cfb 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -5648,6 +5648,7 @@ public static enum ConfVars { "hive.zookeeper.ssl.truststore.password," + "hive.zookeeper.ssl.truststore.type," + "hive.iceberg.allow.datafiles.in.table.location.only," + + "hive.hook.proto.base-directory," + "hive.rewrite.data.policy", "Comma separated list of configuration options which are immutable at runtime"), HIVE_CONF_HIDDEN_LIST("hive.conf.hidden.list", diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestRestrictedList.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestRestrictedList.java index a9c49efc54d3..b4b80d6f8cf0 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestRestrictedList.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestRestrictedList.java @@ -110,6 +110,7 @@ public static void startServices() throws Exception { addToExpectedRestrictedMap("hive.zookeeper.ssl.truststore.password"); addToExpectedRestrictedMap("hive.zookeeper.ssl.truststore.type"); addToExpectedRestrictedMap("hive.iceberg.allow.datafiles.in.table.location.only"); + addToExpectedRestrictedMap("hive.hook.proto.base-directory"); addToExpectedRestrictedMap("hive.rewrite.data.policy"); checkRestrictedListMatch(); diff --git a/ql/src/test/results/clientpositive/llap/resourceplan.q.out b/ql/src/test/results/clientpositive/llap/resourceplan.q.out index 39bb9ea37196..8a29af8520ee 100644 --- a/ql/src/test/results/clientpositive/llap/resourceplan.q.out +++ b/ql/src/test/results/clientpositive/llap/resourceplan.q.out @@ -218,6 +218,22 @@ sys partitions hive_test_user USER DELETE true -1 hive_test_user sys partitions hive_test_user USER INSERT true -1 hive_test_user sys partitions hive_test_user USER SELECT true -1 hive_test_user sys partitions hive_test_user USER UPDATE true -1 hive_test_user +sys proto_hive_query_data hive_test_user USER DELETE true -1 hive_test_user +sys proto_hive_query_data hive_test_user USER INSERT true -1 hive_test_user +sys proto_hive_query_data hive_test_user USER SELECT true -1 hive_test_user +sys proto_hive_query_data hive_test_user USER UPDATE true -1 hive_test_user +sys proto_tez_app_data hive_test_user USER DELETE true -1 hive_test_user +sys proto_tez_app_data hive_test_user USER INSERT true -1 hive_test_user +sys proto_tez_app_data hive_test_user USER SELECT true -1 hive_test_user +sys proto_tez_app_data hive_test_user USER UPDATE true -1 hive_test_user +sys proto_tez_dag_data hive_test_user USER DELETE true -1 hive_test_user +sys proto_tez_dag_data hive_test_user USER INSERT true -1 hive_test_user +sys proto_tez_dag_data hive_test_user USER SELECT true -1 hive_test_user +sys proto_tez_dag_data hive_test_user USER UPDATE true -1 hive_test_user +sys proto_tez_dag_meta hive_test_user USER DELETE true -1 hive_test_user +sys proto_tez_dag_meta hive_test_user USER INSERT true -1 hive_test_user +sys proto_tez_dag_meta hive_test_user USER SELECT true -1 hive_test_user +sys proto_tez_dag_meta hive_test_user USER UPDATE true -1 hive_test_user sys replication_failover_failback_metrics hive_test_user USER DELETE true -1 hive_test_user sys replication_failover_failback_metrics hive_test_user USER INSERT true -1 hive_test_user sys replication_failover_failback_metrics hive_test_user USER SELECT true -1 hive_test_user diff --git a/ql/src/test/results/clientpositive/llap/strict_managed_tables_sysdb.q.out b/ql/src/test/results/clientpositive/llap/strict_managed_tables_sysdb.q.out index e7b7625e0860..70f240aa6a63 100644 --- a/ql/src/test/results/clientpositive/llap/strict_managed_tables_sysdb.q.out +++ b/ql/src/test/results/clientpositive/llap/strict_managed_tables_sysdb.q.out @@ -308,6 +308,22 @@ sys partitions hive_test_user USER DELETE true -1 hive_test_user sys partitions hive_test_user USER INSERT true -1 hive_test_user sys partitions hive_test_user USER SELECT true -1 hive_test_user sys partitions hive_test_user USER UPDATE true -1 hive_test_user +sys proto_hive_query_data hive_test_user USER DELETE true -1 hive_test_user +sys proto_hive_query_data hive_test_user USER INSERT true -1 hive_test_user +sys proto_hive_query_data hive_test_user USER SELECT true -1 hive_test_user +sys proto_hive_query_data hive_test_user USER UPDATE true -1 hive_test_user +sys proto_tez_app_data hive_test_user USER DELETE true -1 hive_test_user +sys proto_tez_app_data hive_test_user USER INSERT true -1 hive_test_user +sys proto_tez_app_data hive_test_user USER SELECT true -1 hive_test_user +sys proto_tez_app_data hive_test_user USER UPDATE true -1 hive_test_user +sys proto_tez_dag_data hive_test_user USER DELETE true -1 hive_test_user +sys proto_tez_dag_data hive_test_user USER INSERT true -1 hive_test_user +sys proto_tez_dag_data hive_test_user USER SELECT true -1 hive_test_user +sys proto_tez_dag_data hive_test_user USER UPDATE true -1 hive_test_user +sys proto_tez_dag_meta hive_test_user USER DELETE true -1 hive_test_user +sys proto_tez_dag_meta hive_test_user USER INSERT true -1 hive_test_user +sys proto_tez_dag_meta hive_test_user USER SELECT true -1 hive_test_user +sys proto_tez_dag_meta hive_test_user USER UPDATE true -1 hive_test_user sys replication_failover_failback_metrics hive_test_user USER DELETE true -1 hive_test_user sys replication_failover_failback_metrics hive_test_user USER INSERT true -1 hive_test_user sys replication_failover_failback_metrics hive_test_user USER SELECT true -1 hive_test_user @@ -520,6 +536,10 @@ PREHOOK: Output: sys@partition_keys PREHOOK: Output: sys@partition_params PREHOOK: Output: sys@partition_stats_view PREHOOK: Output: sys@partitions +PREHOOK: Output: sys@proto_hive_query_data +PREHOOK: Output: sys@proto_tez_app_data +PREHOOK: Output: sys@proto_tez_dag_data +PREHOOK: Output: sys@proto_tez_dag_meta PREHOOK: Output: sys@replication_failover_failback_metrics PREHOOK: Output: sys@replication_metrics PREHOOK: Output: sys@replication_metrics_orig @@ -584,6 +604,10 @@ POSTHOOK: Output: sys@partition_keys POSTHOOK: Output: sys@partition_params POSTHOOK: Output: sys@partition_stats_view POSTHOOK: Output: sys@partitions +POSTHOOK: Output: sys@proto_hive_query_data +POSTHOOK: Output: sys@proto_tez_app_data +POSTHOOK: Output: sys@proto_tez_dag_data +POSTHOOK: Output: sys@proto_tez_dag_meta POSTHOOK: Output: sys@replication_failover_failback_metrics POSTHOOK: Output: sys@replication_metrics POSTHOOK: Output: sys@replication_metrics_orig diff --git a/ql/src/test/results/clientpositive/llap/sysdb.q.out b/ql/src/test/results/clientpositive/llap/sysdb.q.out index 4503715f2300..cd351783b513 100644 --- a/ql/src/test/results/clientpositive/llap/sysdb.q.out +++ b/ql/src/test/results/clientpositive/llap/sysdb.q.out @@ -264,6 +264,22 @@ sys partitions hive_test_user USER DELETE true -1 hive_test_user sys partitions hive_test_user USER INSERT true -1 hive_test_user sys partitions hive_test_user USER SELECT true -1 hive_test_user sys partitions hive_test_user USER UPDATE true -1 hive_test_user +sys proto_hive_query_data hive_test_user USER DELETE true -1 hive_test_user +sys proto_hive_query_data hive_test_user USER INSERT true -1 hive_test_user +sys proto_hive_query_data hive_test_user USER SELECT true -1 hive_test_user +sys proto_hive_query_data hive_test_user USER UPDATE true -1 hive_test_user +sys proto_tez_app_data hive_test_user USER DELETE true -1 hive_test_user +sys proto_tez_app_data hive_test_user USER INSERT true -1 hive_test_user +sys proto_tez_app_data hive_test_user USER SELECT true -1 hive_test_user +sys proto_tez_app_data hive_test_user USER UPDATE true -1 hive_test_user +sys proto_tez_dag_data hive_test_user USER DELETE true -1 hive_test_user +sys proto_tez_dag_data hive_test_user USER INSERT true -1 hive_test_user +sys proto_tez_dag_data hive_test_user USER SELECT true -1 hive_test_user +sys proto_tez_dag_data hive_test_user USER UPDATE true -1 hive_test_user +sys proto_tez_dag_meta hive_test_user USER DELETE true -1 hive_test_user +sys proto_tez_dag_meta hive_test_user USER INSERT true -1 hive_test_user +sys proto_tez_dag_meta hive_test_user USER SELECT true -1 hive_test_user +sys proto_tez_dag_meta hive_test_user USER UPDATE true -1 hive_test_user sys replication_failover_failback_metrics hive_test_user USER DELETE true -1 hive_test_user sys replication_failover_failback_metrics hive_test_user USER INSERT true -1 hive_test_user sys replication_failover_failback_metrics hive_test_user USER SELECT true -1 hive_test_user @@ -781,6 +797,45 @@ partitions part_id partitions part_name partitions sd_id partitions tbl_id +proto_hive_query_data eventtype +proto_hive_query_data executionmode +proto_hive_query_data hivequeryid +proto_hive_query_data operationid +proto_hive_query_data otherinfo +proto_hive_query_data queue +proto_hive_query_data requestuser +proto_hive_query_data tablesread +proto_hive_query_data tableswritten +proto_hive_query_data timestamp +proto_hive_query_data user +proto_tez_app_data app_attempt_id +proto_tez_app_data app_id +proto_tez_app_data dag_id +proto_tez_app_data event_data +proto_tez_app_data event_time +proto_tez_app_data event_type +proto_tez_app_data task_attempt_id +proto_tez_app_data task_id +proto_tez_app_data user +proto_tez_app_data vertex_id +proto_tez_dag_data app_attempt_id +proto_tez_dag_data app_id +proto_tez_dag_data dag_id +proto_tez_dag_data event_data +proto_tez_dag_data event_time +proto_tez_dag_data event_type +proto_tez_dag_data task_attempt_id +proto_tez_dag_data task_id +proto_tez_dag_data user +proto_tez_dag_data vertex_id +proto_tez_dag_meta app_file_path +proto_tez_dag_meta app_id +proto_tez_dag_meta app_launched_event_offset +proto_tez_dag_meta dag_file_path +proto_tez_dag_meta dag_finished_event_offset +proto_tez_dag_meta dag_id +proto_tez_dag_meta dag_submitted_event_offset +proto_tez_dag_meta writetime replication_failover_failback_metrics db_name replication_failover_failback_metrics db_repl_metrics_name replication_failover_failback_metrics db_repl_metrics_value @@ -1124,6 +1179,10 @@ POSTHOOK: query: select pkey_name, pkey_type from partition_keys order by pkey_n POSTHOOK: type: QUERY POSTHOOK: Input: sys@partition_keys #### A masked pattern was here #### +date string +date string +date string +date string PREHOOK: query: select part_key_val, integer_idx from partition_key_vals order by part_key_val, integer_idx limit 5 PREHOOK: type: QUERY PREHOOK: Input: sys@partition_key_vals @@ -1308,7 +1367,7 @@ POSTHOOK: query: select count(*) from sds POSTHOOK: type: QUERY POSTHOOK: Input: sys@sds #### A masked pattern was here #### -79 +83 PREHOOK: query: select param_key, param_value from sd_params order by param_key, param_value limit 5 PREHOOK: type: QUERY PREHOOK: Input: sys@sd_params @@ -1327,9 +1386,9 @@ POSTHOOK: Input: sys@serdes #### A masked pattern was here #### NULL org.apache.hadoop.hive.ql.io.orc.OrcSerde NULL org.apache.hadoop.hive.ql.io.orc.OrcSerde -NULL org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -NULL org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -NULL org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +NULL org.apache.hadoop.hive.ql.io.protobuf.ProtobufMessageSerDe +NULL org.apache.hadoop.hive.ql.io.protobuf.ProtobufMessageSerDe +NULL org.apache.hadoop.hive.ql.io.protobuf.ProtobufMessageSerDe PREHOOK: query: select param_key, param_value from serde_params order by param_key, param_value limit 5 PREHOOK: type: QUERY PREHOOK: Input: sys@serde_params @@ -1732,6 +1791,10 @@ default sys partition_keys BASE_TABLE NULL NULL NULL NULL NULL YES NO NULL default sys partition_params BASE_TABLE NULL NULL NULL NULL NULL YES NO NULL default sys partition_stats_view VIEW NULL NULL NULL NULL NULL NO NO NULL default sys partitions BASE_TABLE NULL NULL NULL NULL NULL YES NO NULL +default sys proto_hive_query_data BASE_TABLE NULL NULL NULL NULL NULL YES NO NULL +default sys proto_tez_app_data BASE_TABLE NULL NULL NULL NULL NULL YES NO NULL +default sys proto_tez_dag_data BASE_TABLE NULL NULL NULL NULL NULL YES NO NULL +default sys proto_tez_dag_meta BASE_TABLE NULL NULL NULL NULL NULL YES NO NULL default sys replication_failover_failback_metrics VIEW NULL NULL NULL NULL NULL NO NO NULL default sys replication_metrics VIEW NULL NULL NULL NULL NULL NO NO NULL default sys replication_metrics_orig BASE_TABLE NULL NULL NULL NULL NULL YES NO NULL diff --git a/standalone-metastore/metastore-server/src/main/sql/hive/hive-schema-4.1.0.hive.sql b/standalone-metastore/metastore-server/src/main/sql/hive/hive-schema-4.1.0.hive.sql index d02dee3e9224..4049ce32d530 100644 --- a/standalone-metastore/metastore-server/src/main/sql/hive/hive-schema-4.1.0.hive.sql +++ b/standalone-metastore/metastore-server/src/main/sql/hive/hive-schema-4.1.0.hive.sql @@ -1592,6 +1592,57 @@ WHERE AND B.PARAM_KEY LIKE 'repl_metrics%'; +CREATE EXTERNAL TABLE IF NOT EXISTS `PROTO_HIVE_QUERY_DATA` +PARTITIONED BY ( + `date` string +) +ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.protobuf.ProtobufMessageSerDe' +STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.protobuf.ProtobufMessageInputFormat' +OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' +LOCATION '_REPLACE_WITH_QUERY_DATA_LOCATION_' +TBLPROPERTIES ( + 'proto.class'='org.apache.hadoop.hive.ql.hooks.proto.HiveHookEvents$HiveHookEventProto', + 'proto.maptypes'='org.apache.hadoop.hive.ql.hooks.proto.MapFieldEntry' +); + +CREATE EXTERNAL TABLE IF NOT EXISTS `PROTO_TEZ_APP_DATA` +PARTITIONED BY ( + `date` string +) +ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.protobuf.ProtobufMessageSerDe' +STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.protobuf.ProtobufMessageInputFormat' +OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' +LOCATION '_REPLACE_WITH_APP_DATA_LOCATION_' +TBLPROPERTIES ( + 'proto.class'='org.apache.tez.dag.history.logging.proto.HistoryLoggerProtos$HistoryEventProto', + 'proto.maptypes'='KVPair' +); + +CREATE EXTERNAL TABLE IF NOT EXISTS `PROTO_TEZ_DAG_DATA` +PARTITIONED BY ( + `date` string +) +ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.protobuf.ProtobufMessageSerDe' +STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.protobuf.ProtobufMessageInputFormat' +OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' +LOCATION '_REPLACE_WITH_DAG_DATA_LOCATION_' +TBLPROPERTIES ( + 'proto.class'='org.apache.tez.dag.history.logging.proto.HistoryLoggerProtos$HistoryEventProto', + 'proto.maptypes'='KVPair' +); + +CREATE EXTERNAL TABLE IF NOT EXISTS `PROTO_TEZ_DAG_META` +PARTITIONED BY ( + `date` string +) +ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.protobuf.ProtobufMessageSerDe' +STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.protobuf.ProtobufMessageInputFormat' +OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' +LOCATION '_REPLACE_WITH_DAG_META_LOCATION_' +TBLPROPERTIES ( + 'proto.class'='org.apache.tez.dag.history.logging.proto.HistoryLoggerProtos$ManifestEntryProto' +); + CREATE DATABASE IF NOT EXISTS INFORMATION_SCHEMA; diff --git a/standalone-metastore/metastore-server/src/main/sql/hive/upgrade-4.0.0-to-4.1.0.hive.sql b/standalone-metastore/metastore-server/src/main/sql/hive/upgrade-4.0.0-to-4.1.0.hive.sql index 4dd7847fc811..1d42ffb242ba 100644 --- a/standalone-metastore/metastore-server/src/main/sql/hive/upgrade-4.0.0-to-4.1.0.hive.sql +++ b/standalone-metastore/metastore-server/src/main/sql/hive/upgrade-4.0.0-to-4.1.0.hive.sql @@ -172,6 +172,58 @@ FROM \"TAB_COL_STATS\"" ); +-- HIVE-27855 +CREATE EXTERNAL TABLE IF NOT EXISTS `PROTO_HIVE_QUERY_DATA` +PARTITIONED BY ( + `date` string +) +ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.protobuf.ProtobufMessageSerDe' +STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.protobuf.ProtobufMessageInputFormat' +OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' +LOCATION '_REPLACE_WITH_QUERY_DATA_LOCATION_' +TBLPROPERTIES ( + 'proto.class'='org.apache.hadoop.hive.ql.hooks.proto.HiveHookEvents$HiveHookEventProto', + 'proto.maptypes'='org.apache.hadoop.hive.ql.hooks.proto.MapFieldEntry' +); + +CREATE EXTERNAL TABLE IF NOT EXISTS `PROTO_TEZ_APP_DATA` +PARTITIONED BY ( + `date` string +) +ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.protobuf.ProtobufMessageSerDe' +STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.protobuf.ProtobufMessageInputFormat' +OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' +LOCATION '_REPLACE_WITH_APP_DATA_LOCATION_' +TBLPROPERTIES ( + 'proto.class'='org.apache.tez.dag.history.logging.proto.HistoryLoggerProtos$HistoryEventProto', + 'proto.maptypes'='KVPair' +); + +CREATE EXTERNAL TABLE IF NOT EXISTS `PROTO_TEZ_DAG_DATA` +PARTITIONED BY ( + `date` string +) +ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.protobuf.ProtobufMessageSerDe' +STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.protobuf.ProtobufMessageInputFormat' +OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' +LOCATION '_REPLACE_WITH_DAG_DATA_LOCATION_' +TBLPROPERTIES ( + 'proto.class'='org.apache.tez.dag.history.logging.proto.HistoryLoggerProtos$HistoryEventProto', + 'proto.maptypes'='KVPair' +); + +CREATE EXTERNAL TABLE IF NOT EXISTS `PROTO_TEZ_DAG_META` +PARTITIONED BY ( + `date` string +) +ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.protobuf.ProtobufMessageSerDe' +STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.protobuf.ProtobufMessageInputFormat' +OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' +LOCATION '_REPLACE_WITH_DAG_META_LOCATION_' +TBLPROPERTIES ( + 'proto.class'='org.apache.tez.dag.history.logging.proto.HistoryLoggerProtos$ManifestEntryProto' +); + CREATE OR REPLACE VIEW `VERSION` AS SELECT 1 AS `VER_ID`, '4.1.0' AS `SCHEMA_VERSION`, 'Hive release version 4.1.0' AS `VERSION_COMMENT`;