diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/mapreduce/index/IndexToolTableUtil.java b/phoenix-core-client/src/main/java/org/apache/phoenix/mapreduce/index/IndexToolTableUtil.java new file mode 100644 index 00000000000..42dd760cbe3 --- /dev/null +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/mapreduce/index/IndexToolTableUtil.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.mapreduce.index; + + + +import java.io.IOException; +import java.sql.Connection; + +import java.sql.SQLException; +import java.util.UUID; + + +import org.apache.hadoop.hbase.NamespaceDescriptor; +import org.apache.phoenix.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.hbase.TableExistsException; +import org.apache.hadoop.hbase.client.*; +import org.apache.phoenix.coprocessorclient.MetaDataProtocol; +import org.apache.phoenix.query.QueryConstants; + +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.hbase.TableName; + +import org.apache.phoenix.jdbc.PhoenixConnection; + +import org.apache.phoenix.query.ConnectionQueryServices; + +import org.apache.phoenix.schema.PTableType; +import org.apache.phoenix.util.ClientUtil; +import org.apache.phoenix.util.SchemaUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.phoenix.query.QueryConstants.SYSTEM_SCHEMA_NAME; + +/** + * Utility class to create index tables and/or migrate them. + * + */ +public class IndexToolTableUtil extends Configured { + private static final Logger LOGGER = LoggerFactory.getLogger(IndexToolTableUtil.class); + + public final static String OUTPUT_TABLE_NAME = "PHOENIX_INDEX_TOOL"; + public static String SYSTEM_OUTPUT_TABLE_NAME = SchemaUtil.getTableName(SYSTEM_SCHEMA_NAME, + OUTPUT_TABLE_NAME); + + public final static String RESULT_TABLE_NAME = "PHOENIX_INDEX_TOOL_RESULT"; + public static String SYSTEM_RESULT_TABLE_NAME = SchemaUtil.getTableName(SYSTEM_SCHEMA_NAME, + RESULT_TABLE_NAME); + + public static void setIndexToolTableName(Connection connection) throws Exception { + ConnectionQueryServices queryServices = connection.unwrap(PhoenixConnection.class).getQueryServices(); + if (SchemaUtil.isNamespaceMappingEnabled(PTableType.SYSTEM, queryServices.getConfiguration())) { + SYSTEM_OUTPUT_TABLE_NAME = SchemaUtil.getTableName(SYSTEM_SCHEMA_NAME, OUTPUT_TABLE_NAME).replace( + QueryConstants.NAME_SEPARATOR, + QueryConstants.NAMESPACE_SEPARATOR); + SYSTEM_RESULT_TABLE_NAME = SchemaUtil.getTableName(SYSTEM_SCHEMA_NAME, RESULT_TABLE_NAME).replace( + QueryConstants.NAME_SEPARATOR, + QueryConstants.NAMESPACE_SEPARATOR); + } else { + SYSTEM_OUTPUT_TABLE_NAME = SchemaUtil.getTableName(SYSTEM_SCHEMA_NAME, OUTPUT_TABLE_NAME); + SYSTEM_RESULT_TABLE_NAME = SchemaUtil.getTableName(SYSTEM_SCHEMA_NAME, RESULT_TABLE_NAME); + } + } + + public static Table createResultTable(Connection connection) throws IOException, SQLException { + ConnectionQueryServices queryServices = connection.unwrap(PhoenixConnection.class).getQueryServices(); + try (Admin admin = queryServices.getAdmin()) { + TableName resultTableName = TableName.valueOf(SYSTEM_RESULT_TABLE_NAME); + createSystemNamespaceTable(connection); + return createTable(admin, resultTableName); + } + } + + public static Table createOutputTable(Connection connection) throws IOException, SQLException { + ConnectionQueryServices queryServices = connection.unwrap(PhoenixConnection.class).getQueryServices(); + try (Admin admin = queryServices.getAdmin()) { + TableName outputTableName = TableName.valueOf(SYSTEM_OUTPUT_TABLE_NAME); + createSystemNamespaceTable(connection); + return createTable(admin, outputTableName); + } + } + + public static void createSystemNamespaceTable(Connection connection) throws IOException, SQLException { + ConnectionQueryServices queryServices = connection.unwrap(PhoenixConnection.class).getQueryServices(); + if (SchemaUtil.isNamespaceMappingEnabled(PTableType.SYSTEM, queryServices.getConfiguration())) { + try (Admin admin = queryServices.getAdmin()) { + if (!ClientUtil.isHBaseNamespaceAvailable(admin, SYSTEM_SCHEMA_NAME)) { + NamespaceDescriptor namespaceDescriptor = + NamespaceDescriptor.create(SYSTEM_SCHEMA_NAME).build(); + admin.createNamespace(namespaceDescriptor); + } + } + } + } + + @VisibleForTesting + private static Table createTable(Admin admin, TableName tableName) throws IOException { + if (!admin.tableExists(tableName)) { + ColumnFamilyDescriptor columnDescriptor = + ColumnFamilyDescriptorBuilder + .newBuilder(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES) + .setTimeToLive(MetaDataProtocol.DEFAULT_LOG_TTL) + .build(); + TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName) + .setColumnFamily(columnDescriptor).build(); + try { + admin.createTable(tableDescriptor); + } catch (TableExistsException e) { + LOGGER.warn("Table exists, ignoring", e); + } + } + return admin.getConnection().getTable(tableName); + } + + + public static void createNewIndexToolTables(Connection connection) throws Exception { + setIndexToolTableName(connection); + + migrateTable(connection, OUTPUT_TABLE_NAME); + migrateTable(connection, RESULT_TABLE_NAME); + } + + private static void migrateTable(Connection connection, String tableName) throws Exception { + if (!tableName.equals(OUTPUT_TABLE_NAME) && !tableName.equals(RESULT_TABLE_NAME)) { + LOGGER.info("Only migrating PHOENIX_INDEX_TOOL tables!"); + } else { + ConnectionQueryServices queryServices = connection.unwrap(PhoenixConnection.class).getQueryServices(); + try (Admin admin = queryServices.getAdmin()) { + TableName oldTableName = TableName.valueOf(tableName); + String newTableNameString = tableName.equals(OUTPUT_TABLE_NAME) ? + SYSTEM_OUTPUT_TABLE_NAME : SYSTEM_RESULT_TABLE_NAME; + + TableName newTableName = TableName.valueOf(newTableNameString); + + if (admin.tableExists(oldTableName)) { + String snapshotName = tableName + "_" + UUID.randomUUID(); + admin.disableTable(oldTableName); + admin.snapshot(snapshotName, oldTableName); + admin.cloneSnapshot(snapshotName, newTableName); + admin.deleteSnapshot(snapshotName); + admin.deleteTable(oldTableName); + } else { + createTable(admin, newTableName); + } + } + } + } +} diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java index bb6a648c7d5..9cda0ebabe6 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java @@ -323,6 +323,8 @@ import org.apache.phoenix.thirdparty.com.google.common.collect.Maps; import org.apache.phoenix.thirdparty.com.google.common.collect.Sets; +import org.apache.phoenix.mapreduce.index.IndexToolTableUtil; + public class ConnectionQueryServicesImpl extends DelegateQueryServices implements ConnectionQueryServices { private static final Logger LOGGER = LoggerFactory.getLogger(ConnectionQueryServicesImpl.class); @@ -4025,6 +4027,12 @@ private void createOtherSystemTables(PhoenixConnection metaConnection) throws SQ try { metaConnection.createStatement().executeUpdate(getCDCStreamDDL()); } catch (TableAlreadyExistsException ignore) {} + try { + // check if we have old PHOENIX_INDEX_TOOL tables + // move data to the new tables under System, or simply create the new tables + IndexToolTableUtil.createNewIndexToolTables(metaConnection); + + } catch (Exception ignore) {} } /** @@ -4588,6 +4596,13 @@ public void upgradeSystemTables(final String url, final Properties props) throws // with SYSTEM Namespace createSchemaIfNotExistsSystemNSMappingEnabled(metaConnection); + try { + // check if we have old PHOENIX_INDEX_TOOL tables + // move data to the new tables under System, or simply create the new tables + IndexToolTableUtil.createNewIndexToolTables(metaConnection); + + } catch (Exception ignore) {} + clearUpgradeRequired(); success = true; } catch (UpgradeInProgressException | UpgradeNotRequiredException e) { diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java b/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java index 7e28621461c..eb3918784a0 100644 --- a/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java +++ b/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java @@ -38,6 +38,8 @@ import java.util.Map; import java.util.UUID; +import org.apache.phoenix.query.QueryConstants; +import org.apache.phoenix.schema.PTableType; import org.apache.phoenix.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.phoenix.thirdparty.com.google.common.base.Strings; import org.apache.phoenix.hbase.index.AbstractValueGetter; @@ -985,10 +987,22 @@ private void setupIndexAndDataTable(Connection connection) throws SQLException, indexTable, qDataTable)); } qSchemaName = SchemaUtil.normalizeIdentifier(schemaName); + pIndexTable = connection.unwrap(PhoenixConnection.class).getTable( SchemaUtil.getQualifiedTableName(schemaName, indexTable)); + if (SchemaUtil.isNamespaceMappingEnabled(PTableType.SYSTEM, getConf())) { + pIndexTable = connection.unwrap(PhoenixConnection.class).getTable( + SchemaUtil.getQualifiedTableName(schemaName, indexTable).replace( + QueryConstants.NAME_SEPARATOR, + QueryConstants.NAMESPACE_SEPARATOR)); + } + indexType = pIndexTable.getIndexType(); qIndexTable = SchemaUtil.getQualifiedTableName(schemaName, indexTable); + if (SchemaUtil.isNamespaceMappingEnabled(PTableType.SYSTEM, getConf())) { + qIndexTable = qIndexTable.replace(QueryConstants.NAME_SEPARATOR, QueryConstants.NAMESPACE_SEPARATOR); + } + if (IndexType.LOCAL.equals(indexType)) { isLocalIndexBuild = true; if (useSnapshot) { diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/index/IndexVerificationOutputRepository.java b/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/index/IndexVerificationOutputRepository.java index 57a32dd96cd..358459c07ae 100644 --- a/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/index/IndexVerificationOutputRepository.java +++ b/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/index/IndexVerificationOutputRepository.java @@ -19,20 +19,12 @@ import org.apache.phoenix.thirdparty.com.google.common.annotations.VisibleForTesting; -import org.apache.hadoop.hbase.TableExistsException; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; -import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.client.TableDescriptor; -import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.phoenix.coprocessorclient.MetaDataProtocol; import org.apache.phoenix.hbase.index.table.HTableFactory; import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr; import org.apache.phoenix.jdbc.PhoenixConnection; @@ -60,8 +52,8 @@ public class IndexVerificationOutputRepository implements AutoCloseable { IndexTool.IndexDisableLoggingType.NONE; private boolean shouldLogBeyondMaxLookback = true; - public final static String OUTPUT_TABLE_NAME = "PHOENIX_INDEX_TOOL"; - public final static byte[] OUTPUT_TABLE_NAME_BYTES = Bytes.toBytes(OUTPUT_TABLE_NAME); + public final static String OUTPUT_TABLE_NAME = IndexToolTableUtil.SYSTEM_OUTPUT_TABLE_NAME; + public final static byte[] OUTPUT_TABLE_NAME_BYTES = Bytes.toBytes(IndexToolTableUtil.SYSTEM_OUTPUT_TABLE_NAME); public final static byte[] OUTPUT_TABLE_COLUMN_FAMILY = QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES; public final static String DATA_TABLE_NAME = "DTName"; @@ -177,26 +169,7 @@ private static byte[] generatePartialOutputTableRowKey(long ts, byte[] indexTabl } public void createOutputTable(Connection connection) throws IOException, SQLException { - ConnectionQueryServices queryServices = connection.unwrap(PhoenixConnection.class).getQueryServices(); - try (Admin admin = queryServices.getAdmin()) { - TableName outputTableName = TableName.valueOf(OUTPUT_TABLE_NAME); - if (!admin.tableExists(outputTableName)) { - ColumnFamilyDescriptor columnDescriptor = - ColumnFamilyDescriptorBuilder - .newBuilder(OUTPUT_TABLE_COLUMN_FAMILY) - .setTimeToLive(MetaDataProtocol.DEFAULT_LOG_TTL) - .build(); - TableDescriptor tableDescriptor = TableDescriptorBuilder - .newBuilder(TableName.valueOf(OUTPUT_TABLE_NAME)) - .setColumnFamily(columnDescriptor).build(); - try { - admin.createTable(tableDescriptor); - } catch (TableExistsException e) { - LOGGER.warn("Table exists, ignoring", e); - } - outputTable = admin.getConnection().getTable(outputTableName); - } - } + outputTable = IndexToolTableUtil.createOutputTable(connection); } @VisibleForTesting diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/index/IndexVerificationResultRepository.java b/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/index/IndexVerificationResultRepository.java index c00b2ae9208..601ebfaa71b 100644 --- a/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/index/IndexVerificationResultRepository.java +++ b/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/index/IndexVerificationResultRepository.java @@ -19,27 +19,18 @@ import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.TableExistsException; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; -import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.client.TableDescriptor; -import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.regionserver.Region; import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.coprocessor.IndexToolVerificationResult; -import org.apache.phoenix.coprocessorclient.MetaDataProtocol; import org.apache.phoenix.hbase.index.table.HTableFactory; import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr; import org.apache.phoenix.jdbc.PhoenixConnection; -import org.apache.phoenix.query.ConnectionQueryServices; import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.util.ByteUtil; import org.slf4j.Logger; @@ -58,8 +49,8 @@ public class IndexVerificationResultRepository implements AutoCloseable { private Table indexTable; public static final String ROW_KEY_SEPARATOR = "|"; public static final byte[] ROW_KEY_SEPARATOR_BYTE = Bytes.toBytes(ROW_KEY_SEPARATOR); - public final static String RESULT_TABLE_NAME = "PHOENIX_INDEX_TOOL_RESULT"; - public final static byte[] RESULT_TABLE_NAME_BYTES = Bytes.toBytes(RESULT_TABLE_NAME); + public static String RESULT_TABLE_NAME = IndexToolTableUtil.SYSTEM_RESULT_TABLE_NAME; + public static byte[] RESULT_TABLE_NAME_BYTES = Bytes.toBytes(IndexToolTableUtil.SYSTEM_RESULT_TABLE_NAME); public final static byte[] RESULT_TABLE_COLUMN_FAMILY = QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES; public final static String SCANNED_DATA_ROW_COUNT = "ScannedDataRowCount"; public final static byte[] SCANNED_DATA_ROW_COUNT_BYTES = Bytes.toBytes(SCANNED_DATA_ROW_COUNT); @@ -152,37 +143,18 @@ public IndexVerificationResultRepository(){ } public IndexVerificationResultRepository(Connection conn, byte[] indexNameBytes) throws SQLException { - resultTable = getTable(conn, RESULT_TABLE_NAME_BYTES); + resultTable = getTable(conn, Bytes.toBytes(IndexToolTableUtil.SYSTEM_RESULT_TABLE_NAME)); indexTable = getTable(conn, indexNameBytes); } public IndexVerificationResultRepository(byte[] indexName, HTableFactory hTableFactory) throws IOException { - resultTable = hTableFactory.getTable(new ImmutableBytesPtr(RESULT_TABLE_NAME_BYTES)); + resultTable = hTableFactory.getTable(new ImmutableBytesPtr(Bytes.toBytes(IndexToolTableUtil.SYSTEM_RESULT_TABLE_NAME))); indexTable = hTableFactory.getTable(new ImmutableBytesPtr(indexName)); } public void createResultTable(Connection connection) throws IOException, SQLException { - ConnectionQueryServices queryServices = connection.unwrap(PhoenixConnection.class).getQueryServices(); - try (Admin admin = queryServices.getAdmin()) { - TableName resultTableName = TableName.valueOf(RESULT_TABLE_NAME); - if (!admin.tableExists(resultTableName)) { - ColumnFamilyDescriptor columnDescriptor = - ColumnFamilyDescriptorBuilder - .newBuilder(RESULT_TABLE_COLUMN_FAMILY) - .setTimeToLive(MetaDataProtocol.DEFAULT_LOG_TTL) - .build(); - TableDescriptor tableDescriptor = - TableDescriptorBuilder.newBuilder(resultTableName) - .setColumnFamily(columnDescriptor).build(); - try { - admin.createTable(tableDescriptor); - } catch (TableExistsException e) { - LOGGER.warn("Table exists, ignoring", e); - } - resultTable = admin.getConnection().getTable(resultTableName); - } - } + resultTable = IndexToolTableUtil.createResultTable(connection); } private static byte[] generatePartialResultTableRowKey(long ts, byte[] indexTableName) { @@ -356,7 +328,7 @@ private IndexToolVerificationResult aggregateVerificationResult( public IndexToolVerificationResult getVerificationResult(Connection conn, long ts, byte[] indexTableName) throws IOException, SQLException { - try (Table hTable = getTable(conn, RESULT_TABLE_NAME_BYTES)) { + try (Table hTable = getTable(conn, Bytes.toBytes(IndexToolTableUtil.SYSTEM_RESULT_TABLE_NAME))) { byte[] startRowKey = generatePartialResultTableRowKey(ts, indexTableName); byte[] stopRowKey = ByteUtil.calculateTheClosestNextRowKeyForPrefix( diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/index/PhoenixIndexImportDirectReducer.java b/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/index/PhoenixIndexImportDirectReducer.java index ed011736866..ba06c75e967 100644 --- a/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/index/PhoenixIndexImportDirectReducer.java +++ b/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/index/PhoenixIndexImportDirectReducer.java @@ -35,8 +35,10 @@ import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.mapreduce.util.ConnectionUtil; import org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil; +import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.schema.PIndexState; import org.apache.phoenix.schema.PTable; +import org.apache.phoenix.schema.PTableType; import org.apache.phoenix.schema.task.Task; import org.apache.phoenix.schema.transform.Transform; import org.apache.phoenix.util.SchemaUtil; diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AsyncIndexPermissionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AsyncIndexPermissionIT.java new file mode 100644 index 00000000000..eb8f3fab0fc --- /dev/null +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AsyncIndexPermissionIT.java @@ -0,0 +1,340 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.end2end; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.LocalHBaseCluster; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.security.AccessDeniedException; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.security.access.AccessControlClient; +import org.apache.hadoop.hbase.security.access.Permission; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.mapreduce.Job; +import org.apache.phoenix.mapreduce.index.IndexTool; +import org.apache.phoenix.mapreduce.index.PhoenixIndexImportDirectMapper; +import org.apache.phoenix.mapreduce.index.PhoenixServerBuildIndexMapper; +import org.apache.phoenix.query.BaseTest; +import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.schema.PTable; +import org.apache.phoenix.thirdparty.com.google.common.base.Throwables; +import org.apache.phoenix.thirdparty.com.google.common.collect.Maps; +import org.apache.phoenix.util.PhoenixRuntime; +import org.apache.phoenix.util.PropertiesUtil; +import org.apache.phoenix.util.ReadOnlyProps; +import org.apache.phoenix.util.SchemaUtil; +import org.junit.*; +import org.junit.experimental.categories.Category; + +import java.io.IOException; +import java.lang.reflect.UndeclaredThrowableException; +import java.security.PrivilegedExceptionAction; +import java.sql.*; +import java.sql.Connection; +import java.util.*; +import java.util.concurrent.Callable; + +import static org.apache.phoenix.end2end.BasePermissionsIT.*; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.*; +import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; +import static org.junit.Assert.*; + +@Category(NeedsOwnMiniClusterTest.class) +public class AsyncIndexPermissionIT extends BaseTest{ + +// static HBaseTestingUtility testUtil; + + private static final String SUPER_USER = System.getProperty("user.name"); + + boolean isNamespaceMapped; + + // Super User has all the access + protected static User superUser1 = null; + protected static User superUser2 = null; + + // Regular users are granted and revoked permissions as needed + protected User regularUser1 = null; + protected User regularUser2 = null; + protected User regularUser3 = null; + + public AsyncIndexPermissionIT() throws Exception { + this.isNamespaceMapped = true; + } + + @BeforeClass + public static synchronized void doSetup() throws Exception { + if (null != utility) { + utility.shutdownMiniCluster(); + utility = null; + } + + enablePhoenixHBaseAuthorization(config, false); + configureNamespacesOnServer(config, true); + configureStatsConfigurations(config); + config.setBoolean(LocalHBaseCluster.ASSIGN_RANDOM_PORTS, true); + + + Map serverProps = Maps.newHashMapWithExpectedSize(2); + serverProps.put(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, Boolean.toString(true)); + + Map clientProps = Maps.newHashMapWithExpectedSize(2); + clientProps.put(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, Boolean.toString(true)); + + utility = new HBaseTestingUtility(config); + + setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()), + new ReadOnlyProps(clientProps.entrySet().iterator())); + + superUser1 = User.createUserForTesting(config, SUPER_USER, new String[0]); + superUser2 = User.createUserForTesting(config, "superUser2", new String[0]); + } + + @Before + public void initUsersAndTables() { + regularUser1 = User.createUserForTesting(config, "regularUser1_" + + generateUniqueName(), new String[0]); + regularUser2 = User.createUserForTesting(config, "regularUser2_" + + generateUniqueName(), new String[0]); + regularUser3 = User.createUserForTesting(config, "regularUser3_" + + generateUniqueName(), new String[0]); + } + + private BasePermissionsIT.AccessTestAction createIndex(final String indexName, final String dataTable, final String columns) throws SQLException { + return new BasePermissionsIT.AccessTestAction() { + @Override + public Object run() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl()); Statement stmt = conn.createStatement();) { + String indexStmtSQL = "CREATE index " + indexName + " on " + dataTable + " (" + columns +")"; + assertFalse(stmt.execute(indexStmtSQL)); + } + return null; + } + }; + } + + public static IndexTool runIndexTool(Configuration conf, boolean useSnapshot, String schemaName, + String dataTableName, String indexTableName, String tenantId, + int expectedStatus, IndexTool.IndexVerifyType verifyType, IndexTool.IndexDisableLoggingType disableLoggingType, + String... additionalArgs) throws Exception { + IndexTool indexingTool = new IndexTool(); + conf.set(QueryServices.TRANSACTIONS_ENABLED, Boolean.TRUE.toString()); + indexingTool.setConf(conf); + final String[] cmdArgs = IndexToolIT.getArgValues(useSnapshot, schemaName, dataTableName, + indexTableName, tenantId, verifyType, disableLoggingType); + List cmdArgList = new ArrayList<>(Arrays.asList(cmdArgs)); + cmdArgList.addAll(Arrays.asList(additionalArgs)); + int status = indexingTool.run(cmdArgList.toArray(new String[cmdArgList.size()])); + + if (expectedStatus == 0) { + verifyMapper(indexingTool.getJob(), useSnapshot, schemaName, dataTableName, indexTableName, tenantId); + } + assertEquals(expectedStatus, status); + return indexingTool; + } + + private static void verifyMapper(Job job, boolean useSnapshot, String schemaName, + String dataTableName, String indexTableName, String tenantId) throws Exception { + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + if (tenantId != null) { + props.setProperty(PhoenixRuntime.TENANT_ID_ATTRIB, tenantId); + } + + try (Connection conn = + DriverManager.getConnection(getUrl(), props)) { + PTable indexTable = PhoenixRuntime.getTableNoCache(conn, + SchemaUtil.normalizeFullTableName(SchemaUtil.getTableName(schemaName, indexTableName))); + PTable dataTable = PhoenixRuntime.getTableNoCache(conn, SchemaUtil.normalizeFullTableName(SchemaUtil.getTableName(schemaName, dataTableName))); + boolean transactional = dataTable.isTransactional(); + boolean localIndex = PTable.IndexType.LOCAL.equals(indexTable.getIndexType()); + if ((localIndex || !transactional) && !useSnapshot) { + assertEquals(job.getMapperClass(), PhoenixServerBuildIndexMapper.class); + } else { + assertEquals(job.getMapperClass(), PhoenixIndexImportDirectMapper.class); + } + } + } + + private BasePermissionsIT.AccessTestAction createIndexAsync(final String indexName, final String schema, final String tableName, final String columns, final int status) throws SQLException { + return new BasePermissionsIT.AccessTestAction() { + @Override + public Object run() throws Exception { + final String dataTable = SchemaUtil.getTableName(schema, tableName); + try (Connection conn = DriverManager.getConnection(getUrl()); Statement stmt = conn.createStatement();) { + String indexStmtSQL = "CREATE index " + indexName + " on " + dataTable + " (" + columns +") ASYNC"; + assertFalse(stmt.execute(indexStmtSQL)); + } + try { + IndexToolIT.runIndexTool(false, schema, tableName, indexName, null, status, "-op", "/tmp/regular_User1_dir"); + } catch (Exception ignored) { + // Running the indexTool might fail because of AccessDeniedException + } + return null; + } + }; + } + + @Test(timeout = 80000) + public void testCreateIndex() throws Throwable { + final String schema = generateUniqueName(); + final String tableName = generateUniqueName(); + verifyAllowed(createSchema(schema), superUser1); + grantPermissions(regularUser1.getShortName(), schema, Permission.Action.WRITE, + Permission.Action.READ, Permission.Action.EXEC, Permission.Action.ADMIN); + grantPermissions(regularUser1.getShortName(), "SYSTEM", Permission.Action.WRITE, + Permission.Action.READ, Permission.Action.EXEC); + + Path workDir = new Path("/tmp/regular_User1_dir"); + FileSystem fs = workDir.getFileSystem(config); + + fs.mkdirs(workDir, FsPermission.valueOf("-rwxrwxrwx")); + + fs.setOwner(workDir, regularUser1.getShortName(), ""); + + superUser1.runAs(new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + Admin admin = utility.getAdmin(); + TableDescriptorBuilder tdb = TableDescriptorBuilder.newBuilder(TableName.valueOf(schema + ":" + tableName)); + ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("0")).build(); + tdb.setColumnFamily(cfd); + TableDescriptor td = tdb.build(); + admin.createTable(td); + return null; + } + }); + + + + verifyAllowed(createTable(SchemaUtil.getTableName(schema, tableName), 2), regularUser1); + verifyAllowed(createIndex("ind1", SchemaUtil.getTableName(schema, tableName), "PK"), regularUser1); + + String ind3name = "IND3"; + regularUser1.runAs(createIndexAsync(ind3name, schema, tableName, "PK", 0)); + + validateIndex(ind3name, schema, "a"); + } + + private void validateIndex(String ind3name, String schema, String expectedStatus) throws SQLException { + String sql = "SELECT " + "TABLE_SCHEM,TABLE_NAME,TABLE_TYPE,INDEX_STATE" + " FROM " + SYSTEM_CATALOG_NAME + + " WHERE TABLE_SCHEM = '%s' AND TABLE_NAME = '%s' and TABLE_TYPE = 'i'"; + ResultSet rs = getConnection().createStatement().executeQuery(String.format(sql, schema, ind3name)); + assertTrue(rs.next()); + assertEquals(expectedStatus, rs.getString(4)); + } + + public Connection getConnection() throws SQLException { + return getConnection(null); + } + + public Connection getConnection(String tenantId) throws SQLException { + return DriverManager.getConnection(getUrl(), getClientProperties(tenantId)); + } + + private Properties getClientProperties(String tenantId) { + Properties props = new Properties(); + if(tenantId != null) { + props.setProperty(PhoenixRuntime.TENANT_ID_ATTRIB, tenantId); + } + props.setProperty(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, Boolean.toString(isNamespaceMapped)); + return props; + } + + public BasePermissionsIT.AccessTestAction createSchema(final String schemaName) throws SQLException { + return new BasePermissionsIT.AccessTestAction() { + @Override + public Object run() throws Exception { + if (isNamespaceMapped) { + try (Connection conn = getConnection(); Statement stmt = conn.createStatement();) { + assertFalse(stmt.execute("CREATE SCHEMA " + schemaName)); + } + } + return null; + } + }; + } + + BasePermissionsIT.AccessTestAction createTable(final String tableName, int numRecordsToInsert) throws SQLException { + return new BasePermissionsIT.AccessTestAction() { + @Override + public Object run() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl()); Statement stmt = conn.createStatement();) { + assertFalse(stmt.execute("CREATE TABLE " + tableName + "(pk INTEGER not null primary key, data VARCHAR, val integer)")); + try (PreparedStatement pstmt = conn.prepareStatement("UPSERT INTO " + tableName + " values(?, ?, ?)")) { + for (int i = 0; i < numRecordsToInsert; i++) { + pstmt.setInt(1, i); + pstmt.setString(2, Integer.toString(i)); + pstmt.setInt(3, i); + assertEquals(1, pstmt.executeUpdate()); + } + } + conn.commit(); + } + return null; + } + }; + } + + /** This fails only in case of ADE or empty list for any of the users. */ + public void verifyAllowed(BasePermissionsIT.AccessTestAction action, User... users) throws Exception { + if(users.length == 0) { + throw new Exception("Action needs at least one user to run"); + } + for (User user : users) { + verifyAllowed(user, action); + } + } + + private void verifyAllowed(User user, BasePermissionsIT.AccessTestAction... actions) throws Exception { + for (BasePermissionsIT.AccessTestAction action : actions) { + try { + Object obj = user.runAs(action); + if (obj != null && obj instanceof List) { + List results = (List) obj; + if (results.isEmpty()) { + fail("Empty non null results from action for user '" + user.getShortName() + "'"); + } + } + } catch (AccessDeniedException ade) { + fail("Expected action to pass for user '" + user.getShortName() + "' but was denied"); + } + } + } + + void grantPermissions(String toUser, String namespace, Permission.Action... actions) throws Throwable { + updateACLs(getUtility(), new Callable() { + @Override + public Void call() throws Exception { + try { + AccessControlClient.grant(getUtility().getConnection(), namespace, toUser, actions); + return null; + } catch (Throwable t) { + if (t instanceof Exception) { + throw (Exception) t; + } else { + throw new Exception(t); + } + } + } + }); + } +} diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BasePermissionsIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BasePermissionsIT.java index 53e8cb68947..af544e108f2 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BasePermissionsIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BasePermissionsIT.java @@ -217,8 +217,8 @@ public void initUsersAndTables() { view2TableName = tableName + "_V2"; } - private static void enablePhoenixHBaseAuthorization(Configuration config, - boolean useCustomAccessController) { + static void enablePhoenixHBaseAuthorization(Configuration config, + boolean useCustomAccessController) { config.set("hbase.superuser", SUPER_USER + "," + "superUser2"); config.set("hbase.security.authorization", Boolean.TRUE.toString()); config.set("hbase.security.exec.permission.checks", Boolean.TRUE.toString()); @@ -242,11 +242,11 @@ private static void enablePhoenixHBaseAuthorization(Configuration config, config.set("hbase.regionserver.wal.codec", "org.apache.hadoop.hbase.regionserver.wal.IndexedWALEditCodec"); } - private static void configureNamespacesOnServer(Configuration conf, boolean isNamespaceMapped) { + static void configureNamespacesOnServer(Configuration conf, boolean isNamespaceMapped) { conf.set(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, Boolean.toString(isNamespaceMapped)); } - private static void configureStatsConfigurations(Configuration conf) { + static void configureStatsConfigurations(Configuration conf) { conf.set(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB, Long.toString(20)); conf.set(QueryServices.STATS_UPDATE_FREQ_MS_ATTRIB, Long.toString(5)); conf.set(QueryServices.MAX_SERVER_METADATA_CACHE_TIME_TO_LIVE_MS_ATTRIB, Long.toString(5)); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java index 3291457164f..c8b2d32ac56 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java @@ -847,10 +847,20 @@ private static List getArgList (boolean useSnapshot, String schemaName, } private static List getArgList (boolean useSnapshot, String schemaName, - String dataTable, String indxTable, String tenantId, - IndexTool.IndexVerifyType verifyType, Long startTime, Long endTime, + String dataTable, String indxTable, String tenantId, + IndexTool.IndexVerifyType verifyType, Long startTime, Long endTime, IndexTool.IndexDisableLoggingType disableLoggingType, Long incrementalVerify, boolean useIndexTableAsSource) { + return getArgList(useSnapshot, schemaName, dataTable, indxTable, tenantId,verifyType, startTime, + endTime, disableLoggingType, incrementalVerify, useIndexTableAsSource, "/tmp/" + UUID.randomUUID().toString()); + } + + private static List getArgList (boolean useSnapshot, String schemaName, + String dataTable, String indxTable, String tenantId, + IndexTool.IndexVerifyType verifyType, Long startTime, Long endTime, + IndexTool.IndexDisableLoggingType disableLoggingType, + Long incrementalVerify, boolean useIndexTableAsSource, + String outputPath) { List args = Lists.newArrayList(); if (schemaName != null) { args.add("--schema=" + schemaName); @@ -895,7 +905,7 @@ private static List getArgList (boolean useSnapshot, String schemaName, } args.add("-op"); - args.add("/tmp/" + UUID.randomUUID().toString()); + args.add(outputPath); return args; } @@ -914,6 +924,14 @@ public static String[] getArgValues(boolean useSnapshot, String schemaName, return args.toArray(new String[0]); } + public static String[] getArgValues(boolean useSnapshot, String schemaName, + String dataTable, String indexTable, String tenantId, IndexTool.IndexVerifyType verifyType, + IndexTool.IndexDisableLoggingType disableLoggingType, String outputPath) { + List args = getArgList(useSnapshot, schemaName, dataTable, indexTable, + tenantId, verifyType, null, null, disableLoggingType, null, false, outputPath); + return args.toArray(new String[0]); + } + public static String [] getArgValues(boolean useSnapshot, String schemaName, String dataTable, String indexTable, String tenantId, IndexTool.IndexVerifyType verifyType, Long startTime, Long endTime) { @@ -1010,8 +1028,31 @@ public static IndexTool runIndexTool(Configuration conf, boolean useSnapshot, St IndexTool indexingTool = new IndexTool(); conf.set(QueryServices.TRANSACTIONS_ENABLED, Boolean.TRUE.toString()); indexingTool.setConf(conf); - final String[] cmdArgs = getArgValues(useSnapshot, schemaName, dataTableName, - indexTableName, tenantId, verifyType, disableLoggingType); + boolean additionalArgsContainPath = false; + String path = ""; + List newadditionalArgs = Lists.newArrayList(); + for (String arg : additionalArgs){ + if (additionalArgsContainPath == true) { + path = arg; + } + else if ( arg.equals("-op") || arg.equals("-output-path")) { + additionalArgsContainPath = true; + } + else { + newadditionalArgs.add(arg); + } + } + additionalArgs = newadditionalArgs.toArray(new String[0]); + + String[] cmdArgs; + if (additionalArgsContainPath) { + cmdArgs = getArgValues(useSnapshot, schemaName, dataTableName, + indexTableName, tenantId, verifyType, disableLoggingType, path); + } + else { + cmdArgs = getArgValues(useSnapshot, schemaName, dataTableName, + indexTableName, tenantId, verifyType, disableLoggingType); + } List cmdArgList = new ArrayList<>(Arrays.asList(cmdArgs)); cmdArgList.addAll(Arrays.asList(additionalArgs)); LOGGER.info("Running IndexTool with {}", Arrays.toString(cmdArgList.toArray()), new Exception("Stack Trace")); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/LoadSystemTableSnapshotBase.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/LoadSystemTableSnapshotBase.java index 003ed5ebc05..aae4c520ab6 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/LoadSystemTableSnapshotBase.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/LoadSystemTableSnapshotBase.java @@ -74,22 +74,13 @@ public abstract class LoadSystemTableSnapshotBase extends BaseTest { private static final Logger LOGGER = LoggerFactory.getLogger( LoadSystemTableSnapshotBase.class); - public static final String SNAPSHOT_DIR = "snapshots4_7/"; + public static String SNAPSHOT_DIR; public static String rootDir; - private static final HashMap SNAPSHOTS_TO_LOAD; + private static HashMap SNAPSHOTS_TO_LOAD; public static final byte[] MUTEX_LOCKED = "MUTEX_LOCKED".getBytes(StandardCharsets.UTF_8); - static { - SNAPSHOTS_TO_LOAD = new HashMap<>(); - //Add any HBase tables, including Phoenix System tables - - SNAPSHOTS_TO_LOAD.put("SYSTEM.CATALOG_SNAPSHOT", "SYSTEM.CATALOG"); - SNAPSHOTS_TO_LOAD.put("SYSTEM.FUNCTION_SNAPSHOT", "SYSTEM.FUNCTION"); - SNAPSHOTS_TO_LOAD.put("SYSTEM.SEQUENCE_SNAPSHOT", "SYSTEM.SEQUENCE"); - SNAPSHOTS_TO_LOAD.put("SYSTEM.STATS_SNAPSHOT", "SYSTEM.STATS"); - } private static void decompress(String in, File out) throws IOException { try (TarArchiveInputStream fin = new TarArchiveInputStream(new FileInputStream(in))){ @@ -109,11 +100,24 @@ private static void decompress(String in, File out) throws IOException { } public static synchronized void setupCluster(boolean createBlockUpgradeMutex) throws Exception { + //Add any HBase tables, including Phoenix System tables + HashMap snapshotsToLoad = new HashMap<>(); + snapshotsToLoad.put("SYSTEM.CATALOG_SNAPSHOT", "SYSTEM.CATALOG"); + snapshotsToLoad.put("SYSTEM.FUNCTION_SNAPSHOT", "SYSTEM.FUNCTION"); + snapshotsToLoad.put("SYSTEM.SEQUENCE_SNAPSHOT", "SYSTEM.SEQUENCE"); + snapshotsToLoad.put("SYSTEM.STATS_SNAPSHOT", "SYSTEM.STATS"); + + setupCluster(createBlockUpgradeMutex, "snapshots47.tar.gz", "snapshots4_7/", snapshotsToLoad, "true"); + } + + public static synchronized void setupCluster(boolean createBlockUpgradeMutex, String tarName, String snapshotDir, HashMap snapshotsToLoad, String nameSpaceMapping) throws Exception { + SNAPSHOT_DIR = snapshotDir; + SNAPSHOTS_TO_LOAD = snapshotsToLoad; Map serverProps = Maps.newHashMapWithExpectedSize(2); serverProps.put(QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB, QueryServicesOptions.DEFAULT_EXTRA_JDBC_ARGUMENTS); - serverProps.put(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, "true"); + serverProps.put(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, nameSpaceMapping); Map clientProps = Maps.newHashMapWithExpectedSize(2); - clientProps.put(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, "true"); + clientProps.put(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, nameSpaceMapping); //Start minicluster without Phoenix first checkClusterInitialized(new ReadOnlyProps(serverProps.entrySet().iterator())); @@ -122,7 +126,7 @@ public static synchronized void setupCluster(boolean createBlockUpgradeMutex) th .getResource(SNAPSHOT_DIR); // extract the tar - File archive = new File(folderUrl.getFile() + "snapshots47.tar.gz"); + File archive = new File(folderUrl.getFile() + tarName); File destination = new File(folderUrl.getFile()); decompress(archive.toString(), destination); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java index f9ceccfe653..3bd05e9d7f6 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java @@ -64,6 +64,11 @@ @Category(NeedsOwnMiniClusterTest.class) public class MigrateSystemTablesToSystemNamespaceIT extends BaseTest { + private static final Set HBASE_SYSTEM_TABLES = new HashSet<>(Arrays.asList( + "SYSTEM.PHOENIX_INDEX_TOOL_RESULT", "SYSTEM.PHOENIX_INDEX_TOOL")); + private static final Set HBASE_NAMESPACE_MAPPED_SYSTEM_TABLES = new HashSet<>( + Arrays.asList("SYSTEM:PHOENIX_INDEX_TOOL_RESULT", "SYSTEM:PHOENIX_INDEX_TOOL")); + private static final Set PHOENIX_SYSTEM_TABLES = new HashSet<>(Arrays.asList( "SYSTEM.CATALOG", "SYSTEM.SEQUENCE", "SYSTEM.STATS", "SYSTEM.FUNCTION", "SYSTEM.MUTEX","SYSTEM.LOG", "SYSTEM.CHILD_LINK", "SYSTEM.TASK", "SYSTEM.TRANSFORM", @@ -191,7 +196,7 @@ public Void run() throws Exception { }); hbaseTables = getHBaseTables(); - assertTrue(hbaseTables.size() == PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES.size()); + assertTrue(hbaseTables.size() == PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES.size() + HBASE_NAMESPACE_MAPPED_SYSTEM_TABLES.size()); assertTrue(hbaseTables.containsAll(PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES)); try { @@ -208,7 +213,7 @@ public Void run() throws Exception { } hbaseTables = getHBaseTables(); - assertTrue(hbaseTables.size() == PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES.size()); + assertTrue(hbaseTables.size() == PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES.size() + HBASE_NAMESPACE_MAPPED_SYSTEM_TABLES.size()); assertTrue(hbaseTables.containsAll(PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES)); } @@ -226,7 +231,7 @@ public Void run() throws Exception { }); hbaseTables = getHBaseTables(); - assertTrue(hbaseTables.size() == PHOENIX_SYSTEM_TABLES.size()); + assertTrue(hbaseTables.size() == PHOENIX_SYSTEM_TABLES.size() + HBASE_SYSTEM_TABLES.size()); assertTrue(hbaseTables.containsAll(PHOENIX_SYSTEM_TABLES)); user2.doAs(new PrivilegedExceptionAction() { @@ -239,7 +244,7 @@ public Void run() throws Exception { }); hbaseTables = getHBaseTables(); - assertTrue(hbaseTables.size() == PHOENIX_SYSTEM_TABLES.size()); + assertTrue(hbaseTables.size() == PHOENIX_SYSTEM_TABLES.size() + HBASE_SYSTEM_TABLES.size()); assertTrue(hbaseTables.containsAll(PHOENIX_SYSTEM_TABLES)); try { @@ -258,7 +263,7 @@ public Void run() throws Exception { } hbaseTables = getHBaseTables(); - assertTrue(hbaseTables.size() == PHOENIX_SYSTEM_TABLES.size()); + assertTrue(hbaseTables.size() == PHOENIX_SYSTEM_TABLES.size() + HBASE_SYSTEM_TABLES.size()); assertTrue(hbaseTables.containsAll(PHOENIX_SYSTEM_TABLES)); user2.doAs(new PrivilegedExceptionAction() { @@ -271,7 +276,7 @@ public Void run() throws Exception { }); hbaseTables = getHBaseTables(); - assertTrue(hbaseTables.size() == PHOENIX_SYSTEM_TABLES.size()); + assertTrue(hbaseTables.size() == PHOENIX_SYSTEM_TABLES.size() + HBASE_SYSTEM_TABLES.size()); assertTrue(hbaseTables.containsAll(PHOENIX_SYSTEM_TABLES)); user3.doAs(new PrivilegedExceptionAction() { @@ -283,7 +288,7 @@ public Void run() throws Exception { }); hbaseTables = getHBaseTables(); - assertTrue(hbaseTables.size() == PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES.size()); + assertTrue(hbaseTables.size() == PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES.size() + HBASE_NAMESPACE_MAPPED_SYSTEM_TABLES.size()); assertTrue(hbaseTables.containsAll(PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES)); } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemTablesCreationOnConnectionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemTablesCreationOnConnectionIT.java index 84074c349a7..f007a4392a5 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemTablesCreationOnConnectionIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemTablesCreationOnConnectionIT.java @@ -112,12 +112,12 @@ public class SystemTablesCreationOnConnectionIT { private static final Set PHOENIX_SYSTEM_TABLES = new HashSet<>(Arrays.asList( "SYSTEM.CATALOG", "SYSTEM.SEQUENCE", "SYSTEM.STATS", "SYSTEM.FUNCTION", "SYSTEM.MUTEX", "SYSTEM.LOG", "SYSTEM.CHILD_LINK", "SYSTEM.TASK","SYSTEM.TRANSFORM", - "SYSTEM.CDC_STREAM_STATUS", "SYSTEM.CDC_STREAM")); + "SYSTEM.CDC_STREAM_STATUS", "SYSTEM.CDC_STREAM", "SYSTEM.PHOENIX_INDEX_TOOL_RESULT", "SYSTEM.PHOENIX_INDEX_TOOL")); private static final Set PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES = new HashSet<>( Arrays.asList("SYSTEM:CATALOG", "SYSTEM:SEQUENCE", "SYSTEM:STATS", "SYSTEM:FUNCTION", "SYSTEM:MUTEX", "SYSTEM:LOG", "SYSTEM:CHILD_LINK", "SYSTEM:TASK", "SYSTEM:TRANSFORM", - "SYSTEM:CDC_STREAM_STATUS", "SYSTEM:CDC_STREAM")); + "SYSTEM:CDC_STREAM_STATUS", "SYSTEM:CDC_STREAM", "SYSTEM:PHOENIX_INDEX_TOOL_RESULT", "SYSTEM:PHOENIX_INDEX_TOOL")); private static class PhoenixSysCatCreationServices extends ConnectionQueryServicesImpl { @@ -357,6 +357,7 @@ public void testMigrateToSystemNamespaceAndUpgradeSysCat() throws Exception { assertEquals(PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES, hbaseTables); assertEquals(1, countUpgradeAttempts); assertEquals(1, actualSysCatUpgrades); + // todo this still fails } // Conditions: server-side namespace mapping is enabled, the first connection to the server will diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIndexToolTablesIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIndexToolTablesIT.java new file mode 100644 index 00000000000..f68f827a504 --- /dev/null +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIndexToolTablesIT.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.end2end; + + +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; + +import org.apache.phoenix.mapreduce.index.IndexToolTableUtil; +import org.apache.phoenix.query.QueryConstants; +import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.query.QueryServicesOptions; +import org.apache.phoenix.thirdparty.com.google.common.collect.Maps; +import org.apache.phoenix.util.ReadOnlyProps; +import org.apache.phoenix.util.SchemaUtil; +import org.junit.Before; +import org.junit.Test; +import java.util.HashMap; +import java.util.Map; + +import static org.apache.phoenix.mapreduce.index.IndexToolTableUtil.RESULT_TABLE_NAME; +import static org.apache.phoenix.query.QueryConstants.SYSTEM_SCHEMA_NAME; +import static org.junit.Assert.*; + +public class UpgradeIndexToolTablesIT extends LoadSystemTableSnapshotBase { + protected String nameSpaceMapping = "true"; + + @Before + public synchronized void doSetup() throws Exception { + setupCluster(nameSpaceMapping); + } + + public synchronized void setupCluster(String nameSpaceMappingEnabled) throws Exception { + HashMap snapshotsToLoad = new HashMap<>(); + snapshotsToLoad.put("phoenixtoolresultsnapshot", "PHOENIX_INDEX_TOOL_RESULT"); + setupCluster(false, "indexToolsnapshot.tar.gz", "indexToolResultSnapshot/", snapshotsToLoad, nameSpaceMappingEnabled); + } + + @Test + public void testPhoenixUpgradeIndexToolTables() throws Exception { + try (Admin admin = utility.getAdmin()) { + // we load the RESULT_TABLE_NAME from snapshot + assertTrue(admin.tableExists(TableName.valueOf(IndexToolTableUtil.RESULT_TABLE_NAME))); + assertFalse(admin.tableExists(TableName.valueOf(IndexToolTableUtil.SYSTEM_RESULT_TABLE_NAME))); + // we don't load the OUTPUT_TABLE_NAME + assertFalse(admin.tableExists(TableName.valueOf(IndexToolTableUtil.OUTPUT_TABLE_NAME))); + assertFalse(admin.tableExists(TableName.valueOf(IndexToolTableUtil.SYSTEM_OUTPUT_TABLE_NAME))); + } + + Map serverProps = Maps.newHashMapWithExpectedSize(2); + serverProps.put(QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB, QueryServicesOptions.DEFAULT_EXTRA_JDBC_ARGUMENTS); + serverProps.put(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, nameSpaceMapping); + Map clientProps = Maps.newHashMapWithExpectedSize(2); + clientProps.put(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, nameSpaceMapping); + + + //Now we can start Phoenix + setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()), new ReadOnlyProps(clientProps.entrySet() + .iterator())); + assertTrue(true); + + + // Check the IndexTool Tables after upgrade + try (Admin admin = utility.getAdmin()) { + assertFalse(admin.tableExists(TableName.valueOf(IndexToolTableUtil.OUTPUT_TABLE_NAME))); + assertFalse(admin.tableExists(TableName.valueOf(IndexToolTableUtil.RESULT_TABLE_NAME))); + assertTrue(admin.tableExists(TableName.valueOf(IndexToolTableUtil.SYSTEM_OUTPUT_TABLE_NAME))); + assertTrue(admin.tableExists(TableName.valueOf(IndexToolTableUtil.SYSTEM_RESULT_TABLE_NAME))); + } + + String tableName = SchemaUtil.getTableName(SYSTEM_SCHEMA_NAME, RESULT_TABLE_NAME); + if (nameSpaceMapping.equals("true")) { + assertEquals(IndexToolTableUtil.SYSTEM_RESULT_TABLE_NAME, tableName.replace(QueryConstants.NAME_SEPARATOR, + QueryConstants.NAMESPACE_SEPARATOR)); + } else { + assertEquals(IndexToolTableUtil.SYSTEM_RESULT_TABLE_NAME, tableName); + } + + } + +} diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIndexToolTablesNameSpaceMappingDisabledIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIndexToolTablesNameSpaceMappingDisabledIT.java new file mode 100644 index 00000000000..f339550d619 --- /dev/null +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIndexToolTablesNameSpaceMappingDisabledIT.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.end2end; + +import org.junit.Before; + + +public class UpgradeIndexToolTablesNameSpaceMappingDisabledIT extends UpgradeIndexToolTablesIT { + + @Override + @Before + public synchronized void doSetup() throws Exception { + nameSpaceMapping = "false"; + setupCluster(nameSpaceMapping); + } +} diff --git a/phoenix-core/src/it/resources/indexToolResultSnapshot/indexToolsnapshot.tar.gz b/phoenix-core/src/it/resources/indexToolResultSnapshot/indexToolsnapshot.tar.gz new file mode 100644 index 00000000000..9d757462764 Binary files /dev/null and b/phoenix-core/src/it/resources/indexToolResultSnapshot/indexToolsnapshot.tar.gz differ