diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/AppFabricServiceRuntimeModule.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/AppFabricServiceRuntimeModule.java index a6f666e95c17..c32d44323d09 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/AppFabricServiceRuntimeModule.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/AppFabricServiceRuntimeModule.java @@ -151,6 +151,7 @@ import io.cdap.cdap.scheduler.CoreSchedulerService; import io.cdap.cdap.scheduler.Scheduler; import io.cdap.cdap.securestore.spi.SecretStore; +import io.cdap.cdap.security.encryption.guice.DataStorageAeadEncryptionModule; import io.cdap.cdap.security.impersonation.DefaultOwnerAdmin; import io.cdap.cdap.security.impersonation.DefaultUGIProvider; import io.cdap.cdap.security.impersonation.OwnerAdmin; @@ -200,6 +201,7 @@ public Module getInMemoryModules() { new EntityVerifierModule(), new MasterCredentialProviderModule(), new OperationModule(), + new DataStorageAeadEncryptionModule(), BootstrapModules.getInMemoryModule(), new AbstractModule() { @Override @@ -243,6 +245,7 @@ public Module getStandaloneModules() { new ProvisionerModule(), new MasterCredentialProviderModule(), new OperationModule(), + new DataStorageAeadEncryptionModule(), BootstrapModules.getFileBasedModule(), new AbstractModule() { @Override @@ -298,6 +301,7 @@ public Module getDistributedModules() { new ProvisionerModule(), new MasterCredentialProviderModule(), new OperationModule(), + new DataStorageAeadEncryptionModule(), BootstrapModules.getFileBasedModule(), new AbstractModule() { @Override diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/AppFabricServer.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/AppFabricServer.java index ef489954bcd3..81a187cd0428 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/AppFabricServer.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/AppFabricServer.java @@ -122,7 +122,8 @@ public AppFabricServer(CConfiguration cConf, SConfiguration sConf, @Named("appfabric.handler.hooks") Set handlerHookNames, CoreSchedulerService coreSchedulerService, CredentialProviderService credentialProviderService, - NamespaceCredentialProviderService namespaceCredentialProviderService, ProvisioningService provisioningService, + NamespaceCredentialProviderService namespaceCredentialProviderService, + ProvisioningService provisioningService, BootstrapService bootstrapService, SystemAppManagementService systemAppManagementService, TransactionRunner transactionRunner, diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/CredentialIdentityManager.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/CredentialIdentityManager.java index 863370235cd6..589ee2870c3e 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/CredentialIdentityManager.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/CredentialIdentityManager.java @@ -18,11 +18,10 @@ import io.cdap.cdap.common.AlreadyExistsException; import io.cdap.cdap.common.NotFoundException; -import io.cdap.cdap.internal.credential.store.CredentialIdentityStore; -import io.cdap.cdap.internal.credential.store.CredentialProfileStore; import io.cdap.cdap.proto.credential.CredentialIdentity; import io.cdap.cdap.proto.id.CredentialIdentityId; import io.cdap.cdap.proto.id.CredentialProfileId; +import io.cdap.cdap.security.spi.encryption.CipherException; import io.cdap.cdap.spi.data.StructuredTableContext; import io.cdap.cdap.spi.data.transaction.TransactionRunner; import io.cdap.cdap.spi.data.transaction.TransactionRunners; @@ -70,7 +69,11 @@ public Collection list(String namespace) throws IOExceptio */ public Optional get(CredentialIdentityId id) throws IOException { return TransactionRunners.run(transactionRunner, context -> { - return identityStore.get(context, id); + try { + return identityStore.get(context, id); + } catch (CipherException e) { + throw new IOException("Failed to decrypt identity", e); + } }, IOException.class); } @@ -86,7 +89,7 @@ public Optional get(CredentialIdentityId id) throws IOExcept public void create(CredentialIdentityId id, CredentialIdentity identity) throws AlreadyExistsException, IOException, NotFoundException { TransactionRunners.run(transactionRunner, context -> { - if (identityStore.get(context, id).isPresent()) { + if (identityStore.exists(context, id)) { throw new AlreadyExistsException(String.format("Credential identity '%s:%s' already exists", id.getNamespace(), id.getName())); } @@ -106,7 +109,7 @@ public void create(CredentialIdentityId id, CredentialIdentity identity) public void update(CredentialIdentityId id, CredentialIdentity identity) throws IOException, NotFoundException { TransactionRunners.run(transactionRunner, context -> { - if (!identityStore.get(context, id).isPresent()) { + if (!identityStore.exists(context, id)) { throw new NotFoundException(String.format("Credential identity '%s:%s' not found", id.getNamespace(), id.getName())); } @@ -123,7 +126,7 @@ public void update(CredentialIdentityId id, CredentialIdentity identity) */ public void delete(CredentialIdentityId id) throws IOException, NotFoundException { TransactionRunners.run(transactionRunner, context -> { - if (!identityStore.get(context, id).isPresent()) { + if (!identityStore.exists(context, id)) { throw new NotFoundException(String.format("Credential identity '%s:%s' not found", id.getNamespace(), id.getName())); } @@ -136,10 +139,14 @@ private void validateAndWriteIdentity(StructuredTableContext context, Credential // Validate the referenced profile exists. CredentialProfileId profileId = new CredentialProfileId(identity.getProfileNamespace(), identity.getProfileName()); - if (!profileStore.get(context, profileId).isPresent()) { + if (!profileStore.exists(context, profileId)) { throw new NotFoundException(String.format("Credential profile '%s:%s' not found", profileId.getNamespace(), profileId.getName())); } - identityStore.write(context, id, identity); + try { + identityStore.write(context, id, identity); + } catch (CipherException e) { + throw new IOException("Failed to encrypt identity", e); + } } } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/store/CredentialIdentityStore.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/CredentialIdentityStore.java similarity index 75% rename from cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/store/CredentialIdentityStore.java rename to cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/CredentialIdentityStore.java index 3ca48f0ff1e8..bda3eab7af9e 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/store/CredentialIdentityStore.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/CredentialIdentityStore.java @@ -14,13 +14,16 @@ * the License. */ -package io.cdap.cdap.internal.credential.store; +package io.cdap.cdap.internal.credential; import com.google.gson.Gson; import io.cdap.cdap.api.dataset.lib.CloseableIterator; import io.cdap.cdap.proto.credential.CredentialIdentity; import io.cdap.cdap.proto.id.CredentialIdentityId; import io.cdap.cdap.proto.id.CredentialProfileId; +import io.cdap.cdap.security.encryption.AeadCipher; +import io.cdap.cdap.security.encryption.guice.DataStorageAeadEncryptionModule; +import io.cdap.cdap.security.spi.encryption.CipherException; import io.cdap.cdap.spi.data.StructuredRow; import io.cdap.cdap.spi.data.StructuredTable; import io.cdap.cdap.spi.data.StructuredTableContext; @@ -29,6 +32,7 @@ import io.cdap.cdap.spi.data.table.field.Range; import io.cdap.cdap.store.StoreDefinition.CredentialProviderStore; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -38,14 +42,30 @@ import java.util.Spliterators; import java.util.stream.Collectors; import java.util.stream.StreamSupport; +import javax.inject.Inject; +import javax.inject.Named; /** * Storage for credential identities. */ public class CredentialIdentityStore { + /** + * DO NOT CHANGE THIS VALUE! CHANGING THIS VALUE IS BACKWARDS-INCOMPATIBLE. Values encrypted using + * a different value will not decrypt properly! + */ + private static final byte[] CREDENTIAL_IDENTITY_STORE_AD = "CredentialIdentityStore" + .getBytes(StandardCharsets.UTF_8); private static final Gson GSON = new Gson(); + private final AeadCipher dataStorageCipher; + + @Inject + CredentialIdentityStore(@Named(DataStorageAeadEncryptionModule.DATA_STORAGE_ENCRYPTION) + AeadCipher dataStorageCipher) { + this.dataStorageCipher = dataStorageCipher; + } + /** * Lists entries in the credential identity table for a given namespace. * @@ -84,6 +104,18 @@ public Collection listForProfile(StructuredTableContext co } } + /** + * Returns whether an entry exists in the identity table. + * + * @param context The transaction context to use. + * @param id The identity reference to fetch. + * @return Whether the credential identity exists. + */ + public boolean exists(StructuredTableContext context, CredentialIdentityId id) + throws IOException { + return readIdentity(context, id).isPresent(); + } + /** * Fetch an entry from the identity table. * @@ -93,15 +125,13 @@ public Collection listForProfile(StructuredTableContext co * @throws IOException If any failure reading from storage occurs. */ public Optional get(StructuredTableContext context, CredentialIdentityId id) - throws IOException { - StructuredTable table = context.getTable(CredentialProviderStore.CREDENTIAL_IDENTITIES); - Collection> key = Arrays.asList( - Fields.stringField(CredentialProviderStore.NAMESPACE_FIELD, - id.getNamespace()), - Fields.stringField(CredentialProviderStore.IDENTITY_NAME_FIELD, - id.getName())); - return table.read(key).map(row -> GSON.fromJson(row - .getString(CredentialProviderStore.IDENTITY_DATA_FIELD), CredentialIdentity.class)); + throws CipherException, IOException { + return readIdentity(context, id) + .map(row -> row.getBytes(CredentialProviderStore.IDENTITY_DATA_FIELD)) + .map(encryptedIdentity -> dataStorageCipher + .decrypt(encryptedIdentity, CREDENTIAL_IDENTITY_STORE_AD)) + .map(decrypted -> new String(decrypted, StandardCharsets.UTF_8)) + .map(decryptedStr -> GSON.fromJson(decryptedStr, CredentialIdentity.class)); } /** @@ -113,7 +143,7 @@ public Optional get(StructuredTableContext context, Credenti * @throws IOException If any failure reading from storage occurs. */ public void write(StructuredTableContext context, CredentialIdentityId id, - CredentialIdentity identity) throws IOException { + CredentialIdentity identity) throws CipherException, IOException { StructuredTable identityTable = context.getTable(CredentialProviderStore.CREDENTIAL_IDENTITIES); Collection> row = Arrays.asList( @@ -121,8 +151,9 @@ public void write(StructuredTableContext context, CredentialIdentityId id, id.getNamespace()), Fields.stringField(CredentialProviderStore.IDENTITY_NAME_FIELD, id.getName()), - Fields.stringField(CredentialProviderStore.IDENTITY_DATA_FIELD, - GSON.toJson(identity)), + Fields.bytesField(CredentialProviderStore.IDENTITY_DATA_FIELD, + dataStorageCipher.encrypt(GSON.toJson(identity).getBytes(StandardCharsets.UTF_8), + CREDENTIAL_IDENTITY_STORE_AD)), Fields.stringField(CredentialProviderStore.IDENTITY_PROFILE_INDEX_FIELD, toProfileIndex(identity.getProfileNamespace(), identity.getProfileName()))); identityTable.upsert(row); @@ -159,4 +190,15 @@ private static Collection identitiesFromRowIterator( private static String toProfileIndex(String profileNamespace, String profileName) { return String.format("%s:%s", profileNamespace, profileName); } + + private Optional readIdentity(StructuredTableContext context, + CredentialIdentityId id) throws IOException { + StructuredTable table = context.getTable(CredentialProviderStore.CREDENTIAL_IDENTITIES); + Collection> key = Arrays.asList( + Fields.stringField(CredentialProviderStore.NAMESPACE_FIELD, + id.getNamespace()), + Fields.stringField(CredentialProviderStore.IDENTITY_NAME_FIELD, + id.getName())); + return table.read(key); + } } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/CredentialProfileManager.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/CredentialProfileManager.java index 5681076f36f2..e00f780ba776 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/CredentialProfileManager.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/CredentialProfileManager.java @@ -21,13 +21,12 @@ import io.cdap.cdap.common.BadRequestException; import io.cdap.cdap.common.ConflictException; import io.cdap.cdap.common.NotFoundException; -import io.cdap.cdap.internal.credential.store.CredentialIdentityStore; -import io.cdap.cdap.internal.credential.store.CredentialProfileStore; import io.cdap.cdap.proto.credential.CredentialProfile; import io.cdap.cdap.proto.id.CredentialIdentityId; import io.cdap.cdap.proto.id.CredentialProfileId; import io.cdap.cdap.security.spi.credential.CredentialProvider; import io.cdap.cdap.security.spi.credential.ProfileValidationException; +import io.cdap.cdap.security.spi.encryption.CipherException; import io.cdap.cdap.spi.data.transaction.TransactionRunner; import io.cdap.cdap.spi.data.transaction.TransactionRunners; import java.io.IOException; @@ -80,7 +79,11 @@ public Collection list(String namespace) throws IOException */ public Optional get(CredentialProfileId id) throws IOException { return TransactionRunners.run(transactionRunner, context -> { - return profileStore.get(context, id); + try { + return profileStore.get(context, id); + } catch (CipherException e) { + throw new IOException("Failed to decrypt profile", e); + } }, IOException.class); } @@ -97,11 +100,15 @@ public void create(CredentialProfileId id, CredentialProfile profile) throws AlreadyExistsException, BadRequestException, IOException { validateProfile(profile); TransactionRunners.run(transactionRunner, context -> { - if (profileStore.get(context, id).isPresent()) { + if (profileStore.exists(context, id)) { throw new AlreadyExistsException(String.format("Credential profile '%s:%s' already exists", id.getNamespace(), id.getName())); } - profileStore.write(context, id, profile); + try { + profileStore.write(context, id, profile); + } catch (CipherException e) { + throw new IOException("Failed to encrypt profile", e); + } }, AlreadyExistsException.class, IOException.class); } @@ -118,11 +125,15 @@ public void update(CredentialProfileId id, CredentialProfile profile) throws BadRequestException, IOException, NotFoundException { validateProfile(profile); TransactionRunners.run(transactionRunner, context -> { - if (!profileStore.get(context, id).isPresent()) { + if (!profileStore.exists(context, id)) { throw new NotFoundException(String.format("Credential profile '%s:%s' not found", id.getNamespace(), id.getName())); } - profileStore.write(context, id, profile); + try { + profileStore.write(context, id, profile); + } catch (CipherException e) { + throw new IOException("Failed to encrypt profile", e); + } }, IOException.class, NotFoundException.class); } @@ -137,7 +148,7 @@ public void update(CredentialProfileId id, CredentialProfile profile) public void delete(CredentialProfileId id) throws ConflictException, IOException, NotFoundException { TransactionRunners.run(transactionRunner, context -> { - if (!profileStore.get(context, id).isPresent()) { + if (!profileStore.exists(context, id)) { throw new NotFoundException(String.format("Credential profile '%s:%s' not found", id.getNamespace(), id.getName())); } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/store/CredentialProfileStore.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/CredentialProfileStore.java similarity index 70% rename from cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/store/CredentialProfileStore.java rename to cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/CredentialProfileStore.java index fbb76048c000..7b83fd9dc929 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/store/CredentialProfileStore.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/CredentialProfileStore.java @@ -14,13 +14,16 @@ * the License. */ -package io.cdap.cdap.internal.credential.store; +package io.cdap.cdap.internal.credential; import com.google.gson.Gson; import io.cdap.cdap.api.dataset.lib.CloseableIterator; import io.cdap.cdap.common.NotFoundException; import io.cdap.cdap.proto.credential.CredentialProfile; import io.cdap.cdap.proto.id.CredentialProfileId; +import io.cdap.cdap.security.encryption.AeadCipher; +import io.cdap.cdap.security.encryption.guice.DataStorageAeadEncryptionModule; +import io.cdap.cdap.security.spi.encryption.CipherException; import io.cdap.cdap.spi.data.StructuredRow; import io.cdap.cdap.spi.data.StructuredTable; import io.cdap.cdap.spi.data.StructuredTableContext; @@ -29,6 +32,7 @@ import io.cdap.cdap.spi.data.table.field.Range; import io.cdap.cdap.store.StoreDefinition.CredentialProviderStore; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -38,14 +42,30 @@ import java.util.Spliterators; import java.util.stream.Collectors; import java.util.stream.StreamSupport; +import javax.inject.Inject; +import javax.inject.Named; /** * Storage for credential identities. */ public class CredentialProfileStore { + /** + * DO NOT CHANGE THIS VALUE! CHANGING THIS VALUE IS BACKWARDS-INCOMPATIBLE. Values encrypted using + * a different value will not decrypt properly! + */ + private static final byte[] CREDENTIAL_PROFILE_STORE_AD = "CredentialProviderStore" + .getBytes(StandardCharsets.UTF_8); private static final Gson GSON = new Gson(); + private final AeadCipher dataStorageCipher; + + @Inject + CredentialProfileStore(@Named(DataStorageAeadEncryptionModule.DATA_STORAGE_ENCRYPTION) + AeadCipher dataStorageCipher) { + this.dataStorageCipher = dataStorageCipher; + } + /** * Lists entries in the credential profile table for a given namespace. * @@ -65,6 +85,17 @@ public Collection list(StructuredTableContext context, Stri } } + /** + * Returns whether an entry exists in the profile table. + * + * @param context The transaction context to use. + * @param id The profile reference to fetch. + * @return Whether the credential profile exists. + */ + public boolean exists(StructuredTableContext context, CredentialProfileId id) throws IOException { + return readProfile(context, id).isPresent(); + } + /** * Fetch an entry from the profile table. * @@ -74,15 +105,12 @@ public Collection list(StructuredTableContext context, Stri * @throws IOException If any failure reading from storage occurs. */ public Optional get(StructuredTableContext context, CredentialProfileId id) - throws IOException { - StructuredTable table = context.getTable(CredentialProviderStore.CREDENTIAL_PROFILES); - Collection> key = Arrays.asList( - Fields.stringField(CredentialProviderStore.NAMESPACE_FIELD, - id.getNamespace()), - Fields.stringField(CredentialProviderStore.PROFILE_NAME_FIELD, - id.getName())); - return table.read(key).map(row -> GSON.fromJson(row - .getString(CredentialProviderStore.PROFILE_DATA_FIELD), CredentialProfile.class)); + throws CipherException, IOException { + return readProfile(context, id) + .map(row -> row.getBytes(CredentialProviderStore.PROFILE_DATA_FIELD)) + .map(encryptedData -> dataStorageCipher.decrypt(encryptedData, CREDENTIAL_PROFILE_STORE_AD)) + .map(decrypted -> new String(decrypted, StandardCharsets.UTF_8)) + .map(decryptedStr -> GSON.fromJson(decryptedStr, CredentialProfile.class)); } /** @@ -94,15 +122,16 @@ public Optional get(StructuredTableContext context, Credentia * @throws IOException If any failure reading from storage occurs. */ public void write(StructuredTableContext context, CredentialProfileId id, - CredentialProfile profile) throws IOException { + CredentialProfile profile) throws CipherException, IOException { StructuredTable table = context.getTable(CredentialProviderStore.CREDENTIAL_PROFILES); Collection> row = Arrays.asList( Fields.stringField(CredentialProviderStore.NAMESPACE_FIELD, id.getNamespace()), Fields.stringField(CredentialProviderStore.PROFILE_NAME_FIELD, id.getName()), - Fields.stringField(CredentialProviderStore.PROFILE_DATA_FIELD, - GSON.toJson(profile))); + Fields.bytesField(CredentialProviderStore.PROFILE_DATA_FIELD, + dataStorageCipher.encrypt(GSON.toJson(profile).getBytes(StandardCharsets.UTF_8), + CREDENTIAL_PROFILE_STORE_AD))); table.upsert(row); } @@ -133,4 +162,15 @@ private static Collection profilesFromRowIterator( row.getString(CredentialProviderStore.PROFILE_NAME_FIELD))) .collect(Collectors.toList()); } + + private Optional readProfile(StructuredTableContext context, + CredentialProfileId id) throws IOException { + StructuredTable table = context.getTable(CredentialProviderStore.CREDENTIAL_PROFILES); + Collection> key = Arrays.asList( + Fields.stringField(CredentialProviderStore.NAMESPACE_FIELD, + id.getNamespace()), + Fields.stringField(CredentialProviderStore.PROFILE_NAME_FIELD, + id.getName())); + return table.read(key); + } } diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/credential/CredentialProfileManagerTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/credential/CredentialProfileManagerTest.java index 59c4a1ef01b0..1fcb0c6000cf 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/credential/CredentialProfileManagerTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/credential/CredentialProfileManagerTest.java @@ -20,7 +20,6 @@ import io.cdap.cdap.common.BadRequestException; import io.cdap.cdap.common.ConflictException; import io.cdap.cdap.common.NotFoundException; -import io.cdap.cdap.internal.credential.store.CredentialProfileStore; import io.cdap.cdap.proto.credential.CredentialIdentity; import io.cdap.cdap.proto.credential.CredentialProfile; import io.cdap.cdap.proto.id.CredentialIdentityId; diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/credential/CredentialProviderTestBase.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/credential/CredentialProviderTestBase.java index 73842c91f96e..cfe57f8cf5e5 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/credential/CredentialProviderTestBase.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/credential/CredentialProviderTestBase.java @@ -31,13 +31,12 @@ import io.cdap.cdap.common.metrics.NoOpMetricsCollectionService; import io.cdap.cdap.data.runtime.StorageModule; import io.cdap.cdap.data.runtime.SystemDatasetRuntimeModule; -import io.cdap.cdap.internal.credential.store.CredentialIdentityStore; -import io.cdap.cdap.internal.credential.store.CredentialProfileStore; import io.cdap.cdap.proto.credential.CredentialProfile; import io.cdap.cdap.proto.credential.CredentialProvisioningException; import io.cdap.cdap.proto.credential.ProvisionedCredential; import io.cdap.cdap.proto.id.CredentialProfileId; import io.cdap.cdap.security.authorization.AuthorizationEnforcementModule; +import io.cdap.cdap.security.encryption.NoOpAeadCipher; import io.cdap.cdap.security.spi.authorization.ContextAccessEnforcer; import io.cdap.cdap.security.spi.credential.CredentialProvider; import io.cdap.cdap.security.spi.credential.ProfileValidationException; @@ -121,8 +120,8 @@ protected void configure() { // Setup credential managers. TransactionRunner runner = injector.getInstance(TransactionRunner.class); - CredentialProfileStore profileStore = new CredentialProfileStore(); - CredentialIdentityStore identityStore = new CredentialIdentityStore(); + CredentialProfileStore profileStore = new CredentialProfileStore(new NoOpAeadCipher()); + CredentialIdentityStore identityStore = new CredentialIdentityStore(new NoOpAeadCipher()); credentialProfileManager = new CredentialProfileManager(identityStore, profileStore, runner, mockCredentialProviderLoader); credentialIdentityManager = new CredentialIdentityManager(identityStore, profileStore, diff --git a/cdap-common/src/main/java/io/cdap/cdap/common/conf/Constants.java b/cdap-common/src/main/java/io/cdap/cdap/common/conf/Constants.java index 352447b40761..3178a7a8d673 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/common/conf/Constants.java +++ b/cdap-common/src/main/java/io/cdap/cdap/common/conf/Constants.java @@ -1850,6 +1850,42 @@ public static final class AuthenticationServer { public static final String SSL_TRUSTSTORE_PASSWORD = "security.auth.server.ssl.truststore.password"; } + /** + * Security configurations for encryption. + */ + public static final class Encryption { + /** + * Directory for encryption extensions. + */ + public static final String EXTENSIONS_DIR = "security.encryption.extensions.dir"; + /** + * Encryption cipher for user credentials. + */ + public static final String USER_CREDENTIAL_ENCRYPTION_CIPHER_NAME = + "security.encryption.user.credential.encryption.cipher.name"; + /** + * Prefix for properties used for user credential encryption. + */ + public static final String USER_CREDENTIAL_ENCRYPTION_PROPERTIES_PREFIX = + "security.encryption.user.credential.encryption.properties."; + /** + * Encryption mode for sensitive data storage. + */ + public static final String DATA_STORAGE_ENCRYPTION_CIPHER_NAME = + "security.encryption.data.storage.encryption.cipher.name"; + /** + * Prefix for properties used for sensitive data storage encryption. + */ + public static final String DATA_STORAGE_ENCRYPTION_PROPERTIES_PREFIX = + "security.encryption.data.storage.encryption.properties."; + + /** + * Associated Data for user credential encryption. + */ + public static final String USER_CREDENTIAL_ENCRYPTION_ASSOCIATED_DATA + = "UserCredentialEncryptionAD"; + } + /** * Path to the Kerberos keytab file used by CDAP master. */ @@ -2228,10 +2264,10 @@ public static final class Event { public static final String START_PROGRAM_EVENT_FETCH_SIZE = "event.reader.start.fetch.size"; public static final String START_EVENTS_READER_EXTENSIONS_DIR = "events.reader.extensions.start.dir"; public static final String START_EVENTS_READER_EXTENSIONS_ENABLED_LIST = - "events.reader.extensions.start.enabled.list"; + "events.reader.extensions.start.enabled.list"; public static final String START_EVENT_PREFIX = "event.reader.start"; public static final String MINIMUM_FREE_CAPACITY_BEFORE_PULL = - "event.readers.capacity.before.pull"; + "event.readers.capacity.before.pull"; } /** diff --git a/cdap-common/src/main/resources/cdap-default.xml b/cdap-common/src/main/resources/cdap-default.xml index 0b33354be333..a6777d87bb82 100644 --- a/cdap-common/src/main/resources/cdap-default.xml +++ b/cdap-common/src/main/resources/cdap-default.xml @@ -6133,6 +6133,34 @@ + + + security.encryption.extensions.dir + /opt/cdap/master/ext/encryption + + Semicolon-separated list of local directories that are scanned for CDAP + encryption extensions. + + + + + security.encryption.user.credential.encryption.cipher.name + NONE + + Encryption mode for user credential encryption. User credential encryption provides protection + against credential compromise as it moves between microservices. + + + + + security.encryption.data.storage.encryption.cipher.name + NONE + + Encryption mode for sensitive data storage. Sensitive data (e.g. stored passwords, etc) + may be encrypted by CDAP for additional protections at rest. + + + artifact.localizer.preload.list diff --git a/cdap-data-fabric/src/main/java/io/cdap/cdap/store/StoreDefinition.java b/cdap-data-fabric/src/main/java/io/cdap/cdap/store/StoreDefinition.java index 0c71048b439f..10cb3304e4ae 100644 --- a/cdap-data-fabric/src/main/java/io/cdap/cdap/store/StoreDefinition.java +++ b/cdap-data-fabric/src/main/java/io/cdap/cdap/store/StoreDefinition.java @@ -1298,7 +1298,7 @@ public static final class CredentialProviderStore { .withId(CREDENTIAL_PROFILES) .withFields(Fields.stringType(NAMESPACE_FIELD), Fields.stringType(PROFILE_NAME_FIELD), - Fields.stringType(PROFILE_DATA_FIELD)) + Fields.bytesType(PROFILE_DATA_FIELD)) .withPrimaryKeys(NAMESPACE_FIELD, PROFILE_NAME_FIELD) .build(); @@ -1308,7 +1308,7 @@ public static final class CredentialProviderStore { .withFields(Fields.stringType(NAMESPACE_FIELD), Fields.stringType(IDENTITY_NAME_FIELD), Fields.stringType(IDENTITY_PROFILE_INDEX_FIELD), - Fields.stringType(IDENTITY_DATA_FIELD)) + Fields.bytesType(IDENTITY_DATA_FIELD)) .withPrimaryKeys(NAMESPACE_FIELD, IDENTITY_NAME_FIELD) .withIndexes(IDENTITY_PROFILE_INDEX_FIELD) .build(); diff --git a/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/dataset2/lib/table/leveldb/LevelDBMetricsTableTest.java b/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/dataset2/lib/table/leveldb/LevelDBMetricsTableTest.java index 96539e04c0ea..5f018ce6f728 100644 --- a/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/dataset2/lib/table/leveldb/LevelDBMetricsTableTest.java +++ b/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/dataset2/lib/table/leveldb/LevelDBMetricsTableTest.java @@ -56,16 +56,16 @@ public static void setup() throws Exception { CConfiguration conf = CConfiguration.create(); conf.set(Constants.CFG_LOCAL_DATA_DIR, tmpFolder.newFolder().getAbsolutePath()); Injector injector = Guice.createInjector( - new ConfigModule(conf), - new NonCustomLocationUnitTestModule(), - new AuthorizationTestModule(), - new AuthorizationEnforcementModule().getInMemoryModules(), - new AuthenticationContextModules().getMasterModule(), - new InMemoryDiscoveryModule(), - new SystemDatasetRuntimeModule().getInMemoryModules(), - new DataSetsModules().getInMemoryModules(), - new DataFabricLevelDBModule(), - new TransactionMetricsModule()); + new ConfigModule(conf), + new NonCustomLocationUnitTestModule(), + new AuthorizationTestModule(), + new AuthorizationEnforcementModule().getInMemoryModules(), + new AuthenticationContextModules().getMasterModule(), + new InMemoryDiscoveryModule(), + new SystemDatasetRuntimeModule().getInMemoryModules(), + new DataSetsModules().getInMemoryModules(), + new DataFabricLevelDBModule(), + new TransactionMetricsModule()); dsFramework = injector.getInstance(DatasetFramework.class); } @@ -73,7 +73,8 @@ public static void setup() throws Exception { @Override protected MetricsTable getTable(String name) throws Exception { DatasetId metricsDatasetInstanceId = NamespaceId.SYSTEM.dataset(name); - return DatasetsUtil.getOrCreateDataset(dsFramework, metricsDatasetInstanceId, MetricsTable.class.getName(), - DatasetProperties.EMPTY, null); + return DatasetsUtil + .getOrCreateDataset(dsFramework, metricsDatasetInstanceId, MetricsTable.class.getName(), + DatasetProperties.EMPTY, null); } } diff --git a/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/dataset2/lib/table/leveldb/LevelDBTableCoreTest.java b/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/dataset2/lib/table/leveldb/LevelDBTableCoreTest.java index df5237bab92c..5b0d0e1b4171 100644 --- a/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/dataset2/lib/table/leveldb/LevelDBTableCoreTest.java +++ b/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/dataset2/lib/table/leveldb/LevelDBTableCoreTest.java @@ -46,6 +46,7 @@ public class LevelDBTableCoreTest { + @ClassRule public static TemporaryFolder tmpFolder = new TemporaryFolder(); @@ -60,14 +61,14 @@ public static void init() throws Exception { CConfiguration conf = CConfiguration.create(); conf.set(Constants.CFG_LOCAL_DATA_DIR, tmpFolder.newFolder().getAbsolutePath()); injector = Guice.createInjector( - new ConfigModule(conf), - new NonCustomLocationUnitTestModule(), - new InMemoryDiscoveryModule(), - new DataFabricLevelDBModule(), - new TransactionMetricsModule(), - new AuthorizationTestModule(), - new AuthorizationEnforcementModule().getStandaloneModules(), - new AuthenticationContextModules().getMasterModule()); + new ConfigModule(conf), + new NonCustomLocationUnitTestModule(), + new InMemoryDiscoveryModule(), + new DataFabricLevelDBModule(), + new TransactionMetricsModule(), + new AuthorizationTestModule(), + new AuthorizationEnforcementModule().getStandaloneModules(), + new AuthenticationContextModules().getMasterModule()); service = injector.getInstance(LevelDBTableService.class); } @@ -107,7 +108,8 @@ public void testGetAndPut() throws Exception { // After deleting the value at default version (i.e. max version), // reading highest-versioned value should return the value at a proper version. deleteRowColDefaultVersion(table, row, col); - Assert.assertTrue(readRowColLatest(table, row, col).equals(String.format("%s-%d", val, numVersion - 1))); + Assert.assertTrue( + readRowColLatest(table, row, col).equals(String.format("%s-%d", val, numVersion - 1))); // Reading value at default version (i.e. max version) should return null as it has been deleted above. Assert.assertEquals(null, readRowColDefaultVersion(table, row, col)); @@ -133,9 +135,10 @@ public void testScan() throws Exception { writeData(table, rowNamePrefix, numRows, colName, 1024, numVersion); // Scan only the first row and make sure no data from other rows are returned. - try (Scanner scanner = table.scan(getRowName(rowNamePrefix, 0).getBytes(StandardCharsets.UTF_8), - getRowName(rowNamePrefix, 1).getBytes(StandardCharsets.UTF_8), - null, null, null)) { + try (Scanner scanner = table + .scan(getRowName(rowNamePrefix, 0).getBytes(StandardCharsets.UTF_8), + getRowName(rowNamePrefix, 1).getBytes(StandardCharsets.UTF_8), + null, null, null)) { Row row; while ((row = scanner.next()) != null) { String rowName = new String(row.getRow(), StandardCharsets.UTF_8); @@ -147,9 +150,10 @@ public void testScan() throws Exception { // because scan uses the max version at row i + 1 as scan end key (excluded) and we want to make sure // nothing from row i + 1 gets returned in such case. writeRowColDefaultVersion(table, getRowName(rowNamePrefix, 1), colName, "dummy-value"); - try (Scanner scanner = table.scan(getRowName(rowNamePrefix, 0).getBytes(StandardCharsets.UTF_8), - getRowName(rowNamePrefix, 1).getBytes(StandardCharsets.UTF_8), - null, null, null)) { + try (Scanner scanner = table + .scan(getRowName(rowNamePrefix, 0).getBytes(StandardCharsets.UTF_8), + getRowName(rowNamePrefix, 1).getBytes(StandardCharsets.UTF_8), + null, null, null)) { Row row; while ((row = scanner.next()) != null) { String rowName = new String(row.getRow(), StandardCharsets.UTF_8); @@ -251,12 +255,14 @@ public void testDelete() throws Exception { } // Ensure reading the most recent version returns the latest version 9. - Assert.assertTrue(readRowColLatest(table, row, col).equals(String.format("%s-%d", val, numVersion - 1))); + Assert.assertTrue( + readRowColLatest(table, row, col).equals(String.format("%s-%d", val, numVersion - 1))); // Delete the highest version (i.e. KeyValue.LATEST_TIMESTAMP) should be an noop // since there was no value written at that version. deleteRowColDefaultVersion(table, row, col); - Assert.assertTrue(readRowColLatest(table, row, col).equals(String.format("%s-%d", val, numVersion - 1))); + Assert.assertTrue( + readRowColLatest(table, row, col).equals(String.format("%s-%d", val, numVersion - 1))); // Write at the highest version (i.e. KeyValue.LATEST_TIMESTAMP) should hide all old versions. writeRowColDefaultVersion(table, row, col, val); @@ -279,32 +285,33 @@ public void testDelete() throws Exception { * Write the given value as the latest at the target row and col. */ private void writeRowColDefaultVersion(LevelDBTableCore table, String row, String col, String val) - throws IOException { + throws IOException { table.putDefaultVersion(row.getBytes(StandardCharsets.UTF_8), - col.getBytes(StandardCharsets.UTF_8), - val.getBytes(StandardCharsets.UTF_8)); + col.getBytes(StandardCharsets.UTF_8), + val.getBytes(StandardCharsets.UTF_8)); } /** * Write the given value as specified version at the target row and col. */ private void writeRowCol(LevelDBTableCore table, String row, String col, String val, long version) - throws IOException { + throws IOException { table.put(row.getBytes(StandardCharsets.UTF_8), - col.getBytes(StandardCharsets.UTF_8), - val.getBytes(StandardCharsets.UTF_8), - version); + col.getBytes(StandardCharsets.UTF_8), + val.getBytes(StandardCharsets.UTF_8), + version); } /** - * Read the value from the target row and col at default version. - * Return null if there is no value for the specified row and col. + * Read the value from the target row and col at default version. Return null if there is no value + * for the specified row and col. */ @Nullable - private String readRowColDefaultVersion(LevelDBTableCore table, String row, String col) throws IOException { + private String readRowColDefaultVersion(LevelDBTableCore table, String row, String col) + throws IOException { byte[] val = null; val = table.getDefaultVersion(row.getBytes(StandardCharsets.UTF_8), - col.getBytes(StandardCharsets.UTF_8)); + col.getBytes(StandardCharsets.UTF_8)); if (val == null) { return null; } @@ -312,11 +319,12 @@ private String readRowColDefaultVersion(LevelDBTableCore table, String row, Stri } @Nullable - private String readRowCol(LevelDBTableCore table, String row, String col, long version) throws IOException { + private String readRowCol(LevelDBTableCore table, String row, String col, long version) + throws IOException { byte[] val = null; val = table.get(row.getBytes(StandardCharsets.UTF_8), - col.getBytes(StandardCharsets.UTF_8), - version); + col.getBytes(StandardCharsets.UTF_8), + version); if (val == null) { return null; } @@ -324,11 +332,12 @@ private String readRowCol(LevelDBTableCore table, String row, String col, long v } @Nullable - private String readRowColLatest(LevelDBTableCore table, String row, String col) throws IOException { + private String readRowColLatest(LevelDBTableCore table, String row, String col) + throws IOException { byte[] val = null; val = table.getLatest(row.getBytes(StandardCharsets.UTF_8), - col.getBytes(StandardCharsets.UTF_8), - null); + col.getBytes(StandardCharsets.UTF_8), + null); if (val == null) { return null; } @@ -339,26 +348,29 @@ private String readRowColLatest(LevelDBTableCore table, String row, String col) /** * Delete the value at latest version (i.e. KeyValue.LATEST_TIMESTAMP) in the target row and col. */ - private void deleteRowColDefaultVersion(LevelDBTableCore table, String row, String col) throws IOException { + private void deleteRowColDefaultVersion(LevelDBTableCore table, String row, String col) + throws IOException { table.deleteDefaultVersion(row.getBytes(StandardCharsets.UTF_8), - col.getBytes(StandardCharsets.UTF_8)); + col.getBytes(StandardCharsets.UTF_8)); } /** * Delete the value at the specified version in the target row and col. */ - private void deleteRowCol(LevelDBTableCore table, String row, String col, long version) throws IOException { + private void deleteRowCol(LevelDBTableCore table, String row, String col, long version) + throws IOException { table.delete(row.getBytes(StandardCharsets.UTF_8), - col.getBytes(StandardCharsets.UTF_8), - version); + col.getBytes(StandardCharsets.UTF_8), + version); } /** - * Write a number of rows to the table. There is only one col per row. For each row and col, write a number of - * values at different versions. + * Write a number of rows to the table. There is only one col per row. For each row and col, write + * a number of values at different versions. */ - private void writeData(LevelDBTableCore table, String rowPrefix, long numRows, String col, int valNumBytes, - int numVersions) throws IOException { + private void writeData(LevelDBTableCore table, String rowPrefix, long numRows, String col, + int valNumBytes, + int numVersions) throws IOException { Random r = new Random(); byte[] value = new byte[valNumBytes]; for (long rowIndex = 0; rowIndex < numRows; rowIndex++) { diff --git a/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/dataset2/lib/table/leveldb/LevelDBTableServiceTest.java b/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/dataset2/lib/table/leveldb/LevelDBTableServiceTest.java index 2cc0418e02ee..f35a074dad17 100644 --- a/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/dataset2/lib/table/leveldb/LevelDBTableServiceTest.java +++ b/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/dataset2/lib/table/leveldb/LevelDBTableServiceTest.java @@ -55,6 +55,7 @@ */ @RunWith(TestRunner.class) public class LevelDBTableServiceTest { + @ClassRule public static TemporaryFolder tmpFolder = new TemporaryFolder(); @@ -66,14 +67,14 @@ public static void init() throws Exception { CConfiguration conf = CConfiguration.create(); conf.set(Constants.CFG_LOCAL_DATA_DIR, tmpFolder.newFolder().getAbsolutePath()); injector = Guice.createInjector( - new ConfigModule(conf), - new NonCustomLocationUnitTestModule(), - new InMemoryDiscoveryModule(), - new DataFabricLevelDBModule(), - new TransactionMetricsModule(), - new AuthorizationTestModule(), - new AuthorizationEnforcementModule().getStandaloneModules(), - new AuthenticationContextModules().getMasterModule()); + new ConfigModule(conf), + new NonCustomLocationUnitTestModule(), + new InMemoryDiscoveryModule(), + new DataFabricLevelDBModule(), + new TransactionMetricsModule(), + new AuthorizationTestModule(), + new AuthorizationEnforcementModule().getStandaloneModules(), + new AuthenticationContextModules().getMasterModule()); service = injector.getInstance(LevelDBTableService.class); } @@ -136,7 +137,8 @@ public void testCompression() throws Exception { compressedTableService.ensureTableExists(tableUncompressed); // Write large enough number of rows to ensure some data are flushed to disk (e.g. > 4MB) writeSome(compressedTableService, tableUncompressed, 32768, 1024, true); - long uncompressedDiskSizeBytes = compressedTableService.getTableStats().get(tableUncompressedID).getDiskSizeBytes(); + long uncompressedDiskSizeBytes = compressedTableService.getTableStats().get(tableUncompressedID) + .getDiskSizeBytes(); // Write compressible data to table that enables compression, then record on-disk size, which should be // smaller than that with compressed disabled. @@ -147,7 +149,8 @@ public void testCompression() throws Exception { TableId tableCompressedID = TableId.from("default", "tableCompressed"); uncompressedTableService.ensureTableExists(tableCompressed); writeSome(uncompressedTableService, tableCompressed, 32768, 1024, true); - long compressedDiskSizeBytes = uncompressedTableService.getTableStats().get(tableCompressedID).getDiskSizeBytes(); + long compressedDiskSizeBytes = uncompressedTableService.getTableStats().get(tableCompressedID) + .getDiskSizeBytes(); // Ensure on-disk file size is smaller when compression is enabled. Assert.assertTrue(uncompressedDiskSizeBytes > compressedDiskSizeBytes); @@ -208,14 +211,14 @@ public void testFileMetaDataSize() throws IOException { Assert.assertFalse(fileMetaDatas.isEmpty()); for (FileMetaData fileMetaData : fileMetaDatas) { Assert.assertEquals(fileMetaData.getLargest().getUserKey().length(), - fileMetaData.getLargest().getUserKey().getRawArray().length); + fileMetaData.getLargest().getUserKey().getRawArray().length); Assert.assertEquals(fileMetaData.getSmallest().getUserKey().length(), - fileMetaData.getSmallest().getUserKey().getRawArray().length); + fileMetaData.getSmallest().getUserKey().getRawArray().length); } } private void writeSome(LevelDBTableService service, String tableName, - long numRows, int valNumBytes, boolean compressible) throws IOException { + long numRows, int valNumBytes, boolean compressible) throws IOException { LevelDBTableCore table = new LevelDBTableCore(tableName, service); Random r = new Random(); int keyNumBytes = 64; diff --git a/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/dataset2/lib/table/leveldb/LevelDBTableTest.java b/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/dataset2/lib/table/leveldb/LevelDBTableTest.java index 8951b0f2454f..18e6ffc625cd 100644 --- a/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/dataset2/lib/table/leveldb/LevelDBTableTest.java +++ b/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/dataset2/lib/table/leveldb/LevelDBTableTest.java @@ -60,27 +60,28 @@ public static void init() throws Exception { cConf = CConfiguration.create(); cConf.set(Constants.CFG_LOCAL_DATA_DIR, tmpFolder.newFolder().getAbsolutePath()); Injector injector = Guice.createInjector( - new ConfigModule(cConf), - new NonCustomLocationUnitTestModule(), - new InMemoryDiscoveryModule(), - new DataFabricLevelDBModule(), - new TransactionMetricsModule(), - new AuthorizationTestModule(), - new AuthorizationEnforcementModule().getStandaloneModules(), - new AuthenticationContextModules().getMasterModule()); + new ConfigModule(cConf), + new NonCustomLocationUnitTestModule(), + new InMemoryDiscoveryModule(), + new DataFabricLevelDBModule(), + new TransactionMetricsModule(), + new AuthorizationTestModule(), + new AuthorizationEnforcementModule().getStandaloneModules(), + new AuthenticationContextModules().getMasterModule()); service = injector.getInstance(LevelDBTableService.class); } @Override protected LevelDBTable getTable(DatasetContext datasetContext, String name, - DatasetProperties props, Map runtimeArguments) throws Exception { - DatasetSpecification spec = DatasetSpecification.builder(name, "table").properties(props.getProperties()).build(); + DatasetProperties props, Map runtimeArguments) throws Exception { + DatasetSpecification spec = DatasetSpecification.builder(name, "table") + .properties(props.getProperties()).build(); return new LevelDBTable(datasetContext, name, service, cConf, spec); } @Override protected LevelDBTableAdmin getTableAdmin(DatasetContext datasetContext, String name, - DatasetProperties props) throws IOException { + DatasetProperties props) throws IOException { DatasetSpecification spec = TABLE_DEFINITION.configure(name, props); return new LevelDBTableAdmin(datasetContext, spec, service, cConf); } @@ -94,7 +95,7 @@ protected boolean isReadlessIncrementSupported() { public void testTablesSurviveAcrossRestart() throws Exception { // todo make this test run for hbase, too - requires refactoring of their injection // test on ASCII table name but also on some non-ASCII ones - final String[] tableNames = { "table", "t able", "t\u00C3ble", "100%" }; + final String[] tableNames = {"table", "t able", "t\u00C3ble", "100%"}; // create a table and verify it is in the list of tables for (String tableName : tableNames) { @@ -107,7 +108,7 @@ public void testTablesSurviveAcrossRestart() throws Exception { service.clearTables(); for (String tableName : tableNames) { Preconditions.checkState(service.list().contains( - PrefixedNamespaces.namespace(cConf, CONTEXT1.getNamespaceId(), tableName))); + PrefixedNamespaces.namespace(cConf, CONTEXT1.getNamespaceId(), tableName))); } } } diff --git a/cdap-data-fabric/src/test/java/io/cdap/cdap/spi/metadata/dataset/DatasetMetadataStorageTest.java b/cdap-data-fabric/src/test/java/io/cdap/cdap/spi/metadata/dataset/DatasetMetadataStorageTest.java index 21350177a6d9..82f5036b2fd3 100644 --- a/cdap-data-fabric/src/test/java/io/cdap/cdap/spi/metadata/dataset/DatasetMetadataStorageTest.java +++ b/cdap-data-fabric/src/test/java/io/cdap/cdap/spi/metadata/dataset/DatasetMetadataStorageTest.java @@ -83,26 +83,27 @@ public static void setup() throws IOException { doSetup(); } - public static Injector doSetup(Module ... additionalModules) throws IOException { + public static Injector doSetup(Module... additionalModules) throws IOException { List modules = ImmutableList.builder() - .add( - new ConfigModule(), - new LocalLocationModule(), - new TransactionInMemoryModule(), - new SystemDatasetRuntimeModule().getInMemoryModules(), - new NamespaceAdminTestModule(), - new AuthorizationTestModule(), - new AuthorizationEnforcementModule().getInMemoryModules(), - new AuthenticationContextModules().getMasterModule(), - new StorageModule(), - new AbstractModule() { - @Override - protected void configure() { - bind(MetricsCollectionService.class).to(NoOpMetricsCollectionService.class).in(Scopes.SINGLETON); - } - }) - .add(additionalModules) - .build(); + .add( + new ConfigModule(), + new LocalLocationModule(), + new TransactionInMemoryModule(), + new SystemDatasetRuntimeModule().getInMemoryModules(), + new NamespaceAdminTestModule(), + new AuthorizationTestModule(), + new AuthorizationEnforcementModule().getInMemoryModules(), + new AuthenticationContextModules().getMasterModule(), + new StorageModule(), + new AbstractModule() { + @Override + protected void configure() { + bind(MetricsCollectionService.class).to(NoOpMetricsCollectionService.class) + .in(Scopes.SINGLETON); + } + }) + .add(additionalModules) + .build(); Injector injector = Guice.createInjector(modules); txManager = injector.getInstance(TransactionManager.class); @@ -146,42 +147,53 @@ public void testSearchWeight() throws IOException { // Add metadata String multiWordValue = "aV1 av2 , - , av3 - av4_av5 av6"; - Map userProps = ImmutableMap.of("key1", "value1", "key2", "value2", "multiword", multiWordValue); + Map userProps = ImmutableMap + .of("key1", "value1", "key2", "value2", "multiword", multiWordValue); Map systemProps = ImmutableMap.of("sysKey1", "sysValue1"); Set userTags = ImmutableSet.of("tag1", "tag2"); Set temporaryUserTags = ImmutableSet.of("tag3", "tag4"); Map dataset1UserProps = ImmutableMap.of("sKey1", "sValuee1 sValuee2"); - Map dataset2UserProps = ImmutableMap.of("sKey1", "sValue1 sValue2", "Key1", "Value1"); + Map dataset2UserProps = ImmutableMap + .of("sKey1", "sValue1 sValue2", "Key1", "Value1"); Set sysTags = ImmutableSet.of("sysTag1"); MetadataRecord service1Record = new MetadataRecord( - service1, union(new Metadata(USER, userTags, userProps), new Metadata(SYSTEM, sysTags, systemProps))); - mds.apply(new Update(service1Record.getEntity(), service1Record.getMetadata()), MutationOptions.DEFAULT); + service1, + union(new Metadata(USER, userTags, userProps), new Metadata(SYSTEM, sysTags, systemProps))); + mds.apply(new Update(service1Record.getEntity(), service1Record.getMetadata()), + MutationOptions.DEFAULT); // dd and then remove some metadata for dataset2 - mds.apply(new Update(dataset2, new Metadata(USER, temporaryUserTags, userProps)), MutationOptions.DEFAULT); + mds.apply(new Update(dataset2, new Metadata(USER, temporaryUserTags, userProps)), + MutationOptions.DEFAULT); mds.apply(new Remove(dataset2, temporaryUserTags.stream() - .map(tag -> new ScopedNameOfKind(TAG, USER, tag)).collect(Collectors.toSet())), MutationOptions.DEFAULT); + .map(tag -> new ScopedNameOfKind(TAG, USER, tag)).collect(Collectors.toSet())), + MutationOptions.DEFAULT); mds.apply(new Remove(dataset2, userProps.keySet().stream() - .map(tag -> new ScopedNameOfKind(PROPERTY, USER, tag)).collect(Collectors.toSet())), MutationOptions.DEFAULT); + .map(tag -> new ScopedNameOfKind(PROPERTY, USER, tag)).collect(Collectors.toSet())), + MutationOptions.DEFAULT); - MetadataRecord dataset1Record = new MetadataRecord(dataset1, new Metadata(USER, tags(), dataset1UserProps)); - MetadataRecord dataset2Record = new MetadataRecord(dataset2, new Metadata(USER, tags(), dataset2UserProps)); + MetadataRecord dataset1Record = new MetadataRecord(dataset1, + new Metadata(USER, tags(), dataset1UserProps)); + MetadataRecord dataset2Record = new MetadataRecord(dataset2, + new Metadata(USER, tags(), dataset2UserProps)); mds.batch(ImmutableList.of(new Update(dataset1Record.getEntity(), dataset1Record.getMetadata()), - new Update(dataset2Record.getEntity(), dataset2Record.getMetadata())), - MutationOptions.DEFAULT); + new Update(dataset2Record.getEntity(), dataset2Record.getMetadata())), + MutationOptions.DEFAULT); // Test score and metadata match assertInOrder(mds, SearchRequest.of("value1 multiword:av2").addNamespace(ns).build(), - service1Record, dataset2Record); - assertInOrder(mds, SearchRequest.of("value1 sValue*").addNamespace(ns).setLimit(Integer.MAX_VALUE).build(), - dataset2Record, dataset1Record, service1Record); + service1Record, dataset2Record); + assertInOrder(mds, + SearchRequest.of("value1 sValue*").addNamespace(ns).setLimit(Integer.MAX_VALUE).build(), + dataset2Record, dataset1Record, service1Record); assertResults(mds, SearchRequest.of("*").addNamespace(ns).setLimit(Integer.MAX_VALUE).build(), - dataset2Record, dataset1Record, service1Record); + dataset2Record, dataset1Record, service1Record); // clean up - mds.batch(ImmutableList.of(new Drop(service1), new Drop(dataset1), new Drop(dataset2)), MutationOptions.DEFAULT); + mds.batch(ImmutableList.of(new Drop(service1), new Drop(dataset1), new Drop(dataset2)), + MutationOptions.DEFAULT); } // this test is specific to teh DatasetMetadataStorage, because of the specific way it tests pagination: @@ -200,11 +212,11 @@ public void testCrossNamespacePagination() throws IOException { MetadataEntity ns2app2 = ns2Id.app("a2").toMetadataEntity(); mds.batch(ImmutableList.of(new Update(ns1app1, new Metadata(USER, tags("v1"))), - new Update(ns1app2, new Metadata(USER, tags("v1"))), - new Update(ns1app3, new Metadata(USER, tags("v1"))), - new Update(ns2app1, new Metadata(USER, tags("v1"))), - new Update(ns2app2, new Metadata(USER, tags("v1")))), - MutationOptions.DEFAULT); + new Update(ns1app2, new Metadata(USER, tags("v1"))), + new Update(ns1app3, new Metadata(USER, tags("v1"))), + new Update(ns2app1, new Metadata(USER, tags("v1"))), + new Update(ns2app2, new Metadata(USER, tags("v1")))), + MutationOptions.DEFAULT); MetadataRecord record11 = new MetadataRecord(ns1app1, new Metadata(USER, tags("v1"))); MetadataRecord record12 = new MetadataRecord(ns1app2, new Metadata(USER, tags("v1"))); @@ -213,30 +225,36 @@ public void testCrossNamespacePagination() throws IOException { MetadataRecord record22 = new MetadataRecord(ns2app2, new Metadata(USER, tags("v1"))); SearchResponse response = - assertResults(mds, SearchRequest.of("*").setLimit(Integer.MAX_VALUE).setCursorRequested(true).build(), - record11, record12, record13, record21, record22); + assertResults(mds, + SearchRequest.of("*").setLimit(Integer.MAX_VALUE).setCursorRequested(true).build(), + record11, record12, record13, record21, record22); // iterate over results to find the order in which they are returned Iterator resultIter = response.getResults().iterator(); MetadataRecord[] results = { - resultIter.next(), resultIter.next(), resultIter.next(), resultIter.next(), resultIter.next() }; + resultIter.next(), resultIter.next(), resultIter.next(), resultIter.next(), + resultIter.next()}; // get 4 results (guaranteed to have at least one from each namespace), offset 1 - assertResults(mds, SearchRequest.of("*").setCursorRequested(true).setOffset(1).setLimit(4).build(), - results[1], results[2], results[3], results[4]); + assertResults(mds, + SearchRequest.of("*").setCursorRequested(true).setOffset(1).setLimit(4).build(), + results[1], results[2], results[3], results[4]); // get the first four - assertResults(mds, SearchRequest.of("*").setCursorRequested(true).setOffset(0).setLimit(4).build(), - results[0], results[1], results[2], results[3]); + assertResults(mds, + SearchRequest.of("*").setCursorRequested(true).setOffset(0).setLimit(4).build(), + results[0], results[1], results[2], results[3]); // get middle 3 - assertResults(mds, SearchRequest.of("*").setCursorRequested(true).setOffset(1).setLimit(3).build(), - results[1], results[2], results[3], results[3]); + assertResults(mds, + SearchRequest.of("*").setCursorRequested(true).setOffset(1).setLimit(3).build(), + results[1], results[2], results[3], results[3]); // clean up mds.batch(ImmutableList.of( - new Drop(ns1app1), new Drop(ns1app2), new Drop(ns1app3), new Drop(ns2app1), new Drop(ns2app2)), - MutationOptions.DEFAULT); + new Drop(ns1app1), new Drop(ns1app2), new Drop(ns1app3), new Drop(ns2app1), + new Drop(ns2app2)), + MutationOptions.DEFAULT); } @Test @@ -245,19 +263,22 @@ public void testNsScopes() { testNsScopes(null, null, EnumSet.allOf(EntityScope.class), false); testNsScopes(Collections.emptySet(), null, EnumSet.allOf(EntityScope.class), false); // system only - testNsScopes(ImmutableSet.of("system"), NamespaceId.SYSTEM, EnumSet.of(EntityScope.SYSTEM), false); + testNsScopes(ImmutableSet.of("system"), NamespaceId.SYSTEM, EnumSet.of(EntityScope.SYSTEM), + false); // user namespace only - testNsScopes(ImmutableSet.of("myns"), new NamespaceId("myns"), EnumSet.of(EntityScope.USER), false); + testNsScopes(ImmutableSet.of("myns"), new NamespaceId("myns"), EnumSet.of(EntityScope.USER), + false); // user and system namespace - testNsScopes(ImmutableSet.of("myns", "system"), new NamespaceId("myns"), EnumSet.allOf(EntityScope.class), false); + testNsScopes(ImmutableSet.of("myns", "system"), new NamespaceId("myns"), + EnumSet.allOf(EntityScope.class), false); // multiple user namespaces testNsScopes(ImmutableSet.of("myns", "yourns"), null, null, true); testNsScopes(ImmutableSet.of("myns", "system", "yourns"), null, null, true); } private void testNsScopes(Set namespaces, - NamespaceId expectedNamespace, Set expectedScopes, - boolean expectUnsupportedOperation) { + NamespaceId expectedNamespace, Set expectedScopes, + boolean expectUnsupportedOperation) { if (expectUnsupportedOperation) { try { DatasetMetadataStorage.determineNamespaceAndScopes(namespaces); @@ -266,8 +287,10 @@ private void testNsScopes(Set namespaces, return; // expected } } - ImmutablePair> pair = DatasetMetadataStorage.determineNamespaceAndScopes(namespaces); - Assert.assertEquals("namespace does not match for " + namespaces, expectedNamespace, pair.getFirst()); + ImmutablePair> pair = DatasetMetadataStorage + .determineNamespaceAndScopes(namespaces); + Assert.assertEquals("namespace does not match for " + namespaces, expectedNamespace, + pair.getFirst()); Assert.assertEquals("scopes don't match for " + namespaces, expectedScopes, pair.getSecond()); } @@ -285,15 +308,15 @@ public void testCursorOffsetAndLimits() { } private void testCursorsOffsetsAndLimits(Cursor cursor, boolean cursorRequested, - int offsetRequested, int limitRequested, - String expectedCursor, - int expectedOffsetToRequest, int expectedOffsetToRespond, - int expectedLimitToRequest, int expectedLimitToRespond) { + int offsetRequested, int limitRequested, + String expectedCursor, + int expectedOffsetToRequest, int expectedOffsetToRespond, + int expectedLimitToRequest, int expectedLimitToRespond) { SearchRequest request = SearchRequest.of("*") - .setCursor(cursor == null ? null : cursor.toString()).setCursorRequested(cursorRequested) - .setOffset(offsetRequested).setLimit(limitRequested).build(); + .setCursor(cursor == null ? null : cursor.toString()).setCursorRequested(cursorRequested) + .setOffset(offsetRequested).setLimit(limitRequested).build(); DatasetMetadataStorage.CursorAndOffsetInfo info = - DatasetMetadataStorage.determineCursorOffsetAndLimits(request, cursor); + DatasetMetadataStorage.determineCursorOffsetAndLimits(request, cursor); Assert.assertEquals(expectedCursor, info.getCursor()); Assert.assertEquals(expectedOffsetToRequest, info.getOffsetToRequest()); Assert.assertEquals(expectedOffsetToRespond, info.getOffsetToRespond()); diff --git a/cdap-encryption-ext-tink/pom.xml b/cdap-encryption-ext-tink/pom.xml new file mode 100644 index 000000000000..540f24f7775e --- /dev/null +++ b/cdap-encryption-ext-tink/pom.xml @@ -0,0 +1,165 @@ + + + + + + cdap + io.cdap.cdap + 6.11.0-SNAPSHOT + + 4.0.0 + + cdap-encryption-ext-tink + CDAP Tink Encryption Extension + jar + + + 1.11.0 + 1.9.0 + + + 3.24.4 + + 32.1.2-jre + 26.23.0 + + + + + + com.google.cloud + libraries-bom + ${gcp-libraries-bom.version} + pom + import + + + + + + + + io.cdap.cdap + cdap-security-spi + ${project.version} + provided + + + org.slf4j + slf4j-api + provided + + + + + com.google.crypto.tink + tink + ${tink.version} + + + com.google.crypto.tink + tink-gcpkms + ${tink-gcpkms.version} + + + com.google.protobuf + protobuf-java + + + com.google.guava + guava + + + + + com.google.cloud + google-cloud-kms + + + + + com.google.cloud + google-cloud-core + + + + + junit + junit + test + + + org.mockito + mockito-core + test + + + + + + dist + + + + org.apache.maven.plugins + maven-dependency-plugin + 2.8 + + + copy-dependencies + prepare-package + + copy-dependencies + + + ${project.build.directory}/libexec + false + false + true + true + true + runtime + + + + + + org.apache.maven.plugins + maven-jar-plugin + 2.4 + + + jar + prepare-package + + ${project.build.directory}/libexec + ${project.groupId}.${project.build.finalName} + + + jar + + + + + + + + + diff --git a/cdap-encryption-ext-tink/src/main/java/io/cdap/cdap/encryption/tink/AbstractTinkAeadCipherCryptor.java b/cdap-encryption-ext-tink/src/main/java/io/cdap/cdap/encryption/tink/AbstractTinkAeadCipherCryptor.java new file mode 100644 index 000000000000..0888e0aa97ca --- /dev/null +++ b/cdap-encryption-ext-tink/src/main/java/io/cdap/cdap/encryption/tink/AbstractTinkAeadCipherCryptor.java @@ -0,0 +1,85 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.encryption.tink; + +import com.google.crypto.tink.Aead; +import io.cdap.cdap.security.spi.encryption.AeadCipherContext; +import io.cdap.cdap.security.spi.encryption.AeadCipherCryptor; +import io.cdap.cdap.security.spi.encryption.CipherInitializationException; +import io.cdap.cdap.security.spi.encryption.CipherOperationException; +import java.security.GeneralSecurityException; + +/** + * An {@link AeadCipherCryptor} backed by Tink. + */ +public abstract class AbstractTinkAeadCipherCryptor implements AeadCipherCryptor { + + /** + * Tink AEAD (Authenticated Encryption with Associated Data) primitive. + */ + private Aead aead; + + /** + * Initializes and returns an {@link Aead} primitive. + * + * @return An AEAD primitive for encryption. + */ + protected abstract Aead initializeAead(AeadCipherContext context) + throws CipherInitializationException; + + @Override + public void initialize(AeadCipherContext context) throws CipherInitializationException { + this.aead = initializeAead(context); + } + + /** + * Encrypt the data. + * + * @param plainData data to be encrypted + * @param associatedData used for integrity checking during decryption. + * @return encrypted data + * @throws CipherOperationException if encryption fails + */ + @Override + public byte[] encrypt(byte[] plainData, byte[] associatedData) + throws CipherOperationException { + try { + return aead.encrypt(plainData, associatedData); + } catch (GeneralSecurityException e) { + throw new CipherOperationException("Failed to encrypt: " + e.getMessage(), e); + } + } + + /** + * Decrypt the cipher data. + * + * @param cipherData data to be decrypted + * @param associatedData used for integrity checking, must be the same as that used during + * encryption. + * @return decrypted data + * @throws CipherOperationException if decryption fails + */ + @Override + public byte[] decrypt(byte[] cipherData, byte[] associatedData) + throws CipherOperationException { + try { + return aead.decrypt(cipherData, associatedData); + } catch (GeneralSecurityException e) { + throw new CipherOperationException("Failed to decrypt: " + e.getMessage(), e); + } + } +} diff --git a/cdap-encryption-ext-tink/src/main/java/io/cdap/cdap/encryption/tink/CleartextTinkCipherCryptor.java b/cdap-encryption-ext-tink/src/main/java/io/cdap/cdap/encryption/tink/CleartextTinkCipherCryptor.java new file mode 100644 index 000000000000..595c6c75857e --- /dev/null +++ b/cdap-encryption-ext-tink/src/main/java/io/cdap/cdap/encryption/tink/CleartextTinkCipherCryptor.java @@ -0,0 +1,67 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.encryption.tink; + +import com.google.crypto.tink.Aead; +import com.google.crypto.tink.CleartextKeysetHandle; +import com.google.crypto.tink.JsonKeysetReader; +import com.google.crypto.tink.KeysetHandle; +import com.google.crypto.tink.aead.AeadConfig; +import io.cdap.cdap.security.spi.encryption.AeadCipherContext; +import io.cdap.cdap.security.spi.encryption.CipherInitializationException; +import java.io.IOException; +import java.security.GeneralSecurityException; + +/** + * Tink cipher that allows encrypting and decrypting data using keyset provided via properties. + */ +public class CleartextTinkCipherCryptor extends AbstractTinkAeadCipherCryptor { + + /** + * References a keyset stored in secure properties. + */ + public static final String TINK_CLEARTEXT_KEYSET_KEY = "tink.cleartext.keyset"; + + @Override + public String getName() { + return "tink-cleartext"; + } + + @Override + public Aead initializeAead(AeadCipherContext context) throws CipherInitializationException { + try { + // Init Tink with AEAD primitive + AeadConfig.register(); + + // Load keyset from secure properties + String keysetJson = context.getSecureProperties().get(TINK_CLEARTEXT_KEYSET_KEY); + if (keysetJson == null || keysetJson.isEmpty()) { + throw new IllegalArgumentException(String.format("Tink keyset cannot be null or empty")); + } + KeysetHandle keysetHandle = CleartextKeysetHandle + .read(JsonKeysetReader.withString(keysetJson)); + + return keysetHandle.getPrimitive(Aead.class); + } catch (GeneralSecurityException e) { + throw new CipherInitializationException("Failed to init Tink cipher: " + e.getMessage(), e); + } catch (IOException e) { + throw new CipherInitializationException("Failed to load Tink keyset: " + e.getMessage(), e); + } + } +} + + diff --git a/cdap-encryption-ext-tink/src/main/java/io/cdap/cdap/encryption/tink/CloudKmsClient.java b/cdap-encryption-ext-tink/src/main/java/io/cdap/cdap/encryption/tink/CloudKmsClient.java new file mode 100644 index 000000000000..f4bd489bfe77 --- /dev/null +++ b/cdap-encryption-ext-tink/src/main/java/io/cdap/cdap/encryption/tink/CloudKmsClient.java @@ -0,0 +1,167 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.encryption.tink; + +import com.google.api.gax.rpc.ApiException; +import com.google.api.gax.rpc.StatusCode.Code; +import com.google.cloud.ServiceOptions; +import com.google.cloud.kms.v1.CryptoKey; +import com.google.cloud.kms.v1.CryptoKey.CryptoKeyPurpose; +import com.google.cloud.kms.v1.CryptoKeyName; +import com.google.cloud.kms.v1.CryptoKeyVersion.CryptoKeyVersionAlgorithm; +import com.google.cloud.kms.v1.CryptoKeyVersionTemplate; +import com.google.cloud.kms.v1.KeyManagementServiceClient; +import com.google.cloud.kms.v1.KeyRing; +import com.google.cloud.kms.v1.KeyRingName; +import com.google.cloud.kms.v1.LocationName; +import com.google.protobuf.Duration; +import com.google.protobuf.Timestamp; +import io.cdap.cdap.security.spi.encryption.CipherInitializationException; +import java.io.Closeable; +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.HashSet; +import java.util.Set; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Client wrapping Cloud KMS. + */ +public class CloudKmsClient implements Closeable { + + private static final Logger LOG = LoggerFactory.getLogger(CloudKmsClient.class); + + private final KeyManagementServiceClient client; + private final String projectId; + private final String location; + private final String keyRingId; + // In-memory cache to hold created crypto keys, this is to avoid checking if a given crypto key exists. + private final Set knownCryptoKeys; + + /** + * Constructs Cloud KMS client. + * + * @throws IOException if cloud kms client can not be created + */ + CloudKmsClient(String projectId, String location, String keyRingId) throws IOException { + if (projectId != null) { + this.projectId = projectId; + } else { + this.projectId = ServiceOptions.getDefaultProjectId(); + } + this.location = location; + this.keyRingId = keyRingId; + this.client = KeyManagementServiceClient.create(); + this.knownCryptoKeys = new HashSet<>(); + } + + /** + * Creates a new key ring with the given ID. + * + * @throws CipherInitializationException If there's an error while creating the key ring + */ + void createKeyRingIfNotExists() throws CipherInitializationException { + LOG.debug("Creating key ring with id {} in projects/{}/locations/{}.", keyRingId, projectId, + location); + + try { + client.getKeyRing(KeyRingName.of(projectId, location, keyRingId)); + // If we get here, the key ring was found, so we skip creating the key ring. + return; + } catch (ApiException e) { + if (!e.getStatusCode().getCode().equals(Code.NOT_FOUND)) { + throw new CipherInitializationException("Failed to check if Cloud KMS key ring exists", e); + } + } + + LocationName locationName = LocationName.of(projectId, location); + try { + client.createKeyRing(locationName, keyRingId, KeyRing.newBuilder().build()); + } catch (ApiException e) { + if (!e.getStatusCode().getCode().equals(Code.ALREADY_EXISTS)) { + throw new CipherInitializationException("Failed to create new Cloud KMS key ring", e); + } + LOG.info("Failed to create new Cloud KMS key ring, likely already exists."); + } + } + + /** + * Creates a new crypto key with the given ID. + * + * @param cryptoKeyId The crypto key ID + * @throws CipherInitializationException If there's an error creating crypto key + */ + void createCryptoKeyIfNotExists(String cryptoKeyId) throws CipherInitializationException { + // If crypto key is already created, do not attempt to create it again. + if (knownCryptoKeys.contains(cryptoKeyId)) { + return; + } + + try { + client.getCryptoKey(CryptoKeyName.of(projectId, location, keyRingId, cryptoKeyId)); + // If we get here, the crypto key was found, so we skip creating the crypto key. + return; + } catch (ApiException e) { + if (!e.getStatusCode().getCode().equals(Code.NOT_FOUND)) { + throw new CipherInitializationException("Failed to check if Cloud KMS crypto key exists", + e); + } + } + + // Calculate the date 24 hours from now (this is used below). + long tomorrow = Instant.now().plus(24, ChronoUnit.HOURS).getEpochSecond(); + + // Build the key to create with a rotation schedule. + com.google.cloud.kms.v1.CryptoKey key = + com.google.cloud.kms.v1.CryptoKey.newBuilder() + .setPurpose(CryptoKeyPurpose.ENCRYPT_DECRYPT) + .setVersionTemplate( + CryptoKeyVersionTemplate.newBuilder() + .setAlgorithm(CryptoKeyVersionAlgorithm.GOOGLE_SYMMETRIC_ENCRYPTION)) + + // Rotate every 30 days. + .setRotationPeriod( + Duration.newBuilder().setSeconds(java.time.Duration.ofDays(90).getSeconds())) + + // Start the first rotation in 24 hours. + .setNextRotationTime(Timestamp.newBuilder().setSeconds(tomorrow)) + .build(); + + CryptoKey createdKey; + // Create the key. + try { + createdKey = client + .createCryptoKey(KeyRingName.of(projectId, location, keyRingId), cryptoKeyId, key); + LOG.info("Created key with rotation schedule {}", createdKey.getName()); + } catch (ApiException e) { + if (!e.getStatusCode().getCode().equals(Code.ALREADY_EXISTS)) { + throw new CipherInitializationException("Failed to create crypto key", e); + } + LOG.info("Failed to create Cloud KMS crypto key, likely already exists."); + } + + // In-memory cache to keep list of crypto keys created so far. + knownCryptoKeys.add(cryptoKeyId); + } + + @Override + public void close() throws IOException { + knownCryptoKeys.clear(); + } +} diff --git a/cdap-encryption-ext-tink/src/main/java/io/cdap/cdap/encryption/tink/GcpEnvelopeTinkCipherCryptor.java b/cdap-encryption-ext-tink/src/main/java/io/cdap/cdap/encryption/tink/GcpEnvelopeTinkCipherCryptor.java new file mode 100644 index 000000000000..208e98a7a8dc --- /dev/null +++ b/cdap-encryption-ext-tink/src/main/java/io/cdap/cdap/encryption/tink/GcpEnvelopeTinkCipherCryptor.java @@ -0,0 +1,153 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.encryption.tink; + +import com.google.auth.oauth2.GoogleCredentials; +import com.google.crypto.tink.Aead; +import com.google.crypto.tink.KmsClient; +import com.google.crypto.tink.aead.AeadConfig; +import com.google.crypto.tink.aead.AeadParameters; +import com.google.crypto.tink.aead.KmsEnvelopeAead; +import com.google.crypto.tink.aead.PredefinedAeadParameters; +import com.google.crypto.tink.integration.gcpkms.GcpKmsClient; +import io.cdap.cdap.security.spi.encryption.AeadCipherContext; +import io.cdap.cdap.security.spi.encryption.CipherInitializationException; +import java.io.IOException; +import java.security.GeneralSecurityException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Tink cipher that allows encrypting and decrypting data using Cloud KMS. + */ +public class GcpEnvelopeTinkCipherCryptor extends AbstractTinkAeadCipherCryptor { + + /** + * KEK URI for use by Tink. For details, see + * https://cloud.google.com/kms/docs/client-side-encryption#connect_tink_and_cloud_kms. + */ + private static final String TINK_GCP_KMS_KEK_URI_FORMAT + = "gcp-kms://projects/%s/locations/%s/keyRings/%s/cryptoKeys/%s"; + private static final String TINK_GCP_KMS_PROJECT_ID = "tink.gcp.kms.project.id"; + private static final String TINK_GCP_KMS_LOCATION = "tink.gcp.kms.location"; + private static final String TINK_GCP_KMS_KEYRING_ID = "tink.gcp.kms.keyring.id"; + private static final String TINK_GCP_KMS_CRYPTOKEY_ID = "tink.gcp.kms.cryptokey.id"; + /** + * The Tink algorithm type. Defaults to AES256_GCM. For supported key templates, see + * https://google.github.io/tink/javadoc/tink/1.3.0/com/google/crypto/tink/aead/AeadKeyTemplates.html. + */ + private static final String TINK_GCP_KMS_KEY_ALGORITHM_KEY = "tink.gcp.kms.key.algorithm"; + + private CloudKmsClient cloudKmsClient; + + @Override + public String getName() { + return "tink-gcpkms"; + } + + @Override + public Aead initializeAead(AeadCipherContext context) throws CipherInitializationException { + validateProperties(context); + String projectId = context.getProperties().get(TINK_GCP_KMS_PROJECT_ID); + String location = context.getProperties().get(TINK_GCP_KMS_LOCATION); + String keyRingId = context.getProperties().get(TINK_GCP_KMS_KEYRING_ID); + String cryptoKeyId = context.getProperties().get(TINK_GCP_KMS_CRYPTOKEY_ID); + try { + cloudKmsClient = new CloudKmsClient(projectId, location, keyRingId); + } catch (IOException e) { + throw new CipherInitializationException("Failed to initialize Cloud KMS client", e); + } + // Create KMS KeyRing and CryptoKey if they do not exist. + cloudKmsClient.createKeyRingIfNotExists(); + cloudKmsClient.createCryptoKeyIfNotExists(cryptoKeyId); + try { + // Initialise Tink: register all AEAD key types with the Tink runtime + AeadConfig.register(); + } catch (GeneralSecurityException e) { + throw new CipherInitializationException("Failed to register Tink AEAD config", e); + } + String kekUri = String + .format(TINK_GCP_KMS_KEK_URI_FORMAT, projectId, location, keyRingId, cryptoKeyId); + Aead remoteAead; + try { + KmsClient client = new GcpKmsClient() + .withCredentials(GoogleCredentials.getApplicationDefault()); + remoteAead = client.getAead(kekUri); + } catch (GeneralSecurityException | IOException e) { + throw new CipherInitializationException("Failed to create GCP KMS client AEAD", e); + } + try { + return KmsEnvelopeAead.create(getAeadParameters(context.getProperties()), remoteAead); + } catch (GeneralSecurityException e) { + throw new CipherInitializationException("Failed to create Tink AEAD cipher", e); + } + } + + private void validateProperties(AeadCipherContext context) throws CipherInitializationException { + Map properties = context.getProperties(); + List invalidProperties = new ArrayList<>(); + String projectId = properties.get(TINK_GCP_KMS_PROJECT_ID); + if (projectId == null || projectId.isEmpty()) { + invalidProperties.add(TINK_GCP_KMS_PROJECT_ID); + } + String location = properties.get(TINK_GCP_KMS_LOCATION); + if (location == null || location.isEmpty()) { + invalidProperties.add(TINK_GCP_KMS_LOCATION); + } + String keyRingId = properties.get(TINK_GCP_KMS_KEYRING_ID); + if (keyRingId == null || keyRingId.isEmpty()) { + invalidProperties.add(TINK_GCP_KMS_KEYRING_ID); + } + String cryptoKeyId = properties.get(TINK_GCP_KMS_CRYPTOKEY_ID); + if (cryptoKeyId == null || cryptoKeyId.isEmpty()) { + invalidProperties.add(TINK_GCP_KMS_CRYPTOKEY_ID); + } + + if (!invalidProperties.isEmpty()) { + throw new CipherInitializationException( + String.format("The following properties must not be null or empty: %s", + invalidProperties.toArray())); + } + } + + private AeadParameters getAeadParameters(Map properties) + throws CipherInitializationException { + String key = properties.getOrDefault(TINK_GCP_KMS_KEY_ALGORITHM_KEY, "AES256_GCM"); + switch (key) { + case "AES128_GCM": + return PredefinedAeadParameters.AES128_GCM; + case "AES256_GCM": + return PredefinedAeadParameters.AES256_GCM; + case "AES128_EAX": + return PredefinedAeadParameters.AES128_EAX; + case "AES256_EAX": + return PredefinedAeadParameters.AES256_EAX; + case "AES128_CTR_HMAC_SHA256": + return PredefinedAeadParameters.AES128_CTR_HMAC_SHA256; + case "AES256_CTR_HMAC_SHA256": + return PredefinedAeadParameters.AES256_CTR_HMAC_SHA256; + case "CHACHA20_POLY1305": + return PredefinedAeadParameters.CHACHA20_POLY1305; + case "XCHACHA20_POLY1305": + return PredefinedAeadParameters.XCHACHA20_POLY1305; + default: + throw new CipherInitializationException( + String.format("Unexpected AEAD algorithm: '%s'", key)); + } + } +} diff --git a/cdap-encryption-ext-tink/src/main/resources/META-INF/services/io.cdap.cdap.security.spi.encryption.AeadCipherCryptor b/cdap-encryption-ext-tink/src/main/resources/META-INF/services/io.cdap.cdap.security.spi.encryption.AeadCipherCryptor new file mode 100644 index 000000000000..c18081e0cbc5 --- /dev/null +++ b/cdap-encryption-ext-tink/src/main/resources/META-INF/services/io.cdap.cdap.security.spi.encryption.AeadCipherCryptor @@ -0,0 +1,18 @@ +# +# Copyright © 2022 Cask Data, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# + +io.cdap.cdap.encryption.tink.CleartextTinkCipherCryptor +io.cdap.cdap.encryption.tink.GcpEnvelopeTinkCipherCryptor diff --git a/cdap-encryption-ext-tink/src/test/java/io/cdap/cdap/encryption/tink/CleartextTinkCipherTest.java b/cdap-encryption-ext-tink/src/test/java/io/cdap/cdap/encryption/tink/CleartextTinkCipherTest.java new file mode 100644 index 000000000000..73da0f16b00d --- /dev/null +++ b/cdap-encryption-ext-tink/src/test/java/io/cdap/cdap/encryption/tink/CleartextTinkCipherTest.java @@ -0,0 +1,113 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.encryption.tink; + +import com.google.crypto.tink.CleartextKeysetHandle; +import com.google.crypto.tink.JsonKeysetWriter; +import com.google.crypto.tink.KeyTemplates; +import com.google.crypto.tink.KeysetHandle; +import com.google.crypto.tink.aead.AeadConfig; +import io.cdap.cdap.security.spi.encryption.AeadCipherContext; +import io.cdap.cdap.security.spi.encryption.CipherInitializationException; +import io.cdap.cdap.security.spi.encryption.CipherOperationException; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.security.GeneralSecurityException; +import java.util.HashMap; +import java.util.Map; +import java.util.Random; +import org.junit.Assert; +import org.junit.Test; + +public class CleartextTinkCipherTest { + + @Test + public void testEncryptionAndDecryption() throws CipherInitializationException, + CipherOperationException, IOException, GeneralSecurityException { + Map properties = new HashMap<>(); + Map secureProperties = new HashMap<>(); + secureProperties.put(CleartextTinkCipherCryptor.TINK_CLEARTEXT_KEYSET_KEY, generateKeySet()); + + CleartextTinkCipherCryptor cipher = new CleartextTinkCipherCryptor(); + cipher.initialize(new AeadCipherContext(properties, secureProperties)); + + byte[] plainData = generateRandomBytes(2 * 1024); + byte[] associatedData = generateRandomBytes(64); + byte[] cipherData = cipher.encrypt(plainData, associatedData); + byte[] decryptedData = cipher.decrypt(cipherData, associatedData); + Assert.assertArrayEquals(plainData, decryptedData); + } + + @Test(expected = CipherInitializationException.class) + public void testInitException() throws CipherInitializationException { + Map properties = new HashMap<>(); + Map secureProperties = new HashMap<>(); + secureProperties.put(CleartextTinkCipherCryptor.TINK_CLEARTEXT_KEYSET_KEY, "invalid-keyset"); + + CleartextTinkCipherCryptor cipher = new CleartextTinkCipherCryptor(); + cipher.initialize(new AeadCipherContext(properties, secureProperties)); + } + + @Test(expected = CipherOperationException.class) + public void testDecryptExceptionTagMismatch() throws CipherInitializationException, + CipherOperationException, IOException, GeneralSecurityException { + Map properties = new HashMap<>(); + Map secureProperties = new HashMap<>(); + secureProperties.put(CleartextTinkCipherCryptor.TINK_CLEARTEXT_KEYSET_KEY, generateKeySet()); + + CleartextTinkCipherCryptor cipher = new CleartextTinkCipherCryptor(); + cipher.initialize(new AeadCipherContext(properties, secureProperties)); + + byte[] plainData = generateRandomBytes(128); + byte[] associatedData = generateRandomBytes(64); + byte[] cipherData = cipher.encrypt(plainData, associatedData); + byte[] invalidAssociatedData = generateRandomBytes(64); + cipher.decrypt(cipherData, invalidAssociatedData); + } + + @Test(expected = CipherOperationException.class) + public void testDecryptExceptionCorruption() throws CipherInitializationException, + CipherOperationException, IOException, GeneralSecurityException { + Map properties = new HashMap<>(); + Map secureProperties = new HashMap<>(); + secureProperties.put(CleartextTinkCipherCryptor.TINK_CLEARTEXT_KEYSET_KEY, generateKeySet()); + + CleartextTinkCipherCryptor cipher = new CleartextTinkCipherCryptor(); + cipher.initialize(new AeadCipherContext(properties, secureProperties)); + + byte[] plainData = generateRandomBytes(128); + byte[] associatedData = generateRandomBytes(64); + byte[] cipherData = cipher.encrypt(plainData, associatedData); + // Intentionally corrupt the cipher data. + cipherData[0] = 0; + byte[] decryptedData = cipher.decrypt(cipherData, associatedData); + } + + private String generateKeySet() throws IOException, GeneralSecurityException { + AeadConfig.register(); + KeysetHandle keysetHandle = KeysetHandle.generateNew(KeyTemplates.get("AES128_GCM")); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + CleartextKeysetHandle.write(keysetHandle, JsonKeysetWriter.withOutputStream(outputStream)); + return outputStream.toString(); + } + + private byte[] generateRandomBytes(int len) { + byte[] bytes = new byte[len]; + new Random().nextBytes(bytes); + return bytes; + } +} diff --git a/cdap-gateway/src/main/java/io/cdap/cdap/gateway/router/NettyRouter.java b/cdap-gateway/src/main/java/io/cdap/cdap/gateway/router/NettyRouter.java index f107af0aeb3c..415fe221a988 100644 --- a/cdap-gateway/src/main/java/io/cdap/cdap/gateway/router/NettyRouter.java +++ b/cdap-gateway/src/main/java/io/cdap/cdap/gateway/router/NettyRouter.java @@ -35,6 +35,8 @@ import io.cdap.cdap.gateway.router.handlers.HttpStatusRequestHandler; import io.cdap.cdap.security.auth.TokenValidator; import io.cdap.cdap.security.auth.UserIdentityExtractor; +import io.cdap.cdap.security.encryption.AeadCipher; +import io.cdap.cdap.security.encryption.guice.UserCredentialAeadEncryptionModule; import io.cdap.cdap.security.impersonation.SecurityUtil; import io.cdap.http.SSLConfig; import io.cdap.http.SSLHandlerFactory; @@ -94,6 +96,7 @@ public class NettyRouter extends AbstractIdleService { private final UserIdentityExtractor userIdentityExtractor; private final boolean sslEnabled; private final DiscoveryServiceClient discoveryServiceClient; + private final AeadCipher userCredentialAeadCipher; private InetSocketAddress boundAddress; private Cancellable serverCancellable; @@ -105,7 +108,9 @@ public NettyRouter(CConfiguration cConf, SConfiguration sConf, @Named(Constants.Router.ADDRESS) InetAddress hostname, RouterServiceLookup serviceLookup, TokenValidator tokenValidator, UserIdentityExtractor userIdentityExtractor, - DiscoveryServiceClient discoveryServiceClient) { + DiscoveryServiceClient discoveryServiceClient, + @Named(UserCredentialAeadEncryptionModule.USER_CREDENTIAL_ENCRYPTION) + AeadCipher userCredentialAeadCipher) { this.cConf = cConf; this.sConf = sConf; this.serverBossThreadPoolSize = cConf.getInt(Constants.Router.SERVER_BOSS_THREADS); @@ -121,6 +126,7 @@ public NettyRouter(CConfiguration cConf, SConfiguration sConf, this.port = sslEnabled ? cConf.getInt(Constants.Router.ROUTER_SSL_PORT) : cConf.getInt(Constants.Router.ROUTER_PORT); + this.userCredentialAeadCipher = userCredentialAeadCipher; } /** @@ -225,7 +231,7 @@ protected void initChannel(SocketChannel ch) { if (securityEnabled) { pipeline.addLast("access-token-authenticator", new AuthenticationHandler(cConf, sConf, discoveryServiceClient, - userIdentityExtractor)); + userIdentityExtractor, userCredentialAeadCipher)); } if (cConf.getBoolean(Constants.Router.ROUTER_AUDIT_LOG_ENABLED)) { pipeline.addLast("audit-log", new AuditLogHandler()); diff --git a/cdap-gateway/src/main/java/io/cdap/cdap/gateway/router/RouterMain.java b/cdap-gateway/src/main/java/io/cdap/cdap/gateway/router/RouterMain.java index d92f185e6c3f..d3348d0d4e83 100644 --- a/cdap-gateway/src/main/java/io/cdap/cdap/gateway/router/RouterMain.java +++ b/cdap-gateway/src/main/java/io/cdap/cdap/gateway/router/RouterMain.java @@ -29,6 +29,7 @@ import io.cdap.cdap.common.guice.ZkClientModule; import io.cdap.cdap.common.guice.ZkDiscoveryModule; import io.cdap.cdap.common.runtime.DaemonMain; +import io.cdap.cdap.security.encryption.guice.UserCredentialAeadEncryptionModule; import io.cdap.cdap.security.guice.CoreSecurityRuntimeModule; import io.cdap.cdap.security.guice.ExternalAuthenticationModule; import io.cdap.cdap.security.impersonation.SecurityUtil; @@ -140,7 +141,8 @@ static Injector createGuiceInjector(CConfiguration cConf) { new RouterModules().getDistributedModules(), CoreSecurityRuntimeModule.getDistributedModule(cConf), new ExternalAuthenticationModule(), - new IOModule() + new IOModule(), + new UserCredentialAeadEncryptionModule() ); } } diff --git a/cdap-gateway/src/main/java/io/cdap/cdap/gateway/router/handlers/AuthenticationHandler.java b/cdap-gateway/src/main/java/io/cdap/cdap/gateway/router/handlers/AuthenticationHandler.java index 2c5d5a9b03de..f408378e5650 100644 --- a/cdap-gateway/src/main/java/io/cdap/cdap/gateway/router/handlers/AuthenticationHandler.java +++ b/cdap-gateway/src/main/java/io/cdap/cdap/gateway/router/handlers/AuthenticationHandler.java @@ -22,18 +22,19 @@ import com.google.gson.JsonPrimitive; import io.cdap.cdap.common.conf.CConfiguration; import io.cdap.cdap.common.conf.Constants; +import io.cdap.cdap.common.conf.Constants.Security.Encryption; import io.cdap.cdap.common.conf.SConfiguration; import io.cdap.cdap.common.logging.AuditLogEntry; import io.cdap.cdap.common.utils.Networks; import io.cdap.cdap.proto.security.Credential; -import io.cdap.cdap.security.auth.CipherException; -import io.cdap.cdap.security.auth.TinkCipher; import io.cdap.cdap.security.auth.UserIdentity; import io.cdap.cdap.security.auth.UserIdentityExtractionResponse; import io.cdap.cdap.security.auth.UserIdentityExtractionState; import io.cdap.cdap.security.auth.UserIdentityExtractor; import io.cdap.cdap.security.auth.UserIdentityPair; +import io.cdap.cdap.security.encryption.AeadCipher; import io.cdap.cdap.security.server.GrantAccessToken; +import io.cdap.cdap.security.spi.encryption.CipherException; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import io.netty.channel.Channel; @@ -86,10 +87,12 @@ public class AuthenticationHandler extends ChannelInboundHandlerAdapter { private final List authServerURLs; private final DiscoveryServiceClient discoveryServiceClient; private final UserIdentityExtractor userIdentityExtractor; + private final AeadCipher userCredentialAeadCipher; public AuthenticationHandler(CConfiguration cConf, SConfiguration sConf, DiscoveryServiceClient discoveryServiceClient, - UserIdentityExtractor userIdentityExtractor) { + UserIdentityExtractor userIdentityExtractor, + AeadCipher userCredentialAeadCipher) { this.cConf = cConf; this.sConf = sConf; this.realm = cConf.get(Constants.Security.CFG_REALM); @@ -98,6 +101,7 @@ public AuthenticationHandler(CConfiguration cConf, SConfiguration sConf, this.authServerURLs = getConfiguredAuthServerURLs(cConf); this.discoveryServiceClient = discoveryServiceClient; this.userIdentityExtractor = userIdentityExtractor; + this.userCredentialAeadCipher = userCredentialAeadCipher; } @Override @@ -239,7 +243,8 @@ public void onChange(ServiceDiscovered serviceDiscovered) { * Get user credential from {@link UserIdentityPair} and return it in encrypted form if enabled. */ @Nullable - private Credential getUserCredential(UserIdentityPair userIdentityPair) throws CipherException { + private Credential getUserCredential(UserIdentityPair userIdentityPair) + throws CipherException { String userCredential = userIdentityPair.getUserCredential(); UserIdentity userIdentity = userIdentityPair.getUserIdentity(); if (userIdentity.getIdentifierType() == UserIdentity.IdentifierType.INTERNAL) { @@ -250,7 +255,9 @@ private Credential getUserCredential(UserIdentityPair userIdentityPair) throws C false)) { return new Credential(userCredential, Credential.CredentialType.EXTERNAL); } - String encryptedCredential = new TinkCipher(sConf).encryptStringToBase64(userCredential, null); + String encryptedCredential = userCredentialAeadCipher + .encryptToBase64(userCredential, + Encryption.USER_CREDENTIAL_ENCRYPTION_ASSOCIATED_DATA.getBytes()); return new Credential(encryptedCredential, Credential.CredentialType.EXTERNAL_ENCRYPTED); } diff --git a/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/AuditLogTest.java b/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/AuditLogTest.java index d9d43826db1c..90fefaf6971f 100644 --- a/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/AuditLogTest.java +++ b/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/AuditLogTest.java @@ -31,6 +31,7 @@ import io.cdap.cdap.common.security.AuditDetail; import io.cdap.cdap.common.security.AuditPolicy; import io.cdap.cdap.security.auth.TokenValidator; +import io.cdap.cdap.security.encryption.NoOpAeadCipher; import io.cdap.http.AbstractHttpHandler; import io.cdap.http.HttpResponder; import io.cdap.http.NettyHttpService; @@ -82,7 +83,8 @@ public class AuditLogTest { public static void init() throws Exception { // Configure a log appender programmatically for the audit log TestLogAppender.addAppender(Constants.Router.AUDIT_LOGGER_NAME); - ((ch.qos.logback.classic.Logger) LoggerFactory.getLogger(Constants.Router.AUDIT_LOGGER_NAME)).setLevel(Level.TRACE); + ((ch.qos.logback.classic.Logger) LoggerFactory.getLogger(Constants.Router.AUDIT_LOGGER_NAME)) + .setLevel(Level.TRACE); CConfiguration cConf = CConfiguration.create(); SConfiguration sConf = SConfiguration.create(); @@ -93,18 +95,21 @@ public static void init() throws Exception { InMemoryDiscoveryService discoveryService = new InMemoryDiscoveryService(); - RouterServiceLookup serviceLookup = new RouterServiceLookup(cConf, discoveryService, new RouterPathLookup()); + RouterServiceLookup serviceLookup = new RouterServiceLookup(cConf, discoveryService, + new RouterPathLookup()); TokenValidator successValidator = new SuccessTokenValidator(); - router = new NettyRouter(cConf, sConf, InetAddress.getLoopbackAddress(), serviceLookup, successValidator, - new MockAccessTokenIdentityExtractor(successValidator), discoveryService); + router = new NettyRouter(cConf, sConf, InetAddress.getLoopbackAddress(), serviceLookup, + successValidator, + new MockAccessTokenIdentityExtractor(successValidator), discoveryService, + new NoOpAeadCipher()); router.startAndWait(); httpService = NettyHttpService.builder("test").setHttpHandlers(new TestHandler()).build(); httpService.start(); cancelDiscovery = discoveryService.register(new Discoverable(Constants.Service.APP_FABRIC_HTTP, - httpService.getBindAddress())); + httpService.getBindAddress())); int port = router.getBoundAddress().orElseThrow(IllegalStateException::new).getPort(); baseURI = URI.create(String.format("http://%s:%d", cConf.get(Constants.Router.ADDRESS), port)); @@ -126,20 +131,23 @@ public void testAuditLog() throws IOException { urlConn = createURLConnection("/put", HttpMethod.PUT); urlConn.getOutputStream().write("Test Put".getBytes(StandardCharsets.UTF_8)); Assert.assertEquals(200, urlConn.getResponseCode()); - Assert.assertEquals("Test Put", new String(ByteStreams.toByteArray(urlConn.getInputStream()), "UTF-8")); + Assert.assertEquals("Test Put", + new String(ByteStreams.toByteArray(urlConn.getInputStream()), "UTF-8")); urlConn.getInputStream().close(); urlConn = createURLConnection("/post", HttpMethod.POST); urlConn.getOutputStream().write("Test Post".getBytes(StandardCharsets.UTF_8)); Assert.assertEquals(200, urlConn.getResponseCode()); - Assert.assertEquals("Test Post", new String(ByteStreams.toByteArray(urlConn.getInputStream()), "UTF-8")); + Assert.assertEquals("Test Post", + new String(ByteStreams.toByteArray(urlConn.getInputStream()), "UTF-8")); urlConn.getInputStream().close(); urlConn = createURLConnection("/postHeaders", HttpMethod.POST); urlConn.setRequestProperty("user-id", "cdap"); urlConn.getOutputStream().write("Post Headers".getBytes(StandardCharsets.UTF_8)); Assert.assertEquals(200, urlConn.getResponseCode()); - Assert.assertEquals("Post Headers", new String(ByteStreams.toByteArray(urlConn.getInputStream()), "UTF-8")); + Assert.assertEquals("Post Headers", + new String(ByteStreams.toByteArray(urlConn.getInputStream()), "UTF-8")); urlConn.getInputStream().close(); List loggedMessages = TestLogAppender.INSTANCE.getLoggedMessages(); @@ -147,9 +155,11 @@ public void testAuditLog() throws IOException { Assert.assertTrue(loggedMessages.get(0).endsWith("\"GET /get HTTP/1.1\" - - 200 0 -")); Assert.assertTrue(loggedMessages.get(1).endsWith("\"PUT /put HTTP/1.1\" - Test Put 200 8 -")); - Assert.assertTrue(loggedMessages.get(2).endsWith("\"POST /post HTTP/1.1\" - Test Post 200 9 Test Post")); Assert.assertTrue( - loggedMessages.get(3).endsWith("\"POST /postHeaders HTTP/1.1\" {user-id=cdap} Post Headers 200 12 Post Headers")); + loggedMessages.get(2).endsWith("\"POST /post HTTP/1.1\" - Test Post 200 9 Test Post")); + Assert.assertTrue( + loggedMessages.get(3).endsWith( + "\"POST /postHeaders HTTP/1.1\" {user-id=cdap} Post Headers 200 12 Post Headers")); } private HttpURLConnection createURLConnection(String path, HttpMethod method) throws IOException { @@ -177,21 +187,25 @@ public void get(HttpRequest request, HttpResponder responder, @QueryParam("q") S @PUT @AuditPolicy(AuditDetail.REQUEST_BODY) public void put(FullHttpRequest request, HttpResponder responder) { - responder.sendContent(HttpResponseStatus.OK, request.content().retainedDuplicate(), EmptyHttpHeaders.INSTANCE); + responder.sendContent(HttpResponseStatus.OK, request.content().retainedDuplicate(), + EmptyHttpHeaders.INSTANCE); } @Path("/post") @POST @AuditPolicy({AuditDetail.REQUEST_BODY, AuditDetail.RESPONSE_BODY}) public void post(FullHttpRequest request, HttpResponder responder) { - responder.sendContent(HttpResponseStatus.OK, request.content().retainedDuplicate(), EmptyHttpHeaders.INSTANCE); + responder.sendContent(HttpResponseStatus.OK, request.content().retainedDuplicate(), + EmptyHttpHeaders.INSTANCE); } @Path("/postHeaders") @POST @AuditPolicy({AuditDetail.REQUEST_BODY, AuditDetail.RESPONSE_BODY, AuditDetail.HEADERS}) - public void postHeaders(FullHttpRequest request, HttpResponder responder, @HeaderParam("user-id") String userId) { - responder.sendContent(HttpResponseStatus.OK, request.content().retainedDuplicate(), EmptyHttpHeaders.INSTANCE); + public void postHeaders(FullHttpRequest request, HttpResponder responder, + @HeaderParam("user-id") String userId) { + responder.sendContent(HttpResponseStatus.OK, request.content().retainedDuplicate(), + EmptyHttpHeaders.INSTANCE); } } @@ -208,7 +222,6 @@ static void addAppender(String loggerName) { LoggerContext loggerContext = (LoggerContext) LoggerFactory.getILoggerFactory(); Logger logger = loggerContext.getLogger(loggerName); - // Check if the logger already contains the logAppender if (Iterators.contains(logger.iteratorForAppenders(), INSTANCE)) { return; diff --git a/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/AuthServerAnnounceTest.java b/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/AuthServerAnnounceTest.java index bbdc23f323d0..79591fd481e9 100644 --- a/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/AuthServerAnnounceTest.java +++ b/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/AuthServerAnnounceTest.java @@ -31,6 +31,7 @@ import io.cdap.cdap.security.auth.AuthenticationMode; import io.cdap.cdap.security.auth.TokenValidator; import io.cdap.cdap.security.auth.UserIdentityExtractor; +import io.cdap.cdap.security.encryption.NoOpAeadCipher; import io.cdap.cdap.security.guice.CoreSecurityRuntimeModule; import io.cdap.cdap.security.guice.ExternalAuthenticationModule; import io.cdap.cdap.security.server.GrantAccessToken; @@ -55,16 +56,19 @@ import org.junit.Test; public class AuthServerAnnounceTest { + private static final String HOSTNAME = "127.0.0.1"; private static final int CONNECTION_IDLE_TIMEOUT_SECS = 2; private static final DiscoveryService DISCOVERY_SERVICE = new InMemoryDiscoveryService(); private static final String ANNOUNCE_URLS = "https://vip.cask.co:80,http://vip.cask.co:1000"; private static final Gson GSON = new Gson(); - private static final Type TYPE = new TypeToken>>() { }.getType(); + private static final Type TYPE = new TypeToken>>() { + }.getType(); @Test public void testEmptyAnnounceAddressURLsConfig() throws Exception { - HttpRouterService routerService = new AuthServerAnnounceTest.HttpRouterService(HOSTNAME, DISCOVERY_SERVICE); + HttpRouterService routerService = new AuthServerAnnounceTest.HttpRouterService(HOSTNAME, + DISCOVERY_SERVICE); routerService.startUp(); try { Assert.assertEquals(Collections.EMPTY_LIST, getAuthURI(routerService)); @@ -75,36 +79,40 @@ public void testEmptyAnnounceAddressURLsConfig() throws Exception { @Test public void testAnnounceURLsConfig() throws Exception { - HttpRouterService routerService = new AuthServerAnnounceTest.HttpRouterService(HOSTNAME, DISCOVERY_SERVICE); + HttpRouterService routerService = new AuthServerAnnounceTest.HttpRouterService(HOSTNAME, + DISCOVERY_SERVICE); routerService.cConf.set(Constants.Security.AUTH_SERVER_ANNOUNCE_URLS, ANNOUNCE_URLS); routerService.startUp(); try { List expected = Stream.of(ANNOUNCE_URLS.split(",")) - .map(url -> String.format("%s/%s", url, GrantAccessToken.Paths.GET_TOKEN)) - .collect(Collectors.toList()); + .map(url -> String.format("%s/%s", url, GrantAccessToken.Paths.GET_TOKEN)) + .collect(Collectors.toList()); Assert.assertEquals(expected, getAuthURI(routerService)); } finally { routerService.shutDown(); } } - private List getAuthURI(HttpRouterService routerService) throws IOException, URISyntaxException { + private List getAuthURI(HttpRouterService routerService) + throws IOException, URISyntaxException { DefaultHttpClient client = new DefaultHttpClient(); String url = resolveURI("/v3/apps", routerService); HttpGet get = new HttpGet(url); HttpResponse response = client.execute(get); Map> responseMap = - GSON.fromJson(new InputStreamReader(response.getEntity().getContent()), TYPE); + GSON.fromJson(new InputStreamReader(response.getEntity().getContent()), TYPE); return responseMap.get("auth_uri"); } - private String resolveURI(String path, HttpRouterService routerService) throws URISyntaxException { + private String resolveURI(String path, HttpRouterService routerService) + throws URISyntaxException { InetSocketAddress address = routerService.getRouterAddress(); return new URI(String.format("%s://%s:%d", "http", address.getHostName(), - address.getPort())).resolve(path).toASCIIString(); + address.getPort())).resolve(path).toASCIIString(); } private static class HttpRouterService extends AbstractIdleService { + private final String hostname; private final DiscoveryService discoveryService; private CConfiguration cConf = CConfiguration.create(); @@ -119,10 +127,11 @@ private HttpRouterService(String hostname, DiscoveryService discoveryService) { protected void startUp() { SConfiguration sConfiguration = SConfiguration.create(); Injector injector = Guice.createInjector(new CoreSecurityRuntimeModule().getInMemoryModules(), - new ExternalAuthenticationModule(), - new InMemoryDiscoveryModule(), - new AppFabricTestModule(cConf)); - DiscoveryServiceClient discoveryServiceClient = injector.getInstance(DiscoveryServiceClient.class); + new ExternalAuthenticationModule(), + new InMemoryDiscoveryModule(), + new AppFabricTestModule(cConf)); + DiscoveryServiceClient discoveryServiceClient = injector + .getInstance(DiscoveryServiceClient.class); TokenValidator validator = new MissingTokenValidator(); UserIdentityExtractor userIdentityExtractor = new MockAccessTokenIdentityExtractor(validator); cConf.set(Constants.Router.ADDRESS, hostname); @@ -132,10 +141,10 @@ protected void startUp() { cConf.setEnum(Constants.Security.Authentication.MODE, AuthenticationMode.MANAGED); router = - new NettyRouter(cConf, sConfiguration, InetAddresses.forString(hostname), - new RouterServiceLookup(cConf, (DiscoveryServiceClient) discoveryService, - new RouterPathLookup()), - validator, userIdentityExtractor, discoveryServiceClient); + new NettyRouter(cConf, sConfiguration, InetAddresses.forString(hostname), + new RouterServiceLookup(cConf, (DiscoveryServiceClient) discoveryService, + new RouterPathLookup()), + validator, userIdentityExtractor, discoveryServiceClient, new NoOpAeadCipher()); router.startAndWait(); } diff --git a/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/ConfigBasedRequestBlockingTest.java b/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/ConfigBasedRequestBlockingTest.java index 5e5bb32b7446..a8c09c9535c2 100644 --- a/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/ConfigBasedRequestBlockingTest.java +++ b/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/ConfigBasedRequestBlockingTest.java @@ -22,6 +22,7 @@ import io.cdap.cdap.common.conf.Constants; import io.cdap.cdap.common.conf.SConfiguration; import io.cdap.cdap.security.auth.TokenValidator; +import io.cdap.cdap.security.encryption.NoOpAeadCipher; import io.cdap.http.NettyHttpService; import io.netty.handler.codec.http.HttpResponseStatus; import java.io.IOException; @@ -63,16 +64,18 @@ public static void init() throws Exception { TokenValidator successValidator = new SuccessTokenValidator(); router = new NettyRouter(cConf, SConfiguration.create(), InetAddress.getLoopbackAddress(), - new RouterServiceLookup(cConf, discoveryService, new RouterPathLookup()), - successValidator, - new MockAccessTokenIdentityExtractor(successValidator), discoveryService); + new RouterServiceLookup(cConf, discoveryService, new RouterPathLookup()), + successValidator, + new MockAccessTokenIdentityExtractor(successValidator), discoveryService, + new NoOpAeadCipher()); router.startAndWait(); - httpService = NettyHttpService.builder("test").setHttpHandlers(new AuditLogTest.TestHandler()).build(); + httpService = NettyHttpService.builder("test").setHttpHandlers(new AuditLogTest.TestHandler()) + .build(); httpService.start(); cancelDiscovery = discoveryService.register(new Discoverable(Constants.Service.APP_FABRIC_HTTP, - httpService.getBindAddress())); + httpService.getBindAddress())); } @Test @@ -95,7 +98,7 @@ public void testResponseBody() throws Exception { // Custom message passed in config cConf.set(Constants.Router.BLOCK_REQUEST_MESSAGE, "custom message"); testGet(cConf.getInt(Constants.Router.BLOCK_REQUEST_STATUS_CODE), - cConf.get(Constants.Router.BLOCK_REQUEST_MESSAGE), "/get"); + cConf.get(Constants.Router.BLOCK_REQUEST_MESSAGE), "/get"); } @Test @@ -122,7 +125,7 @@ public static void finish() throws Exception { } private void testGet(int expectedStatus, String expectedResponse, String path) - throws Exception { + throws Exception { InetSocketAddress address = router.getBoundAddress().orElseThrow(IllegalStateException::new); URL url = new URL("http", address.getHostName(), address.getPort(), path); @@ -146,7 +149,8 @@ private void testGet(int expectedStatus, String expectedResponse, String path) // but Error Stream won't be populated so rely on content-length header instead Assert.assertEquals("0", connection.getHeaderField("content-length")); } else { - Assert.assertEquals(expectedResponse, Bytes.toString(ByteStreams.toByteArray(inputStream))); + Assert + .assertEquals(expectedResponse, Bytes.toString(ByteStreams.toByteArray(inputStream))); } } Assert.assertEquals(expectedStatus, connection.getResponseCode()); diff --git a/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/NettyRouterHttpTest.java b/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/NettyRouterHttpTest.java index 72944ad06ae0..488c0365ead5 100644 --- a/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/NettyRouterHttpTest.java +++ b/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/NettyRouterHttpTest.java @@ -25,6 +25,7 @@ import io.cdap.cdap.common.guice.InMemoryDiscoveryModule; import io.cdap.cdap.internal.guice.AppFabricTestModule; import io.cdap.cdap.security.auth.UserIdentityExtractor; +import io.cdap.cdap.security.encryption.NoOpAeadCipher; import io.cdap.cdap.security.guice.CoreSecurityRuntimeModule; import io.cdap.cdap.security.guice.ExternalAuthenticationModule; import java.net.InetSocketAddress; @@ -60,6 +61,7 @@ protected SocketFactory getSocketFactory() { } private static class HttpRouterService extends RouterService { + private final String hostname; private final DiscoveryService discoveryService; @@ -75,19 +77,22 @@ protected void startUp() { CConfiguration cConf = CConfiguration.create(); SConfiguration sConfiguration = SConfiguration.create(); Injector injector = Guice.createInjector(new CoreSecurityRuntimeModule().getInMemoryModules(), - new ExternalAuthenticationModule(), - new InMemoryDiscoveryModule(), - new AppFabricTestModule(cConf)); - DiscoveryServiceClient discoveryServiceClient = injector.getInstance(DiscoveryServiceClient.class); - UserIdentityExtractor userIdentityExtractor = injector.getInstance(UserIdentityExtractor.class); + new ExternalAuthenticationModule(), + new InMemoryDiscoveryModule(), + new AppFabricTestModule(cConf)); + DiscoveryServiceClient discoveryServiceClient = injector + .getInstance(DiscoveryServiceClient.class); + UserIdentityExtractor userIdentityExtractor = injector + .getInstance(UserIdentityExtractor.class); cConf.set(Constants.Router.ADDRESS, hostname); cConf.setInt(Constants.Router.ROUTER_PORT, 0); cConf.setInt(Constants.Router.CONNECTION_TIMEOUT_SECS, CONNECTION_IDLE_TIMEOUT_SECS); router = - new NettyRouter(cConf, sConfiguration, InetAddresses.forString(hostname), - new RouterServiceLookup(cConf, (DiscoveryServiceClient) discoveryService, - new RouterPathLookup()), - new SuccessTokenValidator(), userIdentityExtractor, discoveryServiceClient); + new NettyRouter(cConf, sConfiguration, InetAddresses.forString(hostname), + new RouterServiceLookup(cConf, (DiscoveryServiceClient) discoveryService, + new RouterPathLookup()), + new SuccessTokenValidator(), userIdentityExtractor, discoveryServiceClient, + new NoOpAeadCipher()); router.startAndWait(); } diff --git a/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/NettyRouterHttpsTest.java b/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/NettyRouterHttpsTest.java index edc3a86c5a0c..cdb2941afe13 100644 --- a/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/NettyRouterHttpsTest.java +++ b/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/NettyRouterHttpsTest.java @@ -27,6 +27,7 @@ import io.cdap.cdap.common.security.KeyStoresTest; import io.cdap.cdap.internal.guice.AppFabricTestModule; import io.cdap.cdap.security.auth.UserIdentityExtractor; +import io.cdap.cdap.security.encryption.NoOpAeadCipher; import io.cdap.cdap.security.guice.CoreSecurityRuntimeModule; import io.cdap.cdap.security.guice.ExternalAuthenticationModule; import io.cdap.common.http.HttpRequests; @@ -67,9 +68,9 @@ public class NettyRouterHttpsTest extends NettyRouterTestBase { @Parameterized.Parameters(name = "{index}: NettyRouterHttpsTest(useKeyStore = {0})") public static Collection parameters() { - return Arrays.asList(new Object[][] { - {true}, - {false} + return Arrays.asList(new Object[][]{ + {true}, + {false} }); } @@ -98,7 +99,7 @@ public NettyRouterHttpsTest(boolean useKeyStore) throws Exception { sConf.set(Constants.Security.Router.SSL_KEYSTORE_PATH, keyStoreFile.getAbsolutePath()); } else { File pemFile = KeyStoresTest.writePEMFile(TEMP_FOLDER.newFile(), keyStore, - keyStore.aliases().nextElement(), keyStorePass); + keyStore.aliases().nextElement(), keyStorePass); cConf.set(Constants.Security.Router.SSL_CERT_PATH, pemFile.getAbsolutePath()); sConf.set(Constants.Security.Router.SSL_CERT_PASSWORD, keyStorePass); } @@ -126,7 +127,8 @@ protected DefaultHttpClient getHTTPClient() throws Exception { SSLContext sslContext = SSLContext.getInstance("TLS"); // set up a TrustManager that trusts everything - sslContext.init(null, InsecureTrustManagerFactory.INSTANCE.getTrustManagers(), new SecureRandom()); + sslContext + .init(null, InsecureTrustManagerFactory.INSTANCE.getTrustManagers(), new SecureRandom()); SSLSocketFactory sf = new SSLSocketFactory(sslContext, new AllowAllHostnameVerifier()); Scheme httpsScheme = new Scheme("https", 10101, sf); @@ -155,7 +157,7 @@ private static class HttpsRouterService extends RouterService { private NettyRouter router; private HttpsRouterService(CConfiguration cConf, SConfiguration sConf, - String hostname, DiscoveryService discoveryService) { + String hostname, DiscoveryService discoveryService) { this.cConf = CConfiguration.copy(cConf); this.sConf = SConfiguration.copy(sConf); this.hostname = hostname; @@ -165,18 +167,21 @@ private HttpsRouterService(CConfiguration cConf, SConfiguration sConf, @Override protected void startUp() { Injector injector = Guice.createInjector(new CoreSecurityRuntimeModule().getInMemoryModules(), - new ExternalAuthenticationModule(), - new InMemoryDiscoveryModule(), - new AppFabricTestModule(cConf)); - DiscoveryServiceClient discoveryServiceClient = injector.getInstance(DiscoveryServiceClient.class); - UserIdentityExtractor userIdentityExtractor = injector.getInstance(UserIdentityExtractor.class); + new ExternalAuthenticationModule(), + new InMemoryDiscoveryModule(), + new AppFabricTestModule(cConf)); + DiscoveryServiceClient discoveryServiceClient = injector + .getInstance(DiscoveryServiceClient.class); + UserIdentityExtractor userIdentityExtractor = injector + .getInstance(UserIdentityExtractor.class); cConf.set(Constants.Router.ADDRESS, hostname); router = - new NettyRouter(cConf, sConf, InetAddresses.forString(hostname), - new RouterServiceLookup(cConf, (DiscoveryServiceClient) discoveryService, - new RouterPathLookup()), - new SuccessTokenValidator(), userIdentityExtractor, discoveryServiceClient); + new NettyRouter(cConf, sConf, InetAddresses.forString(hostname), + new RouterServiceLookup(cConf, (DiscoveryServiceClient) discoveryService, + new RouterPathLookup()), + new SuccessTokenValidator(), userIdentityExtractor, discoveryServiceClient, + new NoOpAeadCipher()); router.startAndWait(); } diff --git a/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/NettyRouterTestBase.java b/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/NettyRouterTestBase.java index 521d96bb167d..bd0bc99a5979 100644 --- a/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/NettyRouterTestBase.java +++ b/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/NettyRouterTestBase.java @@ -39,6 +39,7 @@ import io.cdap.cdap.common.discovery.ResolvingDiscoverable; import io.cdap.cdap.common.http.AbstractBodyConsumer; import io.cdap.cdap.security.auth.TokenValidator; +import io.cdap.cdap.security.encryption.NoOpAeadCipher; import io.cdap.http.AbstractHttpHandler; import io.cdap.http.BodyConsumer; import io.cdap.http.ChannelPipelineModifier; @@ -118,23 +119,30 @@ public abstract class NettyRouterTestBase { private static final String HOSTNAME = InetAddress.getLoopbackAddress().getHostAddress(); private static final String APP_FABRIC_SERVICE = Constants.Service.APP_FABRIC_HTTP; private static final int MAX_UPLOAD_BYTES = 10 * 1024 * 1024; - private static final int CHUNK_SIZE = 1024 * 1024; // NOTE: MAX_UPLOAD_BYTES % CHUNK_SIZE == 0 + private static final int CHUNK_SIZE = + 1024 * 1024; // NOTE: MAX_UPLOAD_BYTES % CHUNK_SIZE == 0 private final DiscoveryService discoveryService = new InMemoryDiscoveryService(); - private final ServerService defaultServer1 = new ServerService(HOSTNAME, discoveryService, APP_FABRIC_SERVICE); - private final ServerService defaultServer2 = new ServerService(HOSTNAME, discoveryService, APP_FABRIC_SERVICE); + private final ServerService defaultServer1 = new ServerService(HOSTNAME, discoveryService, + APP_FABRIC_SERVICE); + private final ServerService defaultServer2 = new ServerService(HOSTNAME, discoveryService, + APP_FABRIC_SERVICE); private final List allServers = Lists.newArrayList(defaultServer1, defaultServer2); private RouterService routerService; - protected abstract RouterService createRouterService(String hostname, DiscoveryService discoveryService); + protected abstract RouterService createRouterService(String hostname, + DiscoveryService discoveryService); + protected abstract String getProtocol(); + protected abstract DefaultHttpClient getHTTPClient() throws Exception; + protected abstract SocketFactory getSocketFactory() throws Exception; private String resolveURI(String path) throws URISyntaxException { InetSocketAddress address = routerService.getRouterAddress(); return new URI(String.format("%s://%s:%d", getProtocol(), - address.getHostName(), address.getPort())).resolve(path).toASCIIString(); + address.getHostName(), address.getPort())).resolve(path).toASCIIString(); } @Before @@ -148,7 +156,8 @@ public void startUp() throws Exception { Futures.allAsList(futures).get(); // Wait for both servers of defaultService to be registered - ServiceDiscovered discover = ((DiscoveryServiceClient) discoveryService).discover(APP_FABRIC_SERVICE); + ServiceDiscovered discover = ((DiscoveryServiceClient) discoveryService) + .discover(APP_FABRIC_SERVICE); final CountDownLatch latch = new CountDownLatch(1); Cancellable cancellable = discover.watchChanges(new ServiceDiscovered.ChangeListener() { @Override @@ -185,16 +194,16 @@ public void testRouterAsync() throws Exception { AsyncHttpClientConfig.Builder configBuilder = new AsyncHttpClientConfig.Builder(); final AsyncHttpClient asyncHttpClient = new AsyncHttpClient( - new NettyAsyncHttpProvider(configBuilder.build()), - configBuilder.build()); + new NettyAsyncHttpProvider(configBuilder.build()), + configBuilder.build()); final CountDownLatch latch = new CountDownLatch(numElements); final AtomicInteger numSuccessfulRequests = new AtomicInteger(0); for (int i = 0; i < numElements; ++i) { final int elem = i; final Request request = new RequestBuilder("GET") - .setUrl(resolveURI(String.format("%s/%s-%d", "/v1/echo", "async", i))) - .build(); + .setUrl(resolveURI(String.format("%s/%s-%d", "/v1/echo", "async", i))) + .build(); asyncHttpClient.executeRequest(request, new AsyncCompletionHandler() { @Override public Void onCompleted(Response response) throws Exception { @@ -223,7 +232,8 @@ public void onThrowable(Throwable t) { Assert.assertEquals(numElements, numSuccessfulRequests.get()); // we use sticky endpoint strategy so the sum of requests from the two gateways should be NUM_ELEMENTS - Assert.assertEquals(numElements, (defaultServer1.getNumRequests() + defaultServer2.getNumRequests())); + Assert.assertEquals(numElements, + (defaultServer1.getNumRequests() + defaultServer2.getNumRequests())); } @Test @@ -265,30 +275,31 @@ public void testUpload() throws Exception { AsyncHttpClientConfig.Builder configBuilder = new AsyncHttpClientConfig.Builder(); final AsyncHttpClient asyncHttpClient = new AsyncHttpClient( - new NettyAsyncHttpProvider(configBuilder.build()), - configBuilder.build()); + new NettyAsyncHttpProvider(configBuilder.build()), + configBuilder.build()); - byte [] requestBody = generatePostData(); + byte[] requestBody = generatePostData(); final Request request = new RequestBuilder("POST") - .setUrl(resolveURI("/v1/upload")) - .setContentLength(requestBody.length) - .setBody(new ByteEntityWriter(requestBody)) - .build(); + .setUrl(resolveURI("/v1/upload")) + .setContentLength(requestBody.length) + .setBody(new ByteEntityWriter(requestBody)) + .build(); final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - Future future = asyncHttpClient.executeRequest(request, new AsyncCompletionHandler() { - @Override - public Void onCompleted(Response response) { - return null; - } + Future future = asyncHttpClient + .executeRequest(request, new AsyncCompletionHandler() { + @Override + public Void onCompleted(Response response) { + return null; + } - @Override - public STATE onBodyPartReceived(HttpResponseBodyPart content) throws Exception { - //TimeUnit.MILLISECONDS.sleep(RANDOM.nextInt(10)); - content.writeTo(byteArrayOutputStream); - return super.onBodyPartReceived(content); - } - }); + @Override + public STATE onBodyPartReceived(HttpResponseBodyPart content) throws Exception { + //TimeUnit.MILLISECONDS.sleep(RANDOM.nextInt(10)); + content.writeTo(byteArrayOutputStream); + return super.onBodyPartReceived(content); + } + }); future.get(); Assert.assertArrayEquals(requestBody, byteArrayOutputStream.toByteArray()); @@ -296,9 +307,9 @@ public STATE onBodyPartReceived(HttpResponseBodyPart content) throws Exception { @Test public void testConnectionClose() throws Exception { - URL[] urls = new URL[] { - new URL(resolveURI("/abc/v1/status")), - new URL(resolveURI("/def/v1/status")) + URL[] urls = new URL[]{ + new URL(resolveURI("/abc/v1/status")), + new URL(resolveURI("/def/v1/status")) }; // Make bunch of requests to one service to 2 difference urls, with the first one keep-alive, second one not. @@ -311,7 +322,7 @@ public void testConnectionClose() throws Exception { HttpURLConnection urlConn = openURL(urls[i % urls.length]); try { urlConn.setRequestProperty(HttpHeaderNames.CONNECTION.toString(), - (keepAlive ? HttpHeaderValues.KEEP_ALIVE : HttpHeaderValues.CLOSE).toString()); + (keepAlive ? HttpHeaderValues.KEEP_ALIVE : HttpHeaderValues.CLOSE).toString()); Assert.assertEquals(HttpURLConnection.HTTP_OK, urlConn.getResponseCode()); } finally { urlConn.getInputStream().close(); @@ -428,12 +439,13 @@ public void testConnectionIdleTimeoutWithMultipleServers() throws Exception { urlConnection.disconnect(); } - @Test (timeout = 5000L) + @Test(timeout = 5000L) public void testExpectContinue() throws Exception { URL url = new URL(resolveURI("/v2/upload")); HttpURLConnection urlConn = openURL(url); urlConn.setRequestMethod("POST"); - urlConn.setRequestProperty(HttpHeaderNames.EXPECT.toString(), HttpHeaderValues.CONTINUE.toString()); + urlConn.setRequestProperty(HttpHeaderNames.EXPECT.toString(), + HttpHeaderValues.CONTINUE.toString()); urlConn.setDoOutput(true); // Forces sending small chunks to have the netty server receives multiple chunks @@ -442,11 +454,12 @@ public void testExpectContinue() throws Exception { urlConn.getOutputStream().write(msg.getBytes(StandardCharsets.UTF_8)); Assert.assertEquals(200, urlConn.getResponseCode()); - String result = new String(ByteStreams.toByteArray(urlConn.getInputStream()), StandardCharsets.UTF_8); + String result = new String(ByteStreams.toByteArray(urlConn.getInputStream()), + StandardCharsets.UTF_8); Assert.assertEquals(msg, result); } - @Test (timeout = 5000L) + @Test(timeout = 5000L) public void testNotFound() throws Exception { URL url = new URL(resolveURI("/v1/not/exists")); HttpURLConnection urlConn = openURL(url); @@ -494,7 +507,8 @@ public void testConnectionClose2() throws Exception { // Make sure the discovery change is in effect Assert.assertNotNull(new RandomEndpointStrategy( - () -> ((DiscoveryServiceClient) discoveryService).discover(APP_FABRIC_SERVICE)).pick(5, TimeUnit.SECONDS)); + () -> ((DiscoveryServiceClient) discoveryService).discover(APP_FABRIC_SERVICE)) + .pick(5, TimeUnit.SECONDS)); // Make a call to sleep for couple seconds urlConn = openURL(url); @@ -546,20 +560,24 @@ public void testConfigReloading() throws Exception { CConfiguration cConfSpy1 = Mockito.spy(CConfiguration.create()); cConfSpy1.setLong(Constants.Router.CCONF_RELOAD_INTERVAL_SECONDS, reloadIntervalSeconds); cConfSpy1.setInt(Constants.Router.ROUTER_PORT, 0); - NettyRouter router1 = new NettyRouter(cConfSpy1, SConfiguration.create(), InetAddress.getLoopbackAddress(), - new RouterServiceLookup(cConfSpy1, discoveryService, new RouterPathLookup()), - successValidator, - new MockAccessTokenIdentityExtractor(successValidator), discoveryService); + NettyRouter router1 = new NettyRouter(cConfSpy1, SConfiguration.create(), + InetAddress.getLoopbackAddress(), + new RouterServiceLookup(cConfSpy1, discoveryService, new RouterPathLookup()), + successValidator, + new MockAccessTokenIdentityExtractor(successValidator), discoveryService, + new NoOpAeadCipher()); router1.startAndWait(); // Configure router with config-reloading time set to 0 CConfiguration cConfSpy2 = Mockito.spy(CConfiguration.create()); cConfSpy2.setLong(Constants.Router.CCONF_RELOAD_INTERVAL_SECONDS, 0); cConfSpy2.setInt(Constants.Router.ROUTER_PORT, 0); - NettyRouter router2 = new NettyRouter(cConfSpy2, SConfiguration.create(), InetAddress.getLoopbackAddress(), - new RouterServiceLookup(cConfSpy2, discoveryService, new RouterPathLookup()), - successValidator, - new MockAccessTokenIdentityExtractor(successValidator), discoveryService); + NettyRouter router2 = new NettyRouter(cConfSpy2, SConfiguration.create(), + InetAddress.getLoopbackAddress(), + new RouterServiceLookup(cConfSpy2, discoveryService, new RouterPathLookup()), + successValidator, + new MockAccessTokenIdentityExtractor(successValidator), discoveryService, + new NoOpAeadCipher()); router2.startAndWait(); // Wait sometime for cConf to reload @@ -587,12 +605,13 @@ private void testSyncServiceUnavailable() throws Exception { for (int i = 0; i < 25; ++i) { LOG.trace("Sending sync unavailable request " + i); HttpResponse response = get(resolveURI(String.format("%s/%s-%d", "/v1/ping", "sync", i))); - Assert.assertEquals(HttpResponseStatus.SERVICE_UNAVAILABLE.code(), response.getStatusLine().getStatusCode()); + Assert.assertEquals(HttpResponseStatus.SERVICE_UNAVAILABLE.code(), + response.getStatusLine().getStatusCode()); } } - private byte [] generatePostData() { - byte [] bytes = new byte [MAX_UPLOAD_BYTES]; + private byte[] generatePostData() { + byte[] bytes = new byte[MAX_UPLOAD_BYTES]; for (int i = 0; i < MAX_UPLOAD_BYTES; ++i) { bytes[i] = (byte) i; @@ -602,7 +621,8 @@ private void testSyncServiceUnavailable() throws Exception { } private static class ByteEntityWriter implements Request.EntityWriter { - private final byte [] bytes; + + private final byte[] bytes; private ByteEntityWriter(byte[] bytes) { this.bytes = bytes; @@ -633,6 +653,7 @@ private HttpResponse get(String url, Header[] headers) throws Exception { * A server for the router. */ public abstract static class RouterService extends AbstractIdleService { + public abstract InetSocketAddress getRouterAddress(); } @@ -640,6 +661,7 @@ public abstract static class RouterService extends AbstractIdleService { * A generic server for testing router. */ public static class ServerService extends AbstractIdleService { + private static final Logger log = LoggerFactory.getLogger(ServerService.class); private final String hostname; @@ -675,7 +697,8 @@ public void channelRegistered(ChannelHandlerContext ctx) throws Exception { } @Override - public void channelUnregistered(io.netty.channel.ChannelHandlerContext ctx) throws Exception { + public void channelUnregistered(io.netty.channel.ChannelHandlerContext ctx) + throws Exception { numConnectionsClosed.incrementAndGet(); super.channelInactive(ctx); } @@ -713,7 +736,7 @@ public void registerServer() { // Register services of test server log.info("Registering service {}", serviceName); cancelDiscovery = discoveryService.register( - ResolvingDiscoverable.of(new Discoverable(serviceName, httpService.getBindAddress()))); + ResolvingDiscoverable.of(new Discoverable(serviceName, httpService.getBindAddress()))); } public void cancelRegistration() { @@ -725,11 +748,14 @@ public void cancelRegistration() { * Simple handler for server. */ public class ServerHandler extends AbstractHttpHandler { + private final Logger log = LoggerFactory.getLogger(ServerHandler.class); + @GET @Path("/v1/echo/{text}") - public void echo(@SuppressWarnings("UnusedParameters") HttpRequest request, final HttpResponder responder, - @PathParam("text") String text) { + public void echo(@SuppressWarnings("UnusedParameters") HttpRequest request, + final HttpResponder responder, + @PathParam("text") String text) { numRequests.incrementAndGet(); log.trace("Got text {}", text); @@ -738,8 +764,9 @@ public void echo(@SuppressWarnings("UnusedParameters") HttpRequest request, fina @GET @Path("/v1/ping/{text}") - public void ping(@SuppressWarnings("UnusedParameters") HttpRequest request, final HttpResponder responder, - @PathParam("text") String text) { + public void ping(@SuppressWarnings("UnusedParameters") HttpRequest request, + final HttpResponder responder, + @PathParam("text") String text) { numRequests.incrementAndGet(); log.trace("Got text {}", text); @@ -748,8 +775,9 @@ public void ping(@SuppressWarnings("UnusedParameters") HttpRequest request, fina @GET @Path("/abc/v1/ping/{text}") - public void abcPing(@SuppressWarnings("UnusedParameters") HttpRequest request, final HttpResponder responder, - @PathParam("text") String text) { + public void abcPing(@SuppressWarnings("UnusedParameters") HttpRequest request, + final HttpResponder responder, + @PathParam("text") String text) { numRequests.incrementAndGet(); log.trace("Got text {}", text); @@ -758,8 +786,9 @@ public void abcPing(@SuppressWarnings("UnusedParameters") HttpRequest request, f @GET @Path("/def/v1/ping/{text}") - public void defPing(@SuppressWarnings("UnusedParameters") HttpRequest request, final HttpResponder responder, - @PathParam("text") String text) { + public void defPing(@SuppressWarnings("UnusedParameters") HttpRequest request, + final HttpResponder responder, + @PathParam("text") String text) { numRequests.incrementAndGet(); log.trace("Got text {}", text); @@ -768,7 +797,8 @@ public void defPing(@SuppressWarnings("UnusedParameters") HttpRequest request, f @GET @Path("/v2/ping") - public void gateway(@SuppressWarnings("UnusedParameters") HttpRequest request, final HttpResponder responder) { + public void gateway(@SuppressWarnings("UnusedParameters") HttpRequest request, + final HttpResponder responder) { numRequests.incrementAndGet(); responder.sendString(HttpResponseStatus.OK, serviceName); @@ -791,7 +821,7 @@ public void defStatus(HttpRequest request, HttpResponder responder) { @GET @Path("/v1/timeout/{timeout-millis}") public void timeout(HttpRequest request, HttpResponder responder, - @PathParam("timeout-millis") int timeoutMillis) throws InterruptedException { + @PathParam("timeout-millis") int timeoutMillis) throws InterruptedException { numRequests.incrementAndGet(); TimeUnit.MILLISECONDS.sleep(timeoutMillis); responder.sendStatus(HttpResponseStatus.OK); @@ -800,7 +830,7 @@ public void timeout(HttpRequest request, HttpResponder responder, @GET @Path("/v1/exception/{message}") public void exception(HttpRequest request, HttpResponder responder, - @PathParam("msg") String msg) throws Exception { + @PathParam("msg") String msg) throws Exception { throw new Exception(msg); } diff --git a/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/RouterResource.java b/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/RouterResource.java index 91904b68b1ae..90c2f640469d 100644 --- a/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/RouterResource.java +++ b/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/RouterResource.java @@ -26,6 +26,7 @@ import io.cdap.cdap.internal.guice.AppFabricTestModule; import io.cdap.cdap.security.auth.TokenValidator; import io.cdap.cdap.security.auth.UserIdentityExtractor; +import io.cdap.cdap.security.encryption.NoOpAeadCipher; import io.cdap.cdap.security.guice.CoreSecurityRuntimeModule; import io.cdap.cdap.security.guice.ExternalAuthenticationModule; import java.net.InetSocketAddress; @@ -36,6 +37,7 @@ import org.junit.rules.ExternalResource; class RouterResource extends ExternalResource { + private final String hostname; private final DiscoveryService discoveryService; private final Map additionalConfig; @@ -46,7 +48,8 @@ class RouterResource extends ExternalResource { this(hostname, discoveryService, new HashMap<>()); } - RouterResource(String hostname, DiscoveryService discoveryService, Map additionalConfig) { + RouterResource(String hostname, DiscoveryService discoveryService, + Map additionalConfig) { this.hostname = hostname; this.discoveryService = discoveryService; this.additionalConfig = additionalConfig; @@ -56,10 +59,11 @@ class RouterResource extends ExternalResource { protected void before() { CConfiguration cConf = CConfiguration.create(); Injector injector = Guice.createInjector(new CoreSecurityRuntimeModule().getStandaloneModules(), - new ExternalAuthenticationModule(), - new InMemoryDiscoveryModule(), - new AppFabricTestModule(cConf)); - DiscoveryServiceClient discoveryServiceClient = injector.getInstance(DiscoveryServiceClient.class); + new ExternalAuthenticationModule(), + new InMemoryDiscoveryModule(), + new AppFabricTestModule(cConf)); + DiscoveryServiceClient discoveryServiceClient = injector + .getInstance(DiscoveryServiceClient.class); TokenValidator mockValidator = new MockTokenValidator("failme"); UserIdentityExtractor extractor = new MockAccessTokenIdentityExtractor(mockValidator); SConfiguration sConf = injector.getInstance(SConfiguration.class); @@ -69,9 +73,10 @@ protected void before() { cConf.set(entry.getKey(), entry.getValue()); } router = - new NettyRouter(cConf, sConf, InetAddresses.forString(hostname), - new RouterServiceLookup(cConf, (DiscoveryServiceClient) discoveryService, new RouterPathLookup()), - mockValidator, extractor, discoveryServiceClient); + new NettyRouter(cConf, sConf, InetAddresses.forString(hostname), + new RouterServiceLookup(cConf, (DiscoveryServiceClient) discoveryService, + new RouterPathLookup()), + mockValidator, extractor, discoveryServiceClient, new NoOpAeadCipher()); router.startAndWait(); } diff --git a/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/RoutingToDataSetsTest.java b/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/RoutingToDataSetsTest.java index cab47c78bb54..14c9a561d09c 100644 --- a/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/RoutingToDataSetsTest.java +++ b/cdap-gateway/src/test/java/io/cdap/cdap/gateway/router/RoutingToDataSetsTest.java @@ -28,6 +28,7 @@ import io.cdap.cdap.common.utils.Networks; import io.cdap.cdap.internal.guice.AppFabricTestModule; import io.cdap.cdap.security.auth.UserIdentityExtractor; +import io.cdap.cdap.security.encryption.NoOpAeadCipher; import io.cdap.cdap.security.guice.CoreSecurityRuntimeModule; import io.cdap.cdap.security.guice.ExternalAuthenticationModule; import io.cdap.http.AbstractHttpHandler; @@ -54,6 +55,7 @@ * TODO: Eventually this can be removed, since we do not have any proxy rules anymore for datasets. */ public class RoutingToDataSetsTest { + private static NettyRouter nettyRouter; private static MockHttpService mockService; private static int port; @@ -62,12 +64,13 @@ public class RoutingToDataSetsTest { public static void before() throws Exception { CConfiguration cConf = CConfiguration.create(); Injector injector = Guice.createInjector(new CoreSecurityRuntimeModule().getInMemoryModules(), - new ExternalAuthenticationModule(), - new InMemoryDiscoveryModule(), - new AppFabricTestModule(cConf)); + new ExternalAuthenticationModule(), + new InMemoryDiscoveryModule(), + new AppFabricTestModule(cConf)); // Starting router - DiscoveryServiceClient discoveryServiceClient = injector.getInstance(DiscoveryServiceClient.class); + DiscoveryServiceClient discoveryServiceClient = injector + .getInstance(DiscoveryServiceClient.class); UserIdentityExtractor userIdentityExtractor = injector.getInstance(UserIdentityExtractor.class); SConfiguration sConf = SConfiguration.create(); @@ -76,14 +79,15 @@ public static void before() throws Exception { cConf.setInt(Constants.Router.ROUTER_PORT, port); nettyRouter = new NettyRouter(cConf, sConf, InetAddresses.forString("127.0.0.1"), - new RouterServiceLookup(cConf, discoveryServiceClient, new RouterPathLookup()), - new SuccessTokenValidator(), userIdentityExtractor, discoveryServiceClient); + new RouterServiceLookup(cConf, discoveryServiceClient, new RouterPathLookup()), + new SuccessTokenValidator(), userIdentityExtractor, discoveryServiceClient, + new NoOpAeadCipher()); nettyRouter.startAndWait(); // Starting mock DataSet service DiscoveryService discoveryService = injector.getInstance(DiscoveryService.class); mockService = new MockHttpService(discoveryService, Constants.Service.DATASET_MANAGER, - new MockDatasetTypeHandler(), new MockDatasetInstanceHandler()); + new MockDatasetTypeHandler(), new MockDatasetInstanceHandler()); mockService.startAndWait(); } @@ -99,26 +103,31 @@ public static void after() { @Test public void testTypeHandlerRequests() throws Exception { Assert.assertEquals("listModules", doRequest("/namespaces/myspace/data/modules", "GET")); - Assert.assertEquals("post:myModule", doRequest("/namespaces/myspace/data/modules/myModule", "POST")); - Assert.assertEquals("delete:myModule", doRequest("/namespaces/myspace/data/modules/myModule", "DELETE")); - Assert.assertEquals("get:myModule", doRequest("/namespaces/myspace/data/modules/myModule", "GET")); + Assert.assertEquals("post:myModule", + doRequest("/namespaces/myspace/data/modules/myModule", "POST")); + Assert.assertEquals("delete:myModule", + doRequest("/namespaces/myspace/data/modules/myModule", "DELETE")); + Assert.assertEquals("get:myModule", + doRequest("/namespaces/myspace/data/modules/myModule", "GET")); Assert.assertEquals("listTypes", doRequest("/namespaces/myspace/data/types", "GET")); - Assert.assertEquals("getType:myType", doRequest("/namespaces/myspace/data/types/myType", "GET")); + Assert + .assertEquals("getType:myType", doRequest("/namespaces/myspace/data/types/myType", "GET")); } @Test public void testInstanceHandlerRequests() throws Exception { Assert.assertEquals("list", doRequest("/namespaces/myspace/data/datasets", "GET")); Assert.assertEquals("post:myInstance", - doRequest("/namespaces/myspace/data/datasets/myInstance", "POST")); + doRequest("/namespaces/myspace/data/datasets/myInstance", "POST")); Assert.assertEquals("delete:myInstance", - doRequest("/namespaces/myspace/data/datasets/myInstance", "DELETE")); + doRequest("/namespaces/myspace/data/datasets/myInstance", "DELETE")); Assert.assertEquals("get:myInstance", - doRequest("/namespaces/myspace/data/datasets/myInstance", "GET")); + doRequest("/namespaces/myspace/data/datasets/myInstance", "GET")); } @Path(Constants.Gateway.API_VERSION_3 + "/namespaces/{namespace-id}") public static final class MockDatasetTypeHandler extends AbstractHttpHandler { + @GET @Path("/data/modules") public void listModules(HttpRequest request, final HttpResponder responder) { @@ -128,19 +137,21 @@ public void listModules(HttpRequest request, final HttpResponder responder) { @POST @Path("/data/modules/{name}") public void addModule(HttpRequest request, final HttpResponder responder, - @PathParam("name") String name) throws IOException { + @PathParam("name") String name) throws IOException { responder.sendString(HttpResponseStatus.OK, "post:" + name); } @DELETE @Path("/data/modules/{name}") - public void deleteModule(HttpRequest request, final HttpResponder responder, @PathParam("name") String name) { + public void deleteModule(HttpRequest request, final HttpResponder responder, + @PathParam("name") String name) { responder.sendString(HttpResponseStatus.OK, "delete:" + name); } @GET @Path("/data/modules/{name}") - public void getModuleInfo(HttpRequest request, final HttpResponder responder, @PathParam("name") String name) { + public void getModuleInfo(HttpRequest request, final HttpResponder responder, + @PathParam("name") String name) { responder.sendString(HttpResponseStatus.OK, "get:" + name); } @@ -153,13 +164,14 @@ public void listTypes(HttpRequest request, final HttpResponder responder) { @GET @Path("/data/types/{name}") public void getTypeInfo(HttpRequest request, final HttpResponder responder, - @PathParam("name") String name) { + @PathParam("name") String name) { responder.sendString(HttpResponseStatus.OK, "getType:" + name); } } @Path(Constants.Gateway.API_VERSION_3 + "/namespaces/{namespace-id}") public static final class MockDatasetInstanceHandler extends AbstractHttpHandler { + @GET @Path("/data/datasets/") public void list(HttpRequest request, final HttpResponder responder) { @@ -169,27 +181,28 @@ public void list(HttpRequest request, final HttpResponder responder) { @GET @Path("/data/datasets/{instance-name}") public void getInfo(HttpRequest request, final HttpResponder responder, - @PathParam("instance-name") String name) { + @PathParam("instance-name") String name) { responder.sendString(HttpResponseStatus.OK, "get:" + name); } @POST @Path("/data/datasets/{instance-name}") public void add(HttpRequest request, final HttpResponder responder, - @PathParam("instance-name") String name) { + @PathParam("instance-name") String name) { responder.sendString(HttpResponseStatus.OK, "post:" + name); } @DELETE @Path("/data/datasets/{instance-name}") public void drop(HttpRequest request, final HttpResponder responder, - @PathParam("instance-name") String instanceName) { + @PathParam("instance-name") String instanceName) { responder.sendString(HttpResponseStatus.OK, "delete:" + instanceName); } } private String doRequest(String resource, String requestMethod) throws Exception { - resource = String.format("http://localhost:%d%s" + resource, port, Constants.Gateway.API_VERSION_3); + resource = String + .format("http://localhost:%d%s" + resource, port, Constants.Gateway.API_VERSION_3); URL url = new URL(resource); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setRequestMethod(requestMethod); diff --git a/cdap-hbase-compat-base/pom.xml b/cdap-hbase-compat-base/pom.xml index 70a756f3d553..3cb5633cd61a 100644 --- a/cdap-hbase-compat-base/pom.xml +++ b/cdap-hbase-compat-base/pom.xml @@ -16,8 +16,8 @@ --> + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> cdap io.cdap.cdap @@ -29,6 +29,11 @@ CDAP HBase Compactability Base jar + + + 2.5.0 + + io.cdap.cdap @@ -138,6 +143,11 @@ org.slf4j jcl-over-slf4j + + com.google.protobuf + protobuf-java + ${protobuf.version} + diff --git a/cdap-master/pom.xml b/cdap-master/pom.xml index 0ca796eb2057..3535421b1d78 100644 --- a/cdap-master/pom.xml +++ b/cdap-master/pom.xml @@ -281,6 +281,7 @@ ${stage.opt.dir}/ext/metadataconsumers/data-catalog-consumer + ${stage.opt.dir}/ext/encryption ${stage.opt.dir}/bootstrap ${stage.opt.dir}/capability-config **/target/*.jar @@ -383,6 +384,12 @@ ${project.version} provided + + io.cdap.cdap + cdap-encryption-ext-tink + ${project.version} + provided + @@ -771,6 +778,28 @@ + + + + copy-encryption-ext-tink + process-resources + + copy-resources + + + ${stage.encryption.ext.dir}/tink + + + + ${project.parent.basedir}/cdap-encryption-ext-tink/target/libexec/ + + + *.jar + + + + + diff --git a/cdap-master/src/main/java/io/cdap/cdap/master/environment/k8s/RouterServiceMain.java b/cdap-master/src/main/java/io/cdap/cdap/master/environment/k8s/RouterServiceMain.java index 7e6314c9ece9..b02ed6dd5cdf 100644 --- a/cdap-master/src/main/java/io/cdap/cdap/master/environment/k8s/RouterServiceMain.java +++ b/cdap-master/src/main/java/io/cdap/cdap/master/environment/k8s/RouterServiceMain.java @@ -32,6 +32,7 @@ import io.cdap.cdap.master.spi.environment.MasterEnvironmentContext; import io.cdap.cdap.messaging.guice.MessagingServiceModule; import io.cdap.cdap.proto.id.NamespaceId; +import io.cdap.cdap.security.encryption.guice.UserCredentialAeadEncryptionModule; import io.cdap.cdap.security.guice.ExternalAuthenticationModule; import java.util.ArrayList; import java.util.List; @@ -63,6 +64,7 @@ protected List getServiceModules(MasterEnvironment masterEnv, modules.add(new RouterModules().getDistributedModules()); modules.add(new DFSLocationModule()); modules.add(new ExternalAuthenticationModule()); + modules.add(new UserCredentialAeadEncryptionModule()); return modules; } diff --git a/cdap-security-spi/src/main/java/io/cdap/cdap/security/spi/encryption/AeadCipherContext.java b/cdap-security-spi/src/main/java/io/cdap/cdap/security/spi/encryption/AeadCipherContext.java new file mode 100644 index 000000000000..e1c631ad06e1 --- /dev/null +++ b/cdap-security-spi/src/main/java/io/cdap/cdap/security/spi/encryption/AeadCipherContext.java @@ -0,0 +1,41 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.security.spi.encryption; + +import java.util.Map; + +/** + * Context for {@link AeadCipherCryptor}. + */ +public class AeadCipherContext { + + private final Map properties; + private final Map secureProperties; + + public AeadCipherContext(Map properties, Map secureProperties) { + this.properties = properties; + this.secureProperties = secureProperties; + } + + public Map getProperties() { + return properties; + } + + public Map getSecureProperties() { + return secureProperties; + } +} diff --git a/cdap-security-spi/src/main/java/io/cdap/cdap/security/spi/encryption/AeadCipherCryptor.java b/cdap-security-spi/src/main/java/io/cdap/cdap/security/spi/encryption/AeadCipherCryptor.java new file mode 100644 index 000000000000..2d12f727e001 --- /dev/null +++ b/cdap-security-spi/src/main/java/io/cdap/cdap/security/spi/encryption/AeadCipherCryptor.java @@ -0,0 +1,56 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.security.spi.encryption; + +/** + * An AEAD symmetric encryption primitive. + */ +public interface AeadCipherCryptor { + + /** + * @return The name of the AeadCipherCryptor. + */ + String getName(); + + /** + * Initialize the cipher. + * + * @throws CipherInitializationException If initialization fails. + */ + void initialize(AeadCipherContext context) throws CipherInitializationException; + + /** + * Encrypt the data. + * + * @param plainData Data to be encrypted. + * @param associatedData Used for integrity checking during decryption. + * @return Encrypted data. + * @throws CipherOperationException If encryption fails. + */ + byte[] encrypt(byte[] plainData, byte[] associatedData) throws CipherOperationException; + + /** + * Decrypt the data. + * + * @param cipherData Data to be decrypted + * @param associatedData Used for integrity checking, must be the same as that used during + * encryption. + * @return Decrypted data. + * @throws CipherOperationException If decryption fails. + */ + byte[] decrypt(byte[] cipherData, byte[] associatedData) throws CipherOperationException; +} diff --git a/cdap-security-spi/src/main/java/io/cdap/cdap/security/spi/encryption/CipherException.java b/cdap-security-spi/src/main/java/io/cdap/cdap/security/spi/encryption/CipherException.java new file mode 100644 index 000000000000..0dd1d7c3d8f3 --- /dev/null +++ b/cdap-security-spi/src/main/java/io/cdap/cdap/security/spi/encryption/CipherException.java @@ -0,0 +1,31 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.security.spi.encryption; + +/** + * This exception indicates a failure in {@link AeadCipherCryptor} operation. + */ +public class CipherException extends RuntimeException { + + public CipherException(String message) { + super(message); + } + + public CipherException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/cdap-security-spi/src/main/java/io/cdap/cdap/security/spi/encryption/CipherInitializationException.java b/cdap-security-spi/src/main/java/io/cdap/cdap/security/spi/encryption/CipherInitializationException.java new file mode 100644 index 000000000000..a4535879ffa1 --- /dev/null +++ b/cdap-security-spi/src/main/java/io/cdap/cdap/security/spi/encryption/CipherInitializationException.java @@ -0,0 +1,31 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.security.spi.encryption; + +/** + * This exception indicates an initialization failure. + */ +public class CipherInitializationException extends CipherException { + + public CipherInitializationException(String message) { + super(message); + } + + public CipherInitializationException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/cdap-security-spi/src/main/java/io/cdap/cdap/security/spi/encryption/CipherOperationException.java b/cdap-security-spi/src/main/java/io/cdap/cdap/security/spi/encryption/CipherOperationException.java new file mode 100644 index 000000000000..39d6eadee602 --- /dev/null +++ b/cdap-security-spi/src/main/java/io/cdap/cdap/security/spi/encryption/CipherOperationException.java @@ -0,0 +1,31 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.security.spi.encryption; + +/** + * This exception indicates an encryption/decryption failure. + */ +public class CipherOperationException extends CipherException { + + public CipherOperationException(String message) { + super(message); + } + + public CipherOperationException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/cdap-security/pom.xml b/cdap-security/pom.xml index f3a5b2958f1d..42298723843a 100644 --- a/cdap-security/pom.xml +++ b/cdap-security/pom.xml @@ -134,10 +134,6 @@ org.apache.hadoop hadoop-minikdc - - com.google.crypto.tink - tink - org.apache.hbase hbase-common @@ -197,50 +193,6 @@ - - org.apache.maven.plugins - maven-shade-plugin - 3.2.4 - - false - - - com.google.crypto.tink:tink - com.google.protobuf:protobuf-java - - - - - com.google.protobuf - io.cdap.cdap.shaded.com.google.protobuf - - - com.google.crypto.tink - io.cdap.cdap.shaded.com.google.crypto.tink - - - - - - shade - package - - shade - - - - dist-shade - package - - shade - - - ${stage.lib.dir} - ${project.groupId}.${project.build.finalName} - - - - org.apache.maven.plugins maven-jar-plugin diff --git a/cdap-security/src/main/java/io/cdap/cdap/security/auth/TinkCipher.java b/cdap-security/src/main/java/io/cdap/cdap/security/auth/TinkCipher.java deleted file mode 100644 index 5ecd97cd9814..000000000000 --- a/cdap-security/src/main/java/io/cdap/cdap/security/auth/TinkCipher.java +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Copyright © 2021 Cask Data, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package io.cdap.cdap.security.auth; - -import com.google.crypto.tink.Aead; -import com.google.crypto.tink.CleartextKeysetHandle; -import com.google.crypto.tink.JsonKeysetReader; -import com.google.crypto.tink.KeysetHandle; -import com.google.crypto.tink.aead.AeadConfig; -import io.cdap.cdap.common.conf.Constants; -import io.cdap.cdap.common.conf.SConfiguration; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.security.GeneralSecurityException; -import java.util.Base64; -import javax.annotation.Nullable; - -/** - * Tink cipher that allows encrypting and decrypting data using keyset stored in {@link - * SConfiguration} - */ -public class TinkCipher { - - /** - * Wraps a keyset with some additional parameters and metadata. - */ - private final KeysetHandle keysetHandle; - - /** - * Tink AEAD (Authenticated Encryption with Associated Data) primitive - */ - private final Aead aead; - - /** - * Default associated data to use if none is specified - */ - private static final byte[] DEFAULT_ASSOCIATED_DATA = "DefaultAssociatedData".getBytes( - StandardCharsets.UTF_8); - - public TinkCipher(SConfiguration sConf) throws CipherException { - try { - // Init Tink with AEAD primitive - AeadConfig.register(); - - // Load keyset from sConf - String jsonKeyset = sConf.get( - Constants.Security.Authentication.USER_CREDENTIAL_ENCRYPTION_KEYSET); - this.keysetHandle = CleartextKeysetHandle.read(JsonKeysetReader.withString(jsonKeyset)); - - this.aead = keysetHandle.getPrimitive(Aead.class); - } catch (GeneralSecurityException e) { - throw new CipherException("Failed to init Tink cipher: " + e.getMessage(), e); - } catch (IOException e) { - throw new CipherException("Failed to load Tink keyset: " + e.getMessage(), e); - } - } - - /** - * Encrypt the data. - * - * @param plainData data to be encrypted - * @param associatedData used for integrity checking during decryption. - * @return encrypted data - * @throws CipherException if encryption fails - */ - public byte[] encrypt(byte[] plainData, @Nullable byte[] associatedData) throws CipherException { - try { - if (associatedData == null) { - associatedData = DEFAULT_ASSOCIATED_DATA; - } - return aead.encrypt(plainData, associatedData); - } catch (GeneralSecurityException e) { - throw new CipherException("Failed to encrypt: " + e.getMessage(), e); - } - } - - /** - * Encrypt the string and return encrypted data in base64 encoded form. - * - * @param plainData data to be encrypted - * @param associatedData used for integrity checking during decryption. - * @return encrypted data in base64 encoded form - * @throws CipherException if encryption fails - */ - public String encryptStringToBase64(String plainData, @Nullable byte[] associatedData) - throws CipherException { - return Base64.getEncoder().encodeToString(encrypt(plainData.getBytes(), associatedData)); - } - - /** - * Encrypt the data and return encrypted data in base64 encoded form. - * - * @param plainData data to be encrypted - * @param associatedData used for integrity checking during decryption. - * @return encrypted data in base64 encoded form - * @throws CipherException if encryption fails - */ - public String encryptToBase64(byte[] plainData, @Nullable byte[] associatedData) - throws CipherException { - return Base64.getEncoder().encodeToString(encrypt(plainData, associatedData)); - } - - /** - * Decrypt the cipher data. - * - * @param cipherData data to be decrypted - * @param associatedData used for integrity checking, must be the same as that used during - * encryption. - * @return decrypted data - * @throws CipherException if decryption fails - */ - public byte[] decrypt(byte[] cipherData, @Nullable byte[] associatedData) throws CipherException { - try { - if (associatedData == null) { - associatedData = DEFAULT_ASSOCIATED_DATA; - } - return aead.decrypt(cipherData, associatedData); - } catch (GeneralSecurityException e) { - throw new CipherException("Failed to decrypt: " + e.getMessage(), e); - } - } - - /** - * Decrypt the cipher data that was base64 encoded. - * - * @param cipherData data in base64 encoded form that needs to be decrypted - * @param associatedData used for integrity checking, must be the same as that used during - * encryption. - * @return decrypted data - * @throws CipherException if decryption fails - */ - public byte[] decryptFromBase64(String cipherData, byte[] associatedData) throws CipherException { - return decrypt(Base64.getDecoder().decode(cipherData), associatedData); - } - - /** - * Decrypt the cipher data that was encrypted and base-encoded from a string. - * - * @param cipherData data in base64 encoded form that needs to be decrypted - * @param associatedData used for integrity checking, must be the same as that used during - * encryption. - * @return decrypted data - * @throws CipherException if decryption fails - */ - public String decryptStringFromBase64(String cipherData, byte[] associatedData) - throws CipherException { - return new String(decrypt(Base64.getDecoder().decode(cipherData), associatedData), - StandardCharsets.UTF_8); - } -} - - diff --git a/cdap-security/src/main/java/io/cdap/cdap/security/authorization/AuthorizationEnforcementModule.java b/cdap-security/src/main/java/io/cdap/cdap/security/authorization/AuthorizationEnforcementModule.java index 727a1cf51928..965938587f90 100644 --- a/cdap-security/src/main/java/io/cdap/cdap/security/authorization/AuthorizationEnforcementModule.java +++ b/cdap-security/src/main/java/io/cdap/cdap/security/authorization/AuthorizationEnforcementModule.java @@ -26,6 +26,7 @@ import io.cdap.cdap.proto.element.EntityType; import io.cdap.cdap.proto.id.EntityId; import io.cdap.cdap.proto.security.Permission; +import io.cdap.cdap.security.encryption.guice.UserCredentialAeadEncryptionModule; import io.cdap.cdap.security.impersonation.SecurityUtil; import io.cdap.cdap.security.spi.authorization.AccessEnforcer; import io.cdap.cdap.security.spi.authorization.ContextAccessEnforcer; @@ -39,6 +40,7 @@ */ public class AuthorizationEnforcementModule extends RuntimeModule { + @Override public Module getInMemoryModules() { return new AbstractModule() { @@ -50,6 +52,7 @@ protected void configure() { .to(NoOpAccessController.class).in(Scopes.SINGLETON); bind(ContextAccessEnforcer.class).to(DefaultContextAccessEnforcer.class) .in(Scopes.SINGLETON); + install(new UserCredentialAeadEncryptionModule()); } }; } @@ -65,6 +68,7 @@ protected void configure() { .to(NoOpAccessController.class).in(Scopes.SINGLETON); bind(ContextAccessEnforcer.class).to(DefaultContextAccessEnforcer.class) .in(Scopes.SINGLETON); + install(new UserCredentialAeadEncryptionModule()); } }; } @@ -118,6 +122,7 @@ protected void configure() { .toProvider(InternalAccessEnforcerProvider.class).in(Scopes.SINGLETON); bind(ContextAccessEnforcer.class).to(DefaultContextAccessEnforcer.class) .in(Scopes.SINGLETON); + install(new UserCredentialAeadEncryptionModule()); } }; } diff --git a/cdap-security/src/main/java/io/cdap/cdap/security/authorization/DefaultAccessEnforcer.java b/cdap-security/src/main/java/io/cdap/cdap/security/authorization/DefaultAccessEnforcer.java index ac42db5d7c64..0ea29c0dd46c 100644 --- a/cdap-security/src/main/java/io/cdap/cdap/security/authorization/DefaultAccessEnforcer.java +++ b/cdap-security/src/main/java/io/cdap/cdap/security/authorization/DefaultAccessEnforcer.java @@ -26,6 +26,7 @@ import io.cdap.cdap.api.security.AccessException; import io.cdap.cdap.common.conf.CConfiguration; import io.cdap.cdap.common.conf.Constants; +import io.cdap.cdap.common.conf.Constants.Security.Encryption; import io.cdap.cdap.common.conf.SConfiguration; import io.cdap.cdap.common.metrics.ProgramTypeMetricTag; import io.cdap.cdap.proto.element.EntityType; @@ -37,11 +38,11 @@ import io.cdap.cdap.proto.security.Credential; import io.cdap.cdap.proto.security.Permission; import io.cdap.cdap.proto.security.Principal; -import io.cdap.cdap.security.auth.CipherException; -import io.cdap.cdap.security.auth.TinkCipher; +import io.cdap.cdap.security.encryption.AeadCipher; +import io.cdap.cdap.security.encryption.guice.UserCredentialAeadEncryptionModule; import io.cdap.cdap.security.impersonation.SecurityUtil; import io.cdap.cdap.security.spi.authorization.AccessEnforcer; -import java.nio.charset.StandardCharsets; +import io.cdap.cdap.security.spi.encryption.CipherException; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -73,12 +74,15 @@ public class DefaultAccessEnforcer extends AbstractAccessEnforcer { private final boolean metricsCollectionEnabled; private final boolean metricsTagsEnabled; private final MetricsCollectionService metricsCollectionService; + private final AeadCipher userEncryptionAeadCipher; @Inject DefaultAccessEnforcer(CConfiguration cConf, SConfiguration sConf, AccessControllerInstantiator accessControllerInstantiator, @Named(INTERNAL_ACCESS_ENFORCER) AccessEnforcer internalAccessEnforcer, - MetricsCollectionService metricsCollectionService) { + MetricsCollectionService metricsCollectionService, + @Named(UserCredentialAeadEncryptionModule.USER_CREDENTIAL_ENCRYPTION) + AeadCipher userEncryptionAeadCipher) { super(cConf); this.sConf = sConf; this.accessControllerInstantiator = accessControllerInstantiator; @@ -94,6 +98,7 @@ public class DefaultAccessEnforcer extends AbstractAccessEnforcer { this.metricsTagsEnabled = cConf.getBoolean(Constants.Metrics.AUTHORIZATION_METRICS_TAGS_ENABLED, false); this.metricsCollectionService = metricsCollectionService; + this.userEncryptionAeadCipher = userEncryptionAeadCipher; } @Override @@ -264,10 +269,9 @@ private Principal getUserPrinciple(Principal principal) throws AccessException { // When user credential encryption is enabled, credential should be encrypted upon arrival // at router and decrypted right here before calling auth extension. try { - String plainCredential = new String( - new TinkCipher(sConf).decryptFromBase64(userCredential.getValue(), - null), - StandardCharsets.UTF_8); + String plainCredential = userEncryptionAeadCipher + .decryptStringFromBase64(userCredential.getValue(), + Encryption.USER_CREDENTIAL_ENCRYPTION_ASSOCIATED_DATA.getBytes()); return new Principal(principal.getName(), principal.getType(), principal.getKerberosPrincipal(), diff --git a/cdap-security/src/main/java/io/cdap/cdap/security/encryption/AeadCipher.java b/cdap-security/src/main/java/io/cdap/cdap/security/encryption/AeadCipher.java new file mode 100644 index 000000000000..f4635e7ac21a --- /dev/null +++ b/cdap-security/src/main/java/io/cdap/cdap/security/encryption/AeadCipher.java @@ -0,0 +1,104 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.security.encryption; + +import io.cdap.cdap.security.spi.encryption.CipherException; +import io.cdap.cdap.security.spi.encryption.CipherOperationException; +import java.nio.charset.StandardCharsets; +import java.util.Base64; + +/** + * An interface intended to represent a service providing AEAD symmetric encryption primitives. + */ +public interface AeadCipher { + + /** + * Encrypt the data. + * + * @param plainData Data to be encrypted. + * @param associatedData Used for integrity checking during decryption. + * @return Encrypted data. + * @throws CipherOperationException If encryption fails. + */ + byte[] encrypt(byte[] plainData, byte[] associatedData) throws CipherException; + + /** + * Decrypt the data. + * + * @param cipherData Data to be decrypted + * @param associatedData Used for integrity checking, must be the same as that used during + * encryption. + * @return Decrypted data. + * @throws CipherOperationException If decryption fails. + */ + byte[] decrypt(byte[] cipherData, byte[] associatedData) throws CipherException; + + /** + * Encrypt the string and return encrypted data in base64 encoded form. + * + * @param plainData data to be encrypted + * @param associatedData used for integrity checking during decryption. + * @return encrypted data in base64 encoded form + * @throws CipherOperationException if encryption fails + */ + default String encryptToBase64(String plainData, byte[] associatedData) + throws CipherException { + return Base64.getEncoder().encodeToString(encrypt(plainData.getBytes(StandardCharsets.UTF_8), + associatedData)); + } + + /** + * Encrypt the data and return encrypted data in base64 encoded form. + * + * @param plainData data to be encrypted + * @param associatedData used for integrity checking during decryption. + * @return encrypted data in base64 encoded form + * @throws CipherOperationException if encryption fails + */ + default String encryptToBase64(byte[] plainData, byte[] associatedData) throws CipherException { + return Base64.getEncoder().encodeToString(encrypt(plainData, associatedData)); + } + + /** + * Decrypt the cipher data that was base64 encoded. + * + * @param cipherData data in base64 encoded form that needs to be decrypted + * @param associatedData used for integrity checking, must be the same as that used during + * encryption. + * @return decrypted data + * @throws CipherOperationException if decryption fails + */ + default byte[] decryptFromBase64(String cipherData, byte[] associatedData) + throws CipherException { + return decrypt(Base64.getDecoder().decode(cipherData), associatedData); + } + + /** + * Decrypt the cipher data that was encrypted and base-encoded from a string. + * + * @param cipherData data in base64 encoded form that needs to be decrypted + * @param associatedData used for integrity checking, must be the same as that used during + * encryption. + * @return decrypted data + * @throws CipherOperationException if decryption fails + */ + default String decryptStringFromBase64(String cipherData, byte[] associatedData) + throws CipherException { + return new String(decrypt(Base64.getDecoder().decode(cipherData), associatedData), + StandardCharsets.UTF_8); + } +} diff --git a/cdap-security/src/main/java/io/cdap/cdap/security/encryption/NoOpAeadCipher.java b/cdap-security/src/main/java/io/cdap/cdap/security/encryption/NoOpAeadCipher.java new file mode 100644 index 000000000000..ca2e4ff99604 --- /dev/null +++ b/cdap-security/src/main/java/io/cdap/cdap/security/encryption/NoOpAeadCipher.java @@ -0,0 +1,33 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.security.encryption; + +/** + * Performs no encryption. + */ +public class NoOpAeadCipher implements AeadCipher { + + @Override + public byte[] encrypt(byte[] plainData, byte[] associatedData) { + return plainData; + } + + @Override + public byte[] decrypt(byte[] cipherData, byte[] associatedData) { + return cipherData; + } +} diff --git a/cdap-security/src/main/java/io/cdap/cdap/security/encryption/guice/AbstractAeadCipherProvider.java b/cdap-security/src/main/java/io/cdap/cdap/security/encryption/guice/AbstractAeadCipherProvider.java new file mode 100644 index 000000000000..e10966733ab5 --- /dev/null +++ b/cdap-security/src/main/java/io/cdap/cdap/security/encryption/guice/AbstractAeadCipherProvider.java @@ -0,0 +1,75 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.security.encryption.guice; + +import io.cdap.cdap.common.conf.CConfiguration; +import io.cdap.cdap.common.conf.SConfiguration; +import io.cdap.cdap.security.encryption.AeadCipher; +import io.cdap.cdap.security.encryption.NoOpAeadCipher; +import java.util.Map; +import javax.inject.Provider; + +/** + * Provider for {@link AeadCipher}. + */ +public abstract class AbstractAeadCipherProvider implements Provider { + + private final String NOOP_AEAD_CIPHER_NAME = "NONE"; + + private final AeadCipherCryptorExtensionLoader aeadCipherCryptorExtensionLoader; + final CConfiguration cConf; + final SConfiguration sConf; + + public AbstractAeadCipherProvider( + AeadCipherCryptorExtensionLoader aeadCipherCryptorExtensionLoader, + CConfiguration cConf, SConfiguration sConf) { + this.cConf = cConf; + this.sConf = sConf; + this.aeadCipherCryptorExtensionLoader = aeadCipherCryptorExtensionLoader; + } + + /** + * Returns the AEAD cipher to use. + * + * @return The AEAD cipher to use. + */ + protected abstract String getCipherName(); + + /** + * Returns the properties to pass to the cipher service. + * + * @return The properties to pass to the cipher service. + */ + protected abstract Map getProperties(); + + /** + * Returns the secure properties to pass to the cipher service. + * + * @return The secure properties to pass to the cipher service. + */ + protected abstract Map getSecureProperties(); + + @Override + public AeadCipher get() { + String cipherName = getCipherName(); + if (NOOP_AEAD_CIPHER_NAME.equals(cipherName)) { + return new NoOpAeadCipher(); + } + return new LazyDelegateAeadCipher(aeadCipherCryptorExtensionLoader.get(cipherName), + getProperties(), getSecureProperties()); + } +} diff --git a/cdap-security/src/main/java/io/cdap/cdap/security/encryption/guice/AeadCipherCryptorExtensionLoader.java b/cdap-security/src/main/java/io/cdap/cdap/security/encryption/guice/AeadCipherCryptorExtensionLoader.java new file mode 100644 index 000000000000..4a76529eee70 --- /dev/null +++ b/cdap-security/src/main/java/io/cdap/cdap/security/encryption/guice/AeadCipherCryptorExtensionLoader.java @@ -0,0 +1,108 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.security.encryption.guice; + +import com.google.inject.Inject; +import io.cdap.cdap.common.conf.CConfiguration; +import io.cdap.cdap.common.conf.Constants; +import io.cdap.cdap.common.lang.ClassPathResources; +import io.cdap.cdap.common.lang.FilterClassLoader; +import io.cdap.cdap.extension.AbstractExtensionLoader; +import io.cdap.cdap.security.spi.encryption.AeadCipherCryptor; +import java.io.IOException; +import java.util.Collections; +import java.util.Set; + +public class AeadCipherCryptorExtensionLoader extends + AbstractExtensionLoader { + + private volatile Set allowedResources; + private volatile Set allowedPackages; + + @Inject + AeadCipherCryptorExtensionLoader(CConfiguration cConf) { + super(cConf.get(Constants.Security.Encryption.EXTENSIONS_DIR)); + } + + @Override + protected Set getSupportedTypesForProvider(AeadCipherCryptor aeadCipherCryptor) { + return Collections.singleton(aeadCipherCryptor.getName()); + } + + @Override + protected FilterClassLoader.Filter getExtensionParentClassLoaderFilter() { + // Only permit cdap-security-spi dependencies + return new FilterClassLoader.Filter() { + @Override + public boolean acceptResource(String resource) { + return getAllowedResources().contains(resource); + } + + @Override + public boolean acceptPackage(String packageName) { + return getAllowedPackages().contains(packageName); + } + }; + } + + /** + * Returns the set of resources that are visible to extensions. + */ + private Set getAllowedResources() { + Set resources = this.allowedResources; + if (resources != null) { + return resources; + } + + synchronized (this) { + resources = this.allowedResources; + if (resources != null) { + return resources; + } + try { + // All cdap-security-spi classes and its dependencies are visible to extensions + // The set of dependencies for cdap-security-spi should be kept at minimal to reduce dependency conflicts + this.allowedResources = resources = ClassPathResources.getResourcesWithDependencies( + getClass().getClassLoader(), AeadCipherCryptor.class); + return resources; + } catch (IOException e) { + throw new RuntimeException("Failed to find security SPI resources", e); + } + } + } + + /** + * Returns the set of package names that are visible to extensions. + */ + private Set getAllowedPackages() { + Set packages = this.allowedPackages; + if (packages != null) { + return packages; + } + synchronized (this) { + packages = this.allowedPackages; + if (packages != null) { + return packages; + } + + packages = createPackageSets(getAllowedResources()); + + this.allowedPackages = packages; + return packages; + } + } +} diff --git a/cdap-security/src/main/java/io/cdap/cdap/security/encryption/guice/DataStorageAeadCipherProvider.java b/cdap-security/src/main/java/io/cdap/cdap/security/encryption/guice/DataStorageAeadCipherProvider.java new file mode 100644 index 000000000000..b89bcd692689 --- /dev/null +++ b/cdap-security/src/main/java/io/cdap/cdap/security/encryption/guice/DataStorageAeadCipherProvider.java @@ -0,0 +1,52 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.security.encryption.guice; + +import io.cdap.cdap.common.conf.CConfiguration; +import io.cdap.cdap.common.conf.Constants.Security.Encryption; +import io.cdap.cdap.common.conf.SConfiguration; +import io.cdap.cdap.security.encryption.AeadCipher; +import java.util.Map; +import javax.inject.Inject; + +/** + * {@link AeadCipher} provider for sensitive data storage encryption. + */ +public class DataStorageAeadCipherProvider extends AbstractAeadCipherProvider { + + @Inject + public DataStorageAeadCipherProvider( + AeadCipherCryptorExtensionLoader aeadCipherCryptorExtensionLoader, + CConfiguration cConf, SConfiguration sConf) { + super(aeadCipherCryptorExtensionLoader, cConf, sConf); + } + + @Override + protected String getCipherName() { + return cConf.get(Encryption.DATA_STORAGE_ENCRYPTION_CIPHER_NAME); + } + + @Override + protected Map getProperties() { + return cConf.getPropsWithPrefix(Encryption.DATA_STORAGE_ENCRYPTION_PROPERTIES_PREFIX); + } + + @Override + protected Map getSecureProperties() { + return sConf.getPropsWithPrefix(Encryption.DATA_STORAGE_ENCRYPTION_PROPERTIES_PREFIX); + } +} diff --git a/cdap-security/src/main/java/io/cdap/cdap/security/encryption/guice/DataStorageAeadEncryptionModule.java b/cdap-security/src/main/java/io/cdap/cdap/security/encryption/guice/DataStorageAeadEncryptionModule.java new file mode 100644 index 000000000000..d202f320910e --- /dev/null +++ b/cdap-security/src/main/java/io/cdap/cdap/security/encryption/guice/DataStorageAeadEncryptionModule.java @@ -0,0 +1,43 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.security.encryption.guice; + +import com.google.inject.PrivateModule; +import com.google.inject.Scopes; +import com.google.inject.name.Names; +import io.cdap.cdap.security.encryption.AeadCipher; + +/** + * Guice module for encryption bindings for data storage encryption. + */ +public class DataStorageAeadEncryptionModule extends PrivateModule { + + public static final String DATA_STORAGE_ENCRYPTION = "DataStorageEncryption"; + + @Override + protected void configure() { + // Bind extension loader. + bind(AeadCipherCryptorExtensionLoader.class).in(Scopes.SINGLETON); + + // Bind sensitive data storage encryption providers. + bind(AeadCipher.class) + .annotatedWith(Names.named(DATA_STORAGE_ENCRYPTION)) + .toProvider(DataStorageAeadCipherProvider.class).in(Scopes.SINGLETON); + expose(AeadCipher.class) + .annotatedWith(Names.named(DATA_STORAGE_ENCRYPTION)); + } +} diff --git a/cdap-security/src/main/java/io/cdap/cdap/security/encryption/guice/LazyDelegateAeadCipher.java b/cdap-security/src/main/java/io/cdap/cdap/security/encryption/guice/LazyDelegateAeadCipher.java new file mode 100644 index 000000000000..20a79630f078 --- /dev/null +++ b/cdap-security/src/main/java/io/cdap/cdap/security/encryption/guice/LazyDelegateAeadCipher.java @@ -0,0 +1,71 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.security.encryption.guice; + +import io.cdap.cdap.security.encryption.AeadCipher; +import io.cdap.cdap.security.spi.encryption.AeadCipherContext; +import io.cdap.cdap.security.spi.encryption.AeadCipherCryptor; +import io.cdap.cdap.security.spi.encryption.CipherException; +import io.cdap.cdap.security.spi.encryption.CipherInitializationException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * Default implementation of {@link AeadCipher} which delegates calls to an instance of the SPI + * {@link AeadCipherCryptor} with lazy initialization. + */ +public class LazyDelegateAeadCipher implements AeadCipher { + + private final AeadCipherCryptor aeadCipherCryptor; + private final Map properties; + private Map secureProperties; + private volatile boolean initialized; + + protected LazyDelegateAeadCipher(AeadCipherCryptor aeadCipherCryptor, + Map properties, + Map secureProperties) { + this.aeadCipherCryptor = aeadCipherCryptor; + this.properties = Collections.unmodifiableMap(new HashMap<>(properties)); + this.secureProperties = Collections.unmodifiableMap(new HashMap<>(secureProperties)); + } + + private void lazyInitialize() throws CipherInitializationException { + if (!initialized) { + synchronized (this) { + if (!initialized) { + aeadCipherCryptor.initialize(new AeadCipherContext(properties, secureProperties)); + // Help garbage collect secure properties to avoid keeping them in memory. + secureProperties = null; + this.initialized = true; + } + } + } + } + + @Override + public byte[] encrypt(byte[] plainData, byte[] associatedData) throws CipherException { + lazyInitialize(); + return aeadCipherCryptor.encrypt(plainData, associatedData); + } + + @Override + public byte[] decrypt(byte[] cipherData, byte[] associatedData) throws CipherException { + lazyInitialize(); + return aeadCipherCryptor.decrypt(cipherData, associatedData); + } +} diff --git a/cdap-security/src/main/java/io/cdap/cdap/security/encryption/guice/UserCredentialAeadCipherProvider.java b/cdap-security/src/main/java/io/cdap/cdap/security/encryption/guice/UserCredentialAeadCipherProvider.java new file mode 100644 index 000000000000..3cf19eccae8d --- /dev/null +++ b/cdap-security/src/main/java/io/cdap/cdap/security/encryption/guice/UserCredentialAeadCipherProvider.java @@ -0,0 +1,70 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.security.encryption.guice; + +import io.cdap.cdap.common.conf.CConfiguration; +import io.cdap.cdap.common.conf.Constants.Security.Authentication; +import io.cdap.cdap.common.conf.Constants.Security.Encryption; +import io.cdap.cdap.common.conf.SConfiguration; +import io.cdap.cdap.security.encryption.AeadCipher; +import java.util.Map; +import javax.inject.Inject; + +/** + * {@link AeadCipher} provider for user credential encryption. + */ +public class UserCredentialAeadCipherProvider extends AbstractAeadCipherProvider { + + // These properties are here for backwards compatibility, because Tink used to be in cdap-security + // rather than an extension. + private static final String TINK_CLEARTEXT_CIPHER_NAME = "tink-cleartext"; + private static final String TINK_CLEARTEXT_KEYSET_KEY = "tink.cleartext.keyset"; + + + @Inject + public UserCredentialAeadCipherProvider( + AeadCipherCryptorExtensionLoader aeadCipherCryptorExtensionLoader, CConfiguration cConf, + SConfiguration sConf) { + super(aeadCipherCryptorExtensionLoader, cConf, sConf); + } + + @Override + protected String getCipherName() { + // Backwards compatibility with older properties + if (sConf.getBoolean(Authentication.USER_CREDENTIAL_ENCRYPTION_ENABLED, false)) { + return TINK_CLEARTEXT_CIPHER_NAME; + } + return cConf.get(Encryption.USER_CREDENTIAL_ENCRYPTION_CIPHER_NAME); + } + + @Override + protected Map getProperties() { + return cConf.getPropsWithPrefix(Encryption.USER_CREDENTIAL_ENCRYPTION_PROPERTIES_PREFIX); + } + + @Override + protected Map getSecureProperties() { + Map secureProps = sConf + .getPropsWithPrefix(Encryption.USER_CREDENTIAL_ENCRYPTION_PROPERTIES_PREFIX); + // Backwards compatibility with older properties + if (sConf.getBoolean(Authentication.USER_CREDENTIAL_ENCRYPTION_ENABLED, false)) { + secureProps.put(TINK_CLEARTEXT_KEYSET_KEY, + sConf.get(Authentication.USER_CREDENTIAL_ENCRYPTION_KEYSET)); + } + return secureProps; + } +} diff --git a/cdap-security/src/main/java/io/cdap/cdap/security/encryption/guice/UserCredentialAeadEncryptionModule.java b/cdap-security/src/main/java/io/cdap/cdap/security/encryption/guice/UserCredentialAeadEncryptionModule.java new file mode 100644 index 000000000000..3e75a0128874 --- /dev/null +++ b/cdap-security/src/main/java/io/cdap/cdap/security/encryption/guice/UserCredentialAeadEncryptionModule.java @@ -0,0 +1,40 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.security.encryption.guice; + +import com.google.inject.PrivateModule; +import com.google.inject.Scopes; +import com.google.inject.name.Names; +import io.cdap.cdap.security.encryption.AeadCipher; + +/** + * Guice module for encryption bindings for user credential encryption. + */ +public class UserCredentialAeadEncryptionModule extends PrivateModule { + + public static final String USER_CREDENTIAL_ENCRYPTION = "UserCredentialEncryption"; + + @Override + protected void configure() { + // Bind user credential encryption providers. + bind(AeadCipherCryptorExtensionLoader.class).in(Scopes.SINGLETON); + bind(AeadCipher.class) + .annotatedWith(Names.named(USER_CREDENTIAL_ENCRYPTION)) + .toProvider(UserCredentialAeadCipherProvider.class).in(Scopes.SINGLETON); + expose(AeadCipher.class).annotatedWith(Names.named(USER_CREDENTIAL_ENCRYPTION)); + } +} diff --git a/cdap-security/src/test/java/io/cdap/cdap/security/auth/TinkCipherTest.java b/cdap-security/src/test/java/io/cdap/cdap/security/auth/TinkCipherTest.java deleted file mode 100644 index c250d4bfe853..000000000000 --- a/cdap-security/src/test/java/io/cdap/cdap/security/auth/TinkCipherTest.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright © 2021 Cask Data, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package io.cdap.cdap.security.auth; - -import com.google.crypto.tink.CleartextKeysetHandle; -import com.google.crypto.tink.JsonKeysetWriter; -import com.google.crypto.tink.KeyTemplates; -import com.google.crypto.tink.KeysetHandle; -import com.google.crypto.tink.aead.AeadConfig; -import io.cdap.cdap.common.conf.Constants; -import io.cdap.cdap.common.conf.SConfiguration; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.security.GeneralSecurityException; -import java.util.Random; -import org.junit.Assert; -import org.junit.Test; - -public class TinkCipherTest { - @Test - public void testEncryptionAndDecryption() throws CipherException, IOException, GeneralSecurityException { - SConfiguration sConf = SConfiguration.create(); - sConf.set(Constants.Security.Authentication.USER_CREDENTIAL_ENCRYPTION_KEYSET, generateKeySet()); - - TinkCipher cipher = new TinkCipher(sConf); - - byte[] plainData = generateRandomBytes(2 * 1024); - byte[] associatedData = generateRandomBytes(64); - byte[] cipherData = cipher.encrypt(plainData, associatedData); - byte[] decryptedData = cipher.decrypt(cipherData, associatedData); - Assert.assertArrayEquals(plainData, decryptedData); - - String cipherDataBase64Encoded = cipher.encryptToBase64(plainData, associatedData); - decryptedData = cipher.decryptFromBase64(cipherDataBase64Encoded, associatedData); - Assert.assertArrayEquals(plainData, decryptedData); - } - - @Test(expected = CipherException.class) - public void testInitException() throws CipherException { - SConfiguration sConf = SConfiguration.create(); - sConf.set(Constants.Security.Authentication.USER_CREDENTIAL_ENCRYPTION_KEYSET, "invalid keyset"); - - TinkCipher cipher = new TinkCipher(sConf); - } - - @Test(expected = CipherException.class) - public void testDecryptExceptionTagMismatch() throws CipherException, IOException, GeneralSecurityException { - SConfiguration sConf = SConfiguration.create(); - sConf.set(Constants.Security.Authentication.USER_CREDENTIAL_ENCRYPTION_KEYSET, generateKeySet()); - - TinkCipher cipher = new TinkCipher(sConf); - - byte[] plainData = generateRandomBytes(128); - byte[] associatedData = generateRandomBytes(64); - byte[] cipherData = cipher.encrypt(plainData, associatedData); - byte[] invalidAssociatedData = generateRandomBytes(64); - byte[] decryptedData = cipher.decrypt(cipherData, invalidAssociatedData); - } - - @Test(expected = CipherException.class) - public void testDecryptExceptionCorruption() throws CipherException, IOException, GeneralSecurityException { - SConfiguration sConf = SConfiguration.create(); - sConf.set(Constants.Security.Authentication.USER_CREDENTIAL_ENCRYPTION_KEYSET, generateKeySet()); - - TinkCipher cipher = new TinkCipher(sConf); - - byte[] plainData = generateRandomBytes(128); - byte[] associatedData = generateRandomBytes(64); - byte[] cipherData = cipher.encrypt(plainData, associatedData); - // Intentionally corrupt the cipher data. - cipherData[0] = 0; - byte[] decryptedData = cipher.decrypt(cipherData, associatedData); - } - - private String generateKeySet() throws IOException, GeneralSecurityException { - AeadConfig.register(); - KeysetHandle keysetHandle = KeysetHandle.generateNew(KeyTemplates.get("AES128_GCM")); - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - CleartextKeysetHandle.write(keysetHandle, JsonKeysetWriter.withOutputStream(outputStream)); - return outputStream.toString(); - } - - private byte[] generateRandomBytes(int len) { - byte[] bytes = new byte[len]; - new Random().nextBytes(bytes); - return bytes; - } -} diff --git a/cdap-security/src/test/java/io/cdap/cdap/security/authorization/DefaultAccessEnforcerTest.java b/cdap-security/src/test/java/io/cdap/cdap/security/authorization/DefaultAccessEnforcerTest.java index 5c518f80eeb6..688a9d87d99d 100644 --- a/cdap-security/src/test/java/io/cdap/cdap/security/authorization/DefaultAccessEnforcerTest.java +++ b/cdap-security/src/test/java/io/cdap/cdap/security/authorization/DefaultAccessEnforcerTest.java @@ -24,16 +24,12 @@ import static org.mockito.Mockito.when; import com.google.common.collect.ImmutableSet; -import com.google.crypto.tink.CleartextKeysetHandle; -import com.google.crypto.tink.JsonKeysetWriter; -import com.google.crypto.tink.KeyTemplates; -import com.google.crypto.tink.KeysetHandle; -import com.google.crypto.tink.aead.AeadConfig; import io.cdap.cdap.api.metrics.MetricsCollectionService; import io.cdap.cdap.api.metrics.MetricsContext; import io.cdap.cdap.api.security.AccessException; import io.cdap.cdap.common.conf.CConfiguration; import io.cdap.cdap.common.conf.Constants; +import io.cdap.cdap.common.conf.Constants.Security.Encryption; import io.cdap.cdap.common.conf.SConfiguration; import io.cdap.cdap.common.metrics.ProgramTypeMetricTag; import io.cdap.cdap.common.test.AppJarHelper; @@ -49,16 +45,13 @@ import io.cdap.cdap.proto.security.Permission; import io.cdap.cdap.proto.security.Principal; import io.cdap.cdap.proto.security.StandardPermission; -import io.cdap.cdap.security.auth.CipherException; -import io.cdap.cdap.security.auth.TinkCipher; +import io.cdap.cdap.security.encryption.FakeAeadCipher; import io.cdap.cdap.security.spi.authorization.AccessController; import io.cdap.cdap.security.spi.authorization.AccessEnforcer; import io.cdap.cdap.security.spi.authorization.NoOpAccessController; import io.cdap.cdap.security.spi.authorization.UnauthorizedException; -import java.io.ByteArrayOutputStream; +import io.cdap.cdap.security.spi.encryption.CipherException; import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.security.GeneralSecurityException; import java.util.Base64; import java.util.Collections; import java.util.EnumSet; @@ -84,14 +77,17 @@ public class DefaultAccessEnforcerTest extends AuthorizationTestBase { private static final Principal BOB = new Principal("bob", Principal.PrincipalType.USER); private static final NamespaceId NS = new NamespaceId("ns"); private static final ApplicationId APP = NS.app("app"); + private static FakeAeadCipher fakeAeadCipherService; private static class ControllerWrapper { + private final AccessController accessController; private final DefaultAccessEnforcer defaultAccessEnforcer; private final MetricsContext mockMetricsContext; - ControllerWrapper(AccessController accessController, DefaultAccessEnforcer defaultAccessEnforcer, - MetricsContext mockMetricsContext) { + ControllerWrapper(AccessController accessController, + DefaultAccessEnforcer defaultAccessEnforcer, + MetricsContext mockMetricsContext) { this.accessController = accessController; this.defaultAccessEnforcer = defaultAccessEnforcer; this.mockMetricsContext = mockMetricsContext; @@ -102,25 +98,32 @@ private static class ControllerWrapper { public ExpectedException thrown = ExpectedException.none(); @BeforeClass - public static void setupClass() throws IOException { + public static void setupClass() throws Exception { Manifest manifest = new Manifest(); - manifest.getMainAttributes().put(Attributes.Name.MAIN_CLASS, InMemoryAccessController.class.getName()); + manifest.getMainAttributes() + .put(Attributes.Name.MAIN_CLASS, InMemoryAccessController.class.getName()); Location externalAuthJar = AppJarHelper.createDeploymentJar( - locationFactory, InMemoryAccessController.class, manifest); + locationFactory, InMemoryAccessController.class, manifest); CCONF.set(Constants.Security.Authorization.EXTENSION_JAR_PATH, externalAuthJar.toString()); + fakeAeadCipherService = new FakeAeadCipher(); + fakeAeadCipherService.initialize(); } - private static ControllerWrapper createControllerWrapper(CConfiguration cConf, SConfiguration sConf, - AccessEnforcer internalAccessEnforcer) { + private static ControllerWrapper createControllerWrapper(CConfiguration cConf, + SConfiguration sConf, + AccessEnforcer internalAccessEnforcer) { MetricsCollectionService mockMetricsCollectionService = mock(MetricsCollectionService.class); MetricsContext mockMetricsContext = mock(MetricsContext.class); when(mockMetricsCollectionService.getContext(any(Map.class))).thenReturn(mockMetricsContext); - AccessControllerInstantiator accessControllerInstantiator = new AccessControllerInstantiator(cConf, - AUTH_CONTEXT_FACTORY); - DefaultAccessEnforcer defaultAccessEnforcer = new DefaultAccessEnforcer(cConf, sConf, accessControllerInstantiator, - internalAccessEnforcer, - mockMetricsCollectionService); - return new ControllerWrapper(accessControllerInstantiator.get(), defaultAccessEnforcer, mockMetricsContext); + AccessControllerInstantiator accessControllerInstantiator = new AccessControllerInstantiator( + cConf, + AUTH_CONTEXT_FACTORY); + DefaultAccessEnforcer defaultAccessEnforcer = new DefaultAccessEnforcer(cConf, sConf, + accessControllerInstantiator, + internalAccessEnforcer, + mockMetricsCollectionService, fakeAeadCipherService); + return new ControllerWrapper(accessControllerInstantiator.get(), defaultAccessEnforcer, + mockMetricsContext); } @Test @@ -138,11 +141,11 @@ public void testAuthorizationDisabled() throws IOException, AccessException { } @Test - public void testPropagationDisabled() throws IOException, AccessException { + public void testPropagationDisabled() throws AccessException { CConfiguration cConfCopy = CConfiguration.copy(CCONF); ControllerWrapper controllerWrapper = createControllerWrapper(cConfCopy, SCONF, null); controllerWrapper.accessController.grant(Authorizable.fromEntityId(NS), ALICE, - ImmutableSet.of(StandardPermission.UPDATE)); + ImmutableSet.of(StandardPermission.UPDATE)); DefaultAccessEnforcer accessEnforcer = controllerWrapper.defaultAccessEnforcer; accessEnforcer.enforce(NS, ALICE, StandardPermission.UPDATE); try { @@ -153,11 +156,11 @@ public void testPropagationDisabled() throws IOException, AccessException { } // Verify the metrics context was called with correct metrics verify(controllerWrapper.mockMetricsContext, times(1)) - .increment(Constants.Metrics.Authorization.EXTENSION_CHECK_SUCCESS_COUNT, 1); + .increment(Constants.Metrics.Authorization.EXTENSION_CHECK_SUCCESS_COUNT, 1); verify(controllerWrapper.mockMetricsContext, times(1)) - .increment(Constants.Metrics.Authorization.EXTENSION_CHECK_FAILURE_COUNT, 1); + .increment(Constants.Metrics.Authorization.EXTENSION_CHECK_FAILURE_COUNT, 1); verify(controllerWrapper.mockMetricsContext, times(2)) - .gauge(eq(Constants.Metrics.Authorization.EXTENSION_CHECK_MILLIS), any(Long.class)); + .gauge(eq(Constants.Metrics.Authorization.EXTENSION_CHECK_MILLIS), any(Long.class)); } @Test @@ -170,27 +173,33 @@ public void testAuthEnforce() throws IOException, AccessException { // grant some test privileges DatasetId ds = NS.dataset("ds"); - accessController.grant(Authorizable.fromEntityId(NS), ALICE, ImmutableSet.of(StandardPermission.GET, - StandardPermission.UPDATE)); - accessController.grant(Authorizable.fromEntityId(ds), BOB, ImmutableSet.of(StandardPermission.UPDATE)); + accessController + .grant(Authorizable.fromEntityId(NS), ALICE, ImmutableSet.of(StandardPermission.GET, + StandardPermission.UPDATE)); + accessController + .grant(Authorizable.fromEntityId(ds), BOB, ImmutableSet.of(StandardPermission.UPDATE)); accessController.grant(Authorizable.fromEntityId(NS, EntityType.DATASET), ALICE, - ImmutableSet.of(StandardPermission.LIST)); + ImmutableSet.of(StandardPermission.LIST)); // auth enforcement for alice should succeed on ns for actions read, write and list datasets - authEnforcementService.enforce(NS, ALICE, ImmutableSet.of(StandardPermission.GET, StandardPermission.UPDATE)); + authEnforcementService + .enforce(NS, ALICE, ImmutableSet.of(StandardPermission.GET, StandardPermission.UPDATE)); authEnforcementService.enforceOnParent(EntityType.DATASET, NS, ALICE, StandardPermission.LIST); - assertAuthorizationFailure(authEnforcementService, NS, ALICE, EnumSet.allOf(StandardPermission.class)); + assertAuthorizationFailure(authEnforcementService, NS, ALICE, + EnumSet.allOf(StandardPermission.class)); // alice do not have CREATE, READ or WRITE on the dataset, so authorization should fail assertAuthorizationFailure(authEnforcementService, ds, ALICE, StandardPermission.GET); assertAuthorizationFailure(authEnforcementService, ds, ALICE, StandardPermission.UPDATE); - assertAuthorizationFailure(authEnforcementService, EntityType.DATASET, NS, ALICE, StandardPermission.CREATE); + assertAuthorizationFailure(authEnforcementService, EntityType.DATASET, NS, ALICE, + StandardPermission.CREATE); // Alice doesn't have Delete right on NS, hence should fail. assertAuthorizationFailure(authEnforcementService, NS, ALICE, StandardPermission.DELETE); // bob enforcement should succeed since we grant him admin privilege authEnforcementService.enforce(ds, BOB, StandardPermission.UPDATE); // revoke all of alice's privileges - accessController.revoke(Authorizable.fromEntityId(NS), ALICE, ImmutableSet.of(StandardPermission.GET)); + accessController + .revoke(Authorizable.fromEntityId(NS), ALICE, ImmutableSet.of(StandardPermission.GET)); assertAuthorizationFailure(authEnforcementService, NS, ALICE, StandardPermission.GET); accessController.revoke(Authorizable.fromEntityId(NS)); @@ -200,15 +209,15 @@ public void testAuthEnforce() throws IOException, AccessException { authEnforcementService.enforce(ds, BOB, StandardPermission.UPDATE); // Verify the metrics context was called with correct metrics verify(controllerWrapper.mockMetricsContext, times(4)) - .increment(Constants.Metrics.Authorization.EXTENSION_CHECK_SUCCESS_COUNT, 1); + .increment(Constants.Metrics.Authorization.EXTENSION_CHECK_SUCCESS_COUNT, 1); verify(controllerWrapper.mockMetricsContext, times(9)) - .increment(Constants.Metrics.Authorization.EXTENSION_CHECK_FAILURE_COUNT, 1); + .increment(Constants.Metrics.Authorization.EXTENSION_CHECK_FAILURE_COUNT, 1); verify(controllerWrapper.mockMetricsContext, times(13)) - .gauge(eq(Constants.Metrics.Authorization.EXTENSION_CHECK_MILLIS), any(Long.class)); + .gauge(eq(Constants.Metrics.Authorization.EXTENSION_CHECK_MILLIS), any(Long.class)); } @Test - public void testIsVisible() throws IOException, AccessException { + public void testIsVisible() throws AccessException { ControllerWrapper controllerWrapper = createControllerWrapper(CCONF, SCONF, null); AccessController accessController = controllerWrapper.accessController; DefaultAccessEnforcer authEnforcementService = controllerWrapper.defaultAccessEnforcer; @@ -222,106 +231,117 @@ public void testIsVisible() throws IOException, AccessException { DatasetId ds23 = ns2.dataset("ds33"); Set namespaces = ImmutableSet.of(ns1, ns2); // Alice has access on ns1, ns2, ds11, ds21, ds23, Bob has access on ds11, ds12, ds22 - accessController.grant(Authorizable.fromEntityId(ns1), ALICE, Collections.singleton(StandardPermission.UPDATE)); - accessController.grant(Authorizable.fromEntityId(ns2), ALICE, Collections.singleton(StandardPermission.UPDATE)); - accessController.grant(Authorizable.fromEntityId(ds11), ALICE, Collections.singleton(StandardPermission.GET)); - accessController.grant(Authorizable.fromEntityId(ds11), BOB, Collections.singleton(StandardPermission.UPDATE)); - accessController.grant(Authorizable.fromEntityId(ds21), ALICE, Collections.singleton(StandardPermission.UPDATE)); - accessController.grant(Authorizable.fromEntityId(ds12), BOB, Collections.singleton(StandardPermission.UPDATE)); - accessController.grant(Authorizable.fromEntityId(ds12), BOB, EnumSet.allOf(StandardPermission.class)); - accessController.grant(Authorizable.fromEntityId(ds21), ALICE, Collections.singleton(StandardPermission.UPDATE)); - accessController.grant(Authorizable.fromEntityId(ds23), ALICE, Collections.singleton(StandardPermission.UPDATE)); - accessController.grant(Authorizable.fromEntityId(ds22), BOB, Collections.singleton(StandardPermission.UPDATE)); - - Assert.assertEquals(namespaces.size(), authEnforcementService.isVisible(namespaces, ALICE).size()); + accessController.grant(Authorizable.fromEntityId(ns1), ALICE, + Collections.singleton(StandardPermission.UPDATE)); + accessController.grant(Authorizable.fromEntityId(ns2), ALICE, + Collections.singleton(StandardPermission.UPDATE)); + accessController.grant(Authorizable.fromEntityId(ds11), ALICE, + Collections.singleton(StandardPermission.GET)); + accessController.grant(Authorizable.fromEntityId(ds11), BOB, + Collections.singleton(StandardPermission.UPDATE)); + accessController.grant(Authorizable.fromEntityId(ds21), ALICE, + Collections.singleton(StandardPermission.UPDATE)); + accessController.grant(Authorizable.fromEntityId(ds12), BOB, + Collections.singleton(StandardPermission.UPDATE)); + accessController + .grant(Authorizable.fromEntityId(ds12), BOB, EnumSet.allOf(StandardPermission.class)); + accessController.grant(Authorizable.fromEntityId(ds21), ALICE, + Collections.singleton(StandardPermission.UPDATE)); + accessController.grant(Authorizable.fromEntityId(ds23), ALICE, + Collections.singleton(StandardPermission.UPDATE)); + accessController.grant(Authorizable.fromEntityId(ds22), BOB, + Collections.singleton(StandardPermission.UPDATE)); + + Assert.assertEquals(namespaces.size(), + authEnforcementService.isVisible(namespaces, ALICE).size()); // bob should also be able to list two namespaces since he has privileges on the dataset in both namespaces - Assert.assertEquals(namespaces.size(), authEnforcementService.isVisible(namespaces, BOB).size()); + Assert + .assertEquals(namespaces.size(), authEnforcementService.isVisible(namespaces, BOB).size()); Set expectedDatasetIds = ImmutableSet.of(ds11, ds21, ds23); - Assert.assertEquals(expectedDatasetIds.size(), authEnforcementService.isVisible(expectedDatasetIds, - ALICE).size()); + Assert.assertEquals(expectedDatasetIds.size(), + authEnforcementService.isVisible(expectedDatasetIds, + ALICE).size()); expectedDatasetIds = ImmutableSet.of(ds12, ds22); // this will be empty since now isVisible will not check the hierarchy privilege for the parent of the entity - Assert.assertEquals(Collections.EMPTY_SET, authEnforcementService.isVisible(expectedDatasetIds, ALICE)); + Assert.assertEquals(Collections.EMPTY_SET, + authEnforcementService.isVisible(expectedDatasetIds, ALICE)); expectedDatasetIds = ImmutableSet.of(ds11, ds12, ds22); - Assert.assertEquals(expectedDatasetIds.size(), authEnforcementService.isVisible(expectedDatasetIds, BOB).size()); + Assert.assertEquals(expectedDatasetIds.size(), + authEnforcementService.isVisible(expectedDatasetIds, BOB).size()); expectedDatasetIds = ImmutableSet.of(ds21, ds23); Assert.assertTrue(authEnforcementService.isVisible(expectedDatasetIds, BOB).isEmpty()); // Verify the metrics context was called with correct metrics verify(controllerWrapper.mockMetricsContext, times(6)) - .increment(Constants.Metrics.Authorization.NON_INTERNAL_VISIBILITY_CHECK_COUNT, 1); + .increment(Constants.Metrics.Authorization.NON_INTERNAL_VISIBILITY_CHECK_COUNT, 1); verify(controllerWrapper.mockMetricsContext, times(6)) - .gauge(eq(Constants.Metrics.Authorization.EXTENSION_VISIBILITY_MILLIS), any(Long.class)); + .gauge(eq(Constants.Metrics.Authorization.EXTENSION_VISIBILITY_MILLIS), any(Long.class)); } @Test - public void testAuthEnforceWithEncryptedCredential() - throws IOException, AccessException, CipherException, GeneralSecurityException { + public void testAuthEnforceWithEncryptedCredential() throws AccessException, CipherException { SConfiguration sConfCopy = enableCredentialEncryption(); - TinkCipher cipher = new TinkCipher(sConfCopy); - - String cred = cipher.encryptToBase64("credential".getBytes(StandardCharsets.UTF_8), null); + String cred = fakeAeadCipherService.encryptToBase64("credential", + Encryption.USER_CREDENTIAL_ENCRYPTION_ASSOCIATED_DATA.getBytes()); Principal userWithCredEncrypted = new Principal("userFoo", Principal.PrincipalType.USER, null, - new Credential(cred, Credential.CredentialType.EXTERNAL_ENCRYPTED)); + new Credential(cred, Credential.CredentialType.EXTERNAL_ENCRYPTED)); ControllerWrapper controllerWrapper = createControllerWrapper(CCONF, sConfCopy, null); AccessController accessController = controllerWrapper.accessController; DefaultAccessEnforcer accessEnforcer = controllerWrapper.defaultAccessEnforcer; - assertAuthorizationFailure(accessEnforcer, NS, userWithCredEncrypted, StandardPermission.UPDATE); + assertAuthorizationFailure(accessEnforcer, NS, userWithCredEncrypted, + StandardPermission.UPDATE); accessController.grant(Authorizable.fromEntityId(NS), userWithCredEncrypted, - ImmutableSet.of(StandardPermission.GET, StandardPermission.UPDATE)); + ImmutableSet.of(StandardPermission.GET, StandardPermission.UPDATE)); accessEnforcer.enforce(NS, userWithCredEncrypted, StandardPermission.GET); accessEnforcer.enforce(NS, userWithCredEncrypted, StandardPermission.UPDATE); // Verify the metrics context was called with correct metrics verify(controllerWrapper.mockMetricsContext, times(2)) - .increment(Constants.Metrics.Authorization.EXTENSION_CHECK_SUCCESS_COUNT, 1); + .increment(Constants.Metrics.Authorization.EXTENSION_CHECK_SUCCESS_COUNT, 1); verify(controllerWrapper.mockMetricsContext, times(1)) - .increment(Constants.Metrics.Authorization.EXTENSION_CHECK_FAILURE_COUNT, 1); + .increment(Constants.Metrics.Authorization.EXTENSION_CHECK_FAILURE_COUNT, 1); verify(controllerWrapper.mockMetricsContext, times(3)) - .gauge(eq(Constants.Metrics.Authorization.EXTENSION_CHECK_MILLIS), any(Long.class)); + .gauge(eq(Constants.Metrics.Authorization.EXTENSION_CHECK_MILLIS), any(Long.class)); } @Test public void testAuthEnforceWithBadEncryptedCredential() - throws IOException, AccessException, CipherException, GeneralSecurityException { + throws AccessException { thrown.expect(Exception.class); thrown.expectMessage("Failed to decrypt credential in principle:"); SConfiguration sConfCopy = enableCredentialEncryption(); - TinkCipher cipher = new TinkCipher(sConfCopy); - - String badCipherCred = Base64.getEncoder().encodeToString("invalid encrypted credential".getBytes()); + String badCipherCred = Base64.getEncoder() + .encodeToString("invalid encrypted credential".getBytes()); Principal userWithCredEncrypted = new Principal("userFoo", Principal.PrincipalType.USER, null, - new Credential(badCipherCred, - Credential.CredentialType.EXTERNAL_ENCRYPTED)); + new Credential(badCipherCred, + Credential.CredentialType.EXTERNAL_ENCRYPTED)); ControllerWrapper controllerWrapper = createControllerWrapper(CCONF, sConfCopy, null); AccessController accessController = controllerWrapper.accessController; DefaultAccessEnforcer accessEnforcer = controllerWrapper.defaultAccessEnforcer; accessController.grant(Authorizable.fromEntityId(NS), userWithCredEncrypted, - ImmutableSet.of(StandardPermission.GET, StandardPermission.GET)); + ImmutableSet.of(StandardPermission.GET, StandardPermission.GET)); accessEnforcer.enforce(NS, userWithCredEncrypted, StandardPermission.GET); // Verify the metrics context was not called verify(controllerWrapper.mockMetricsContext, times(0)) - .increment(any(String.class), any(Long.class)); + .increment(any(String.class), any(Long.class)); verify(controllerWrapper.mockMetricsContext, times(0)) - .gauge(any(String.class), any(Long.class)); + .gauge(any(String.class), any(Long.class)); } @Test - public void testIsVisibleWithEncryptedCredential() - throws IOException, AccessException, CipherException, GeneralSecurityException { + public void testIsVisibleWithEncryptedCredential() throws AccessException, CipherException { SConfiguration sConfCopy = enableCredentialEncryption(); - TinkCipher cipher = new TinkCipher(sConfCopy); - - String cred = cipher.encryptToBase64("credential".getBytes(StandardCharsets.UTF_8), null); + String cred = fakeAeadCipherService.encryptToBase64("credential", + Encryption.USER_CREDENTIAL_ENCRYPTION_ASSOCIATED_DATA.getBytes()); Principal userWithCredEncrypted = new Principal("userFoo", Principal.PrincipalType.USER, null, - new Credential(cred, Credential.CredentialType.EXTERNAL_ENCRYPTED)); + new Credential(cred, Credential.CredentialType.EXTERNAL_ENCRYPTED)); ControllerWrapper controllerWrapper = createControllerWrapper(CCONF, sConfCopy, null); AccessController accessController = controllerWrapper.accessController; @@ -332,75 +352,80 @@ public void testIsVisibleWithEncryptedCredential() Assert.assertEquals(0, accessEnforcer.isVisible(namespaces, userWithCredEncrypted).size()); accessController.grant(Authorizable.fromEntityId(NS), userWithCredEncrypted, - ImmutableSet.of(StandardPermission.GET, StandardPermission.UPDATE)); + ImmutableSet.of(StandardPermission.GET, StandardPermission.UPDATE)); Assert.assertEquals(1, accessEnforcer.isVisible(namespaces, userWithCredEncrypted).size()); // Verify the metrics context was called with correct metrics verify(controllerWrapper.mockMetricsContext, times(2)) - .increment(Constants.Metrics.Authorization.NON_INTERNAL_VISIBILITY_CHECK_COUNT, 1); + .increment(Constants.Metrics.Authorization.NON_INTERNAL_VISIBILITY_CHECK_COUNT, 1); verify(controllerWrapper.mockMetricsContext, times(2)) - .gauge(eq(Constants.Metrics.Authorization.EXTENSION_VISIBILITY_MILLIS), any(Long.class)); + .gauge(eq(Constants.Metrics.Authorization.EXTENSION_VISIBILITY_MILLIS), any(Long.class)); } @Test public void testSystemUser() throws IOException, AccessException { CConfiguration cConfCopy = CConfiguration.copy(CCONF); Principal systemUser = - new Principal(UserGroupInformation.getCurrentUser().getShortUserName(), Principal.PrincipalType.USER); + new Principal(UserGroupInformation.getCurrentUser().getShortUserName(), + Principal.PrincipalType.USER); ControllerWrapper controllerWrapper = createControllerWrapper(cConfCopy, SCONF, null); DefaultAccessEnforcer accessEnforcer = controllerWrapper.defaultAccessEnforcer; NamespaceId ns1 = new NamespaceId("ns1"); accessEnforcer.enforce(NamespaceId.SYSTEM, systemUser, EnumSet.allOf(StandardPermission.class)); accessEnforcer.enforce(NamespaceId.SYSTEM, systemUser, StandardPermission.GET); Assert.assertEquals(ImmutableSet.of(NamespaceId.SYSTEM), - accessEnforcer.isVisible(ImmutableSet.of(ns1, NamespaceId.SYSTEM), - systemUser)); + accessEnforcer.isVisible(ImmutableSet.of(ns1, NamespaceId.SYSTEM), + systemUser)); // Verify the metrics context was called with correct metrics verify(controllerWrapper.mockMetricsContext, times(2)) - .increment(Constants.Metrics.Authorization.EXTENSION_CHECK_BYPASS_COUNT, 1); + .increment(Constants.Metrics.Authorization.EXTENSION_CHECK_BYPASS_COUNT, 1); } @Test public void testInternalAuthEnforce() throws IOException, AccessException { Principal userWithInternalCred = new Principal("system", Principal.PrincipalType.USER, null, - new Credential("credential", - Credential.CredentialType.INTERNAL)); + new Credential("credential", + Credential.CredentialType.INTERNAL)); CConfiguration cConfCopy = CConfiguration.copy(CCONF); cConfCopy.setBoolean(Constants.Security.INTERNAL_AUTH_ENABLED, true); - ControllerWrapper controllerWrapper = createControllerWrapper(cConfCopy, SCONF, new NoOpAccessController()); + ControllerWrapper controllerWrapper = createControllerWrapper(cConfCopy, SCONF, + new NoOpAccessController()); AccessController accessController = controllerWrapper.accessController; DefaultAccessEnforcer accessEnforcer = controllerWrapper.defaultAccessEnforcer; // Make sure that the actual access controller does not have access. assertAuthorizationFailure(accessController, NS, userWithInternalCred, StandardPermission.GET); - assertAuthorizationFailure(accessController, NS, userWithInternalCred, StandardPermission.UPDATE); + assertAuthorizationFailure(accessController, NS, userWithInternalCred, + StandardPermission.UPDATE); // The no-op access enforcer allows all requests through, so this should succeed if it is using the right // access controller. accessEnforcer.enforce(NS, userWithInternalCred, StandardPermission.GET); accessEnforcer.enforce(NS, userWithInternalCred, StandardPermission.UPDATE); // Verify the metrics context was called with correct metrics verify(controllerWrapper.mockMetricsContext, times(2)) - .increment(Constants.Metrics.Authorization.INTERNAL_CHECK_SUCCESS_COUNT, 1); + .increment(Constants.Metrics.Authorization.INTERNAL_CHECK_SUCCESS_COUNT, 1); } @Test - public void testInternalIsVisible() throws IOException, AccessException { + public void testInternalIsVisible() throws AccessException { Principal userWithInternalCred = new Principal("system", Principal.PrincipalType.USER, null, - new Credential("credential", - Credential.CredentialType.INTERNAL)); + new Credential("credential", + Credential.CredentialType.INTERNAL)); CConfiguration cConfCopy = CConfiguration.copy(CCONF); cConfCopy.setBoolean(Constants.Security.INTERNAL_AUTH_ENABLED, true); - ControllerWrapper controllerWrapper = createControllerWrapper(cConfCopy, SCONF, new NoOpAccessController()); + ControllerWrapper controllerWrapper = createControllerWrapper(cConfCopy, SCONF, + new NoOpAccessController()); AccessController accessController = controllerWrapper.accessController; DefaultAccessEnforcer accessEnforcer = controllerWrapper.defaultAccessEnforcer; Set namespaces = ImmutableSet.of(NS); // Make sure that the actual access controller does not have access. - Assert.assertEquals(Collections.emptySet(), accessController.isVisible(namespaces, userWithInternalCred)); + Assert.assertEquals(Collections.emptySet(), + accessController.isVisible(namespaces, userWithInternalCred)); // The no-op access enforcer allows all requests through, so this should succeed if it is using the right // access controller. Assert.assertEquals(namespaces, accessEnforcer.isVisible(namespaces, userWithInternalCred)); // Verify the metrics context was called with correct metrics verify(controllerWrapper.mockMetricsContext, times(1)) - .increment(Constants.Metrics.Authorization.INTERNAL_VISIBILITY_CHECK_COUNT, 1); + .increment(Constants.Metrics.Authorization.INTERNAL_VISIBILITY_CHECK_COUNT, 1); } @Test @@ -411,14 +436,22 @@ public void testMetricsContextNotCalledIfDisabled() throws IOException, AccessEx AccessController accessController = controllerWrapper.accessController; DefaultAccessEnforcer accessEnforcer = controllerWrapper.defaultAccessEnforcer; DatasetId ds = NS.dataset("ds"); - accessController.grant(Authorizable.fromEntityId(NS), ALICE, ImmutableSet.of(StandardPermission.GET, - StandardPermission.UPDATE)); - accessEnforcer.enforce(NS, ALICE, ImmutableSet.of(StandardPermission.GET, StandardPermission.UPDATE)); + accessController + .grant(Authorizable.fromEntityId(NS), ALICE, ImmutableSet.of(StandardPermission.GET, + StandardPermission.UPDATE)); + accessEnforcer + .enforce(NS, ALICE, ImmutableSet.of(StandardPermission.GET, StandardPermission.UPDATE)); // Verify the metrics context was not called verify(controllerWrapper.mockMetricsContext, times(0)) - .increment(any(String.class), any(Long.class)); + .increment(any(String.class), any(Long.class)); verify(controllerWrapper.mockMetricsContext, times(0)) - .gauge(any(String.class), any(Long.class)); + .gauge(any(String.class), any(Long.class)); + } + + private SConfiguration enableCredentialEncryption() { + SConfiguration sConfCopy = SConfiguration.copy(SCONF); + sConfCopy.set(Constants.Security.Authentication.USER_CREDENTIAL_ENCRYPTION_ENABLED, "true"); + return sConfCopy; } private void verifyDisabled(CConfiguration cConf) throws IOException, AccessException { @@ -428,71 +461,57 @@ private void verifyDisabled(CConfiguration cConf) throws IOException, AccessExce DatasetId ds = NS.dataset("ds"); // All enforcement operations should succeed, since authorization is disabled accessController.grant(Authorizable.fromEntityId(ds), BOB, - ImmutableSet.of(StandardPermission.UPDATE)); + ImmutableSet.of(StandardPermission.UPDATE)); authEnforcementService.enforce(NS, ALICE, StandardPermission.UPDATE); authEnforcementService.enforce(ds, BOB, StandardPermission.UPDATE); authEnforcementService.enforce(NS, BOB, StandardPermission.GET); authEnforcementService.enforce(ds, BOB, StandardPermission.GET); - Assert.assertEquals(2, authEnforcementService.isVisible(ImmutableSet.of(NS, ds), BOB).size()); + Assert.assertEquals(2, + authEnforcementService.isVisible(ImmutableSet.of(NS, ds), BOB).size()); // Verify the metrics context was not called verify(controllerWrapper.mockMetricsContext, times(0)) - .increment(any(String.class), any(Long.class)); + .increment(any(String.class), any(Long.class)); verify(controllerWrapper.mockMetricsContext, times(0)) - .gauge(any(String.class), any(Long.class)); + .gauge(any(String.class), any(Long.class)); } private void assertAuthorizationFailure(AccessEnforcer authEnforcementService, - EntityId entityId, Principal principal, - Permission permission) throws AccessException { + EntityId entityId, Principal principal, + Permission permission) throws AccessException { try { authEnforcementService.enforce(entityId, principal, permission); Assert.fail(String.format("Expected %s to not have '%s' privilege on %s but it does.", - principal, permission, entityId)); + principal, permission, entityId)); } catch (UnauthorizedException expected) { // expected } } - private void assertAuthorizationFailure(AccessEnforcer authEnforcementService, EntityType entityType, - EntityId parentId, Principal principal, - Permission permission) throws AccessException { + private void assertAuthorizationFailure(AccessEnforcer authEnforcementService, + EntityType entityType, + EntityId parentId, Principal principal, + Permission permission) throws AccessException { try { authEnforcementService.enforceOnParent(entityType, parentId, principal, permission); Assert.fail(String.format("Expected %s to not have '%s' privilege on %s in %s but it does.", - principal, permission, entityType, parentId)); + principal, permission, entityType, parentId)); } catch (UnauthorizedException expected) { // expected } } private void assertAuthorizationFailure(AccessEnforcer authEnforcementService, - EntityId entityId, Principal principal, - Set permissions) throws AccessException { + EntityId entityId, Principal principal, + Set permissions) throws AccessException { try { authEnforcementService.enforce(entityId, principal, permissions); Assert.fail(String.format("Expected %s to not have '%s' privileges on %s but it does.", - principal, permissions, entityId)); + principal, permissions, entityId)); } catch (UnauthorizedException expected) { // expected } } - private SConfiguration enableCredentialEncryption() throws IOException, GeneralSecurityException { - SConfiguration sConfCopy = SConfiguration.copy(SCONF); - sConfCopy.set(Constants.Security.Authentication.USER_CREDENTIAL_ENCRYPTION_ENABLED, "true"); - sConfCopy.set(Constants.Security.Authentication.USER_CREDENTIAL_ENCRYPTION_KEYSET, - generateEncryptionKeyset()); - return sConfCopy; - } - - private String generateEncryptionKeyset() throws IOException, GeneralSecurityException { - AeadConfig.register(); - KeysetHandle keysetHandle = KeysetHandle.generateNew(KeyTemplates.get("AES128_GCM")); - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - CleartextKeysetHandle.write(keysetHandle, JsonKeysetWriter.withOutputStream(outputStream)); - return outputStream.toString(); - } - @Test public void testExpectedMetricsTagsForEntityId() { String namespaceName = "namespace"; @@ -515,7 +534,8 @@ public void testExpectedMetricsTagsForChildEntityId() { expectedTags.put(Constants.Metrics.Tag.NAMESPACE, namespaceName); expectedTags.put(Constants.Metrics.Tag.APP, appName); expectedTags.put(Constants.Metrics.Tag.PROGRAM, programName); - expectedTags.put(Constants.Metrics.Tag.PROGRAM_TYPE, ProgramTypeMetricTag.getTagName(programId.getType())); + expectedTags.put(Constants.Metrics.Tag.PROGRAM_TYPE, + ProgramTypeMetricTag.getTagName(programId.getType())); Map tags = DefaultAccessEnforcer.createEntityIdMetricsTags(programId); Assert.assertEquals(expectedTags, tags); } diff --git a/cdap-security/src/test/java/io/cdap/cdap/security/encryption/DataStorageAeadEncryptionModuleTest.java b/cdap-security/src/test/java/io/cdap/cdap/security/encryption/DataStorageAeadEncryptionModuleTest.java new file mode 100644 index 000000000000..14aace96fc39 --- /dev/null +++ b/cdap-security/src/test/java/io/cdap/cdap/security/encryption/DataStorageAeadEncryptionModuleTest.java @@ -0,0 +1,39 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.security.encryption; + +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.Key; +import com.google.inject.name.Names; +import io.cdap.cdap.common.guice.ConfigModule; +import io.cdap.cdap.security.encryption.guice.DataStorageAeadEncryptionModule; +import org.junit.Test; + +/** + * Tests for {@link DataStorageAeadEncryptionModule}. + */ +public class DataStorageAeadEncryptionModuleTest { + + @Test + public void testBinding() { + Injector injector = Guice + .createInjector(new ConfigModule(), new DataStorageAeadEncryptionModule()); + injector.getInstance(Key.get(AeadCipher.class, + Names.named(DataStorageAeadEncryptionModule.DATA_STORAGE_ENCRYPTION))); + } +} diff --git a/cdap-security/src/test/java/io/cdap/cdap/security/encryption/FakeAeadCipher.java b/cdap-security/src/test/java/io/cdap/cdap/security/encryption/FakeAeadCipher.java new file mode 100644 index 000000000000..41001e7321af --- /dev/null +++ b/cdap-security/src/test/java/io/cdap/cdap/security/encryption/FakeAeadCipher.java @@ -0,0 +1,167 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.security.encryption; + +import com.google.gson.Gson; +import com.google.gson.JsonSyntaxException; +import io.cdap.cdap.security.spi.encryption.CipherOperationException; +import java.nio.charset.StandardCharsets; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.util.Arrays; +import javax.crypto.Cipher; +import javax.crypto.KeyGenerator; +import javax.crypto.SecretKey; +import javax.crypto.spec.IvParameterSpec; + +/** + * An {@link AeadCipher} used for testing. NOTE: DO NOT USE THIS OUTSIDE OF UNIT TESTS; THIS IS NOT + * INTENDED TO BE SECURE. + */ +public class FakeAeadCipher implements AeadCipher { + + private static final Gson GSON = new Gson(); + // Prefix for validating that the decryption took place successfully. + // This is necessary for throwing an exception. + private static final String VALIDATION_PREFIX = "validation-prefix"; + + private SecureRandom secureRandom; + private SecretKey secretKey; + + public FakeAeadCipher() { + secureRandom = new SecureRandom(); + } + + public void initialize() throws NoSuchAlgorithmException { + KeyGenerator keyGenerator = KeyGenerator.getInstance("AES"); + keyGenerator.init(256); + secretKey = keyGenerator.generateKey(); + } + + @Override + public byte[] encrypt(byte[] plainData, byte[] associatedData) throws CipherOperationException { + Cipher cipher = createCipher(); + byte[] iv = generateIv(); + try { + cipher.init(Cipher.ENCRYPT_MODE, secretKey, new IvParameterSpec(iv)); + } catch (Exception e) { + throw new CipherOperationException("Failed to initialize Cipher", e); + } + byte[] encrypted; + try { + encrypted = cipher.doFinal(encodePlaintext(plainData, associatedData)); + } catch (Exception e) { + throw new CipherOperationException("Failed to encrypt data", e); + } + return encodeCiphertext(iv, encrypted); + } + + @Override + public byte[] decrypt(byte[] cipherData, byte[] associatedData) throws CipherOperationException { + Cipher cipher = createCipher(); + CipherTextWrapper cipherText = decodeCiphertext(cipherData); + try { + cipher.init(Cipher.DECRYPT_MODE, secretKey, new IvParameterSpec(cipherText.iv)); + } catch (Exception e) { + throw new CipherOperationException("Failed to initialize Cipher", e); + } + + byte[] decrypted; + try { + decrypted = cipher.doFinal(cipherText.ciphertext); + } catch (Exception e) { + throw new CipherOperationException("Failed to decrypt data", e); + } + + PlainTextWrapper plainTextWrapper = decodePlaintext(decrypted); + if (!Arrays.equals(associatedData, plainTextWrapper.associatedData)) { + throw new CipherOperationException( + String.format( + "Failed to validate decrypted ciphertext: unexpected associated data: want '%s', " + + "got '%s'", + new String(associatedData, StandardCharsets.UTF_8), + new String(plainTextWrapper.associatedData, StandardCharsets.UTF_8))); + } + return plainTextWrapper.plainData; + } + + private byte[] generateIv() { + byte[] iv = new byte[16]; + secureRandom.nextBytes(iv); + return iv; + } + + private Cipher createCipher() throws CipherOperationException { + Cipher cipher; + try { + cipher = Cipher.getInstance("AES/CBC/PKCS5Padding"); + } catch (Exception e) { + throw new CipherOperationException("Failed to create Cipher", e); + } + return cipher; + } + + private class PlainTextWrapper { + + private final byte[] plainData; + private final byte[] associatedData; + + private PlainTextWrapper(byte[] plainData, byte[] associatedData) { + this.plainData = plainData; + this.associatedData = associatedData; + } + } + + private byte[] encodePlaintext(byte[] plainData, byte[] associatedData) { + return (VALIDATION_PREFIX + GSON.toJson(new PlainTextWrapper(plainData, associatedData))) + .getBytes(StandardCharsets.UTF_8); + } + + private PlainTextWrapper decodePlaintext(byte[] plaintext) throws CipherOperationException { + String plaintextStr = new String(plaintext, StandardCharsets.UTF_8); + if (!plaintextStr.startsWith(VALIDATION_PREFIX)) { + throw new CipherOperationException("Decryption failed! Plaintext does not start with " + + "expected prefix."); + } + return GSON.fromJson(plaintextStr.substring(VALIDATION_PREFIX.length()), + PlainTextWrapper.class); + } + + private class CipherTextWrapper { + + private final byte[] iv; + private final byte[] ciphertext; + + public CipherTextWrapper(byte[] iv, byte[] ciphertext) { + this.iv = iv; + this.ciphertext = ciphertext; + } + } + + private byte[] encodeCiphertext(byte[] iv, byte[] ciphertext) { + return GSON.toJson(new CipherTextWrapper(iv, ciphertext)) + .getBytes(StandardCharsets.UTF_8); + } + + private CipherTextWrapper decodeCiphertext(byte[] ciphertext) throws CipherOperationException { + try { + return GSON.fromJson(new String(ciphertext, StandardCharsets.UTF_8), CipherTextWrapper.class); + } catch (JsonSyntaxException e) { + throw new CipherOperationException("Decryption failed! Ciphertext not in JSON format.", e); + } + } +} diff --git a/cdap-security/src/test/java/io/cdap/cdap/security/encryption/FakeAeadCipherTest.java b/cdap-security/src/test/java/io/cdap/cdap/security/encryption/FakeAeadCipherTest.java new file mode 100644 index 000000000000..23e4359cf3ea --- /dev/null +++ b/cdap-security/src/test/java/io/cdap/cdap/security/encryption/FakeAeadCipherTest.java @@ -0,0 +1,78 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.security.encryption; + +import com.google.gson.Gson; +import io.cdap.cdap.security.spi.encryption.CipherOperationException; +import java.nio.charset.StandardCharsets; +import java.security.SecureRandom; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Tests for {@link FakeAeadCipher}. + */ +public class FakeAeadCipherTest { + + private static final Gson GSON = new Gson(); + private static FakeAeadCipher fakeAeadCipherService; + + @BeforeClass + public static void setup() throws Exception { + fakeAeadCipherService = new FakeAeadCipher(); + fakeAeadCipherService.initialize(); + } + + @Test + public void testEncryptDecrypt() throws Exception { + byte[] originalData = "some-data-is-here-20o397428976209873012973378965tousdhjbo9o" + .getBytes(StandardCharsets.UTF_8); + byte[] validAssociatedData = "valid-associated-data".getBytes(StandardCharsets.UTF_8); + byte[] encryptedData = fakeAeadCipherService.encrypt(originalData, validAssociatedData); + byte[] decryptedData = fakeAeadCipherService.decrypt(encryptedData, validAssociatedData); + Assert.assertArrayEquals(originalData, decryptedData); + } + + @Test(expected = CipherOperationException.class) + public void testInvalidDecrypt() throws Exception { + byte[] iv = new byte[16]; + byte[] ciphertext = "invalid-ciphertext".getBytes(StandardCharsets.UTF_8); + SecureRandom secureRandom = new SecureRandom(); + secureRandom.nextBytes(iv); + fakeAeadCipherService + .decrypt(String.format("{\"iv\":%s,\"ciphertext\":%s}", + GSON.toJson(iv), GSON.toJson(ciphertext)).getBytes(StandardCharsets.UTF_8), + "invalid-associated-data".getBytes(StandardCharsets.UTF_8)); + } + + @Test(expected = CipherOperationException.class) + public void testInvalidCiphertextFormat() throws Exception { + fakeAeadCipherService.decrypt("invalid-ciphertext".getBytes(StandardCharsets.UTF_8), + "invalid-associated-data".getBytes(StandardCharsets.UTF_8)); + } + + @Test(expected = CipherOperationException.class) + public void testMismatchedAssociatedData() throws Exception { + byte[] originalData = "2038979q8ahvwe09w37893o2uihieryhg890w8u09pfuijsdilhg8907u230w8u" + .getBytes(StandardCharsets.UTF_8); + byte[] validAssociatedData = "valid-associated-data".getBytes(StandardCharsets.UTF_8); + byte[] encryptedData = fakeAeadCipherService.encrypt(originalData, validAssociatedData); + fakeAeadCipherService + .decrypt(encryptedData, "invalid-associated-data".getBytes(StandardCharsets.UTF_8)); + } +} diff --git a/cdap-security/src/test/java/io/cdap/cdap/security/encryption/UserCredentialAeadEncryptionModuleTest.java b/cdap-security/src/test/java/io/cdap/cdap/security/encryption/UserCredentialAeadEncryptionModuleTest.java new file mode 100644 index 000000000000..0f9b529cd395 --- /dev/null +++ b/cdap-security/src/test/java/io/cdap/cdap/security/encryption/UserCredentialAeadEncryptionModuleTest.java @@ -0,0 +1,39 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.security.encryption; + +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.Key; +import com.google.inject.name.Names; +import io.cdap.cdap.common.guice.ConfigModule; +import io.cdap.cdap.security.encryption.guice.UserCredentialAeadEncryptionModule; +import org.junit.Test; + +/** + * Tests for {@link UserCredentialAeadEncryptionModule}. + */ +public class UserCredentialAeadEncryptionModuleTest { + + @Test + public void testBinding() { + Injector injector = Guice + .createInjector(new ConfigModule(), new UserCredentialAeadEncryptionModule()); + injector.getInstance(Key.get(AeadCipher.class, + Names.named(UserCredentialAeadEncryptionModule.USER_CREDENTIAL_ENCRYPTION))); + } +} diff --git a/cdap-security/src/test/java/io/cdap/cdap/security/encryption/guice/LazyDelegateAeadCipherTest.java b/cdap-security/src/test/java/io/cdap/cdap/security/encryption/guice/LazyDelegateAeadCipherTest.java new file mode 100644 index 000000000000..4b10db171a63 --- /dev/null +++ b/cdap-security/src/test/java/io/cdap/cdap/security/encryption/guice/LazyDelegateAeadCipherTest.java @@ -0,0 +1,108 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.security.encryption.guice; + +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.argThat; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + +import io.cdap.cdap.security.spi.encryption.AeadCipherCryptor; +import io.cdap.cdap.security.spi.encryption.AeadCipherContext; +import io.cdap.cdap.security.spi.encryption.CipherInitializationException; +import java.util.HashMap; +import java.util.Map; +import org.junit.Test; +import org.mockito.ArgumentMatcher; + +/** + * Tests for {@link LazyDelegateAeadCipher}. + */ +public class LazyDelegateAeadCipherTest { + + @Test + public void testLazyInitializeOnEncryption() throws Exception { + Map properties = new HashMap<>(); + Map secureProperties = new HashMap<>(); + AeadCipherCryptor mockAeadCipherCryptor = mock(AeadCipherCryptor.class); + LazyDelegateAeadCipher delegate = new LazyDelegateAeadCipher(mockAeadCipherCryptor, properties, + secureProperties); + delegate.encrypt("some-data".getBytes(), "some-associated-data".getBytes()); + verify(mockAeadCipherCryptor) + .initialize(argThat(new AeadCipherContextMatcher(properties, secureProperties))); + } + + @Test + public void testLazyInitializeOnDecryption() throws Exception { + Map properties = new HashMap<>(); + Map secureProperties = new HashMap<>(); + AeadCipherCryptor mockAeadCipherCryptor = mock(AeadCipherCryptor.class); + LazyDelegateAeadCipher delegate = new LazyDelegateAeadCipher(mockAeadCipherCryptor, properties, + secureProperties); + delegate.decrypt("some-data".getBytes(), "some-associated-data".getBytes()); + verify(mockAeadCipherCryptor) + .initialize(argThat(new AeadCipherContextMatcher(properties, secureProperties))); + } + + @Test(expected = CipherInitializationException.class) + public void testLazyInitializeOnEncryptionException() throws Exception { + Map properties = new HashMap<>(); + Map secureProperties = new HashMap<>(); + AeadCipherCryptor mockAeadCipherCryptor = mock(AeadCipherCryptor.class); + doThrow(new CipherInitializationException("some initialization error")).when( + mockAeadCipherCryptor) + .initialize(any()); + LazyDelegateAeadCipher delegate = new LazyDelegateAeadCipher(mockAeadCipherCryptor, properties, + secureProperties); + delegate.encrypt("some-data".getBytes(), "some-associated-data".getBytes()); + } + + @Test(expected = CipherInitializationException.class) + public void testLazyInitializeOnDecryptionException() throws Exception { + Map properties = new HashMap<>(); + Map secureProperties = new HashMap<>(); + AeadCipherCryptor mockAeadCipherCryptor = mock(AeadCipherCryptor.class); + doThrow(new CipherInitializationException("some initialization error")).when( + mockAeadCipherCryptor) + .initialize(any()); + LazyDelegateAeadCipher delegate = new LazyDelegateAeadCipher(mockAeadCipherCryptor, properties, + secureProperties); + delegate.decrypt("some-data".getBytes(), "some-associated-data".getBytes()); + } + + private class AeadCipherContextMatcher extends ArgumentMatcher { + + Map properties; + Map secureProperties; + + AeadCipherContextMatcher(Map properties, Map secureProperties) { + this.properties = properties; + this.secureProperties = secureProperties; + } + + @Override + public boolean matches(Object o) { + if (!(o instanceof AeadCipherContext)) { + return false; + } + AeadCipherContext aeadCipherContext = (AeadCipherContext) o; + return properties.equals(aeadCipherContext.getProperties()) && secureProperties + .equals(aeadCipherContext.getSecureProperties()); + } + } +} diff --git a/pom.xml b/pom.xml index 9b40e8be7382..5fd3f5239be4 100644 --- a/pom.xml +++ b/pom.xml @@ -171,7 +171,6 @@ 1.7.0 0.15.0-incubating 0.8.4 - 1.6.0 0.9.3 1.3.1 2.3.6 @@ -1598,17 +1597,6 @@ jackson-module-scala_2.12 ${jackson.version} - - com.google.crypto.tink - tink - ${tink.version} - - - com.google.code.gson - gson - - - commons-collections commons-collections @@ -2051,8 +2039,8 @@ all opt etc --config-files etc/cdap --config-files etc/logrotate.d --config-files etc/security - org.apache.hadoop,org.apache.hbase,com.google.protobuf,\ - org.datanucleus,asm,org.apache.spark,slf4j-log4j12,commons-logging,com.google.crypto.tink + org.apache.hadoop,org.apache.hbase,\ + org.datanucleus,asm,org.apache.spark,slf4j-log4j12,commons-logging @@ -2757,7 +2745,7 @@ cdap-runtime-ext-remote-hadoop cdap-runtime-spi cdap-securestore-ext-cloudkms - cdap-securestore-ext-gcp-secretstore + cdap-securestore-ext-gcp-secretstore cdap-securestore-spi cdap-support-bundle cdap-operational-stats-core @@ -2773,6 +2761,7 @@ cdap-event-reader-spi cdap-authenticator-ext-gcp cdap-source-control + cdap-encryption-ext-tink