Skip to content

Commit 05948bb

Browse files
committed
Replace s3.exclusive-create config with Delta Lake's one
`s3.exclusive-create` config is informative and only consumed by Delta Lake connector. Replace it with a new connector's specific config (`delta.s3.transaction-log-conditional-writes.enabled`). This follows earlier change, commit d3a7fd5, where the S3 config was made informative only.
1 parent 41c18d1 commit 05948bb

File tree

17 files changed

+67
-57
lines changed

17 files changed

+67
-57
lines changed

docs/src/main/sphinx/connector/delta-lake.md

Lines changed: 16 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -604,19 +604,22 @@ Write operations are supported for tables stored on the following systems:
604604
- S3 and S3-compatible storage
605605

606606
Writes to Amazon S3 and S3-compatible storage are controlled by following
607-
configuration properties. When `s3.exclusive-create` is set to `true` (default),
608-
the connector uses S3 conditional writes to detect log write collisions.
609-
This is compatible with any other engines that also use conditional writes.
610-
611-
When `s3.exclusive-create` is false, then writes to Amazon S3 and S3-compatible
612-
storage must be enabled with the `delta.enable-non-concurrent-writes` property.
613-
In this mode, the connector leverages S3 strong consistency guarantees combined
614-
with Trino specific naming strategy to orchestrate creation of new log files.
615-
In this mode, writes to S3 can safely be made from multiple Trino clusters
616-
using same writing mode; however, write collisions are not detected when
617-
writing concurrently from other Delta Lake engines, or from Trino clusters
618-
using S3 conditional writes. You must make sure that no concurrent data
619-
modifications are run to avoid data corruption.
607+
configuration properties. When
608+
`delta.s3.transaction-log-conditional-writes.enabled` is set to `true`
609+
(default), the connector uses S3 conditional writes to detect log write
610+
collisions. This is compatible with any other engines that also use
611+
conditional writes.
612+
613+
When `delta.s3.transaction-log-conditional-writes.enabled` is false, then
614+
writes to Amazon S3 and S3-compatible storage must be enabled with the
615+
`delta.enable-non-concurrent-writes` property. In this mode, the connector
616+
leverages S3 strong consistency guarantees combined with Trino specific naming
617+
strategy to orchestrate creation of new log files. In this mode, writes to S3
618+
can safely be made from multiple Trino clusters using same writing mode;
619+
however, write collisions are not detected when writing concurrently from other
620+
Delta Lake engines, or from Trino clusters using S3 conditional writes. You
621+
must make sure that no concurrent data modifications are run to avoid data
622+
corruption.
620623

621624

622625
(delta-lake-schema-table-management)=

docs/src/main/sphinx/object-storage/file-system-s3.md

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,6 @@ support:
4040
- AWS signing protocol to use for authenticating S3 requests. Supported values are:
4141
`AwsS3V4Signer`, `Aws4Signer`, `AsyncAws4Signer`, `Aws4UnsignedPayloadSigner`,
4242
`EventStreamAws4Signer`.
43-
* - `s3.exclusive-create`
44-
- Whether conditional write is supported by the S3-compatible storage. Defaults to `true`.
4543
* - `s3.canned-acl`
4644
- [Canned ACL](https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl)
4745
to use when uploading files to S3. Defaults to `NONE`, which has the same

lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3Context.java

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -39,8 +39,7 @@ record S3Context(
3939
S3SseContext s3SseContext,
4040
Optional<AwsCredentialsProvider> credentialsProviderOverride,
4141
StorageClassType storageClass,
42-
ObjectCannedAcl cannedAcl,
43-
boolean exclusiveWriteSupported)
42+
ObjectCannedAcl cannedAcl)
4443
{
4544
private static final int MIN_PART_SIZE = 5 * 1024 * 1024; // S3 requirement
4645

@@ -58,7 +57,7 @@ public RequestPayer requestPayer()
5857

5958
public S3Context withKmsKeyId(String kmsKeyId)
6059
{
61-
return new S3Context(partSize, requesterPays, S3SseContext.withKmsKeyId(kmsKeyId), credentialsProviderOverride, storageClass, cannedAcl, exclusiveWriteSupported);
60+
return new S3Context(partSize, requesterPays, S3SseContext.withKmsKeyId(kmsKeyId), credentialsProviderOverride, storageClass, cannedAcl);
6261
}
6362

6463
public S3Context withCredentials(ConnectorIdentity identity)
@@ -75,7 +74,7 @@ public S3Context withCredentials(ConnectorIdentity identity)
7574

7675
public S3Context withSseCustomerKey(String key)
7776
{
78-
return new S3Context(partSize, requesterPays, S3SseContext.withSseCustomerKey(key), credentialsProviderOverride, storageClass, cannedAcl, exclusiveWriteSupported);
77+
return new S3Context(partSize, requesterPays, S3SseContext.withSseCustomerKey(key), credentialsProviderOverride, storageClass, cannedAcl);
7978
}
8079

8180
public S3Context withCredentialsProviderOverride(AwsCredentialsProvider credentialsProviderOverride)
@@ -86,8 +85,7 @@ public S3Context withCredentialsProviderOverride(AwsCredentialsProvider credenti
8685
s3SseContext,
8786
Optional.of(credentialsProviderOverride),
8887
storageClass,
89-
cannedAcl,
90-
exclusiveWriteSupported);
88+
cannedAcl);
9189
}
9290

9391
public void applyCredentialProviderOverride(AwsRequestOverrideConfiguration.Builder builder)

lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3FileSystemConfig.java

Lines changed: 2 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import io.airlift.configuration.Config;
2020
import io.airlift.configuration.ConfigDescription;
2121
import io.airlift.configuration.ConfigSecuritySensitive;
22+
import io.airlift.configuration.DefunctConfig;
2223
import io.airlift.configuration.LegacyConfig;
2324
import io.airlift.units.DataSize;
2425
import io.airlift.units.Duration;
@@ -42,6 +43,7 @@
4243
import static software.amazon.awssdk.awscore.retry.AwsRetryStrategy.legacyRetryStrategy;
4344
import static software.amazon.awssdk.awscore.retry.AwsRetryStrategy.standardRetryStrategy;
4445

46+
@DefunctConfig("s3.exclusive-create")
4547
public class S3FileSystemConfig
4648
{
4749
public enum S3SseType
@@ -175,7 +177,6 @@ public static RetryStrategy getRetryStrategy(RetryMode retryMode)
175177
private ObjectCannedAcl objectCannedAcl = ObjectCannedAcl.NONE;
176178
private RetryMode retryMode = RetryMode.LEGACY;
177179
private int maxErrorRetries = 20;
178-
private boolean supportsExclusiveCreate = true;
179180
private boolean crossRegionAccessEnabled;
180181
private String applicationId = "Trino";
181182

@@ -612,19 +613,6 @@ public S3FileSystemConfig setNonProxyHosts(String nonProxyHosts)
612613
return this;
613614
}
614615

615-
public boolean isSupportsExclusiveCreate()
616-
{
617-
return supportsExclusiveCreate;
618-
}
619-
620-
@Config("s3.exclusive-create")
621-
@ConfigDescription("Whether S3-compatible storage supports exclusive create (true for Minio and AWS S3)")
622-
public S3FileSystemConfig setSupportsExclusiveCreate(boolean supportsExclusiveCreate)
623-
{
624-
this.supportsExclusiveCreate = supportsExclusiveCreate;
625-
return this;
626-
}
627-
628616
public boolean isCrossRegionAccessEnabled()
629617
{
630618
return crossRegionAccessEnabled;

lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3FileSystemLoader.java

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -100,8 +100,7 @@ private S3FileSystemLoader(Optional<S3SecurityMappingProvider> mappingProvider,
100100
config.getSseCustomerKey()),
101101
Optional.empty(),
102102
config.getStorageClass(),
103-
config.getCannedAcl(),
104-
config.isSupportsExclusiveCreate());
103+
config.getCannedAcl());
105104
}
106105

107106
@Override

lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3OutputFile.java

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -59,10 +59,6 @@ public void createOrOverwrite(byte[] data)
5959
public void createExclusive(byte[] data)
6060
throws IOException
6161
{
62-
if (!context.exclusiveWriteSupported()) {
63-
throw new UnsupportedOperationException("createExclusive not supported by " + getClass());
64-
}
65-
6662
try (OutputStream out = create(newSimpleAggregatedMemoryContext(), true)) {
6763
out.write(data);
6864
}

lib/trino-filesystem-s3/src/test/java/io/trino/filesystem/s3/TestS3FileSystemAwsS3.java

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,6 @@ protected S3FileSystemFactory createS3FileSystemFactory()
9393
.setAwsSecretKey(secretKey)
9494
.setRegion(region)
9595
.setEndpoint(endpoint)
96-
.setSupportsExclusiveCreate(true)
9796
.setSignerType(S3FileSystemConfig.SignerType.AwsS3V4Signer)
9897
.setStreamingPartSize(streamingPartSize),
9998
new S3FileSystemStats());

lib/trino-filesystem-s3/src/test/java/io/trino/filesystem/s3/TestS3FileSystemConfig.java

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,6 @@ public void testDefaults()
7575
.setHttpProxyUsername(null)
7676
.setHttpProxyPassword(null)
7777
.setHttpProxyPreemptiveBasicProxyAuth(false)
78-
.setSupportsExclusiveCreate(true)
7978
.setCrossRegionAccessEnabled(false)
8079
.setApplicationId("Trino"));
8180
}
@@ -117,7 +116,6 @@ public void testExplicitPropertyMappings()
117116
.put("s3.http-proxy.username", "test")
118117
.put("s3.http-proxy.password", "test")
119118
.put("s3.http-proxy.preemptive-basic-auth", "true")
120-
.put("s3.exclusive-create", "false")
121119
.put("s3.application-id", "application id")
122120
.put("s3.cross-region-access", "true")
123121
.buildOrThrow();
@@ -156,7 +154,6 @@ public void testExplicitPropertyMappings()
156154
.setHttpProxyUsername("test")
157155
.setHttpProxyPassword("test")
158156
.setHttpProxyPreemptiveBasicProxyAuth(true)
159-
.setSupportsExclusiveCreate(false)
160157
.setCrossRegionAccessEnabled(true)
161158
.setApplicationId("application id");
162159

lib/trino-filesystem-s3/src/test/java/io/trino/filesystem/s3/TestS3FileSystemMinIo.java

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,6 @@ protected S3FileSystemFactory createS3FileSystemFactory()
8989
.setPathStyleAccess(true)
9090
.setAwsAccessKey(Minio.MINIO_ACCESS_KEY)
9191
.setAwsSecretKey(Minio.MINIO_SECRET_KEY)
92-
.setSupportsExclusiveCreate(true)
9392
.setStreamingPartSize(streamingPartSize),
9493
new S3FileSystemStats());
9594
}

lib/trino-filesystem-s3/src/test/java/io/trino/filesystem/s3/TestS3FileSystemMoto.java

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,6 @@ protected S3FileSystemFactory createS3FileSystemFactory()
6868
.setPathStyleAccess(true)
6969
.setAwsAccessKey(MOTO_ACCESS_KEY)
7070
.setAwsSecretKey(MOTO_SECRET_KEY)
71-
.setSupportsExclusiveCreate(true)
7271
.setStreamingPartSize(streamingPartSize),
7372
new S3FileSystemStats());
7473
}

0 commit comments

Comments
 (0)