Skip to content

Commit 22c6419

Browse files
sbandsband
committed
fix#582: Review comments implemented
1 parent 0fa2031 commit 22c6419

File tree

4 files changed

+119
-55
lines changed

4 files changed

+119
-55
lines changed

‎bundles/sirix-core/src/main/java/org/sirix/access/ResourceConfiguration.java

Lines changed: 85 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -28,11 +28,23 @@
2828

2929
package org.sirix.access;
3030

31-
import com.google.common.base.MoreObjects;
32-
import com.google.gson.stream.JsonReader;
33-
import com.google.gson.stream.JsonWriter;
34-
import net.openhft.hashing.LongHashFunction;
31+
import static com.google.common.base.Preconditions.checkArgument;
32+
import static java.util.Objects.requireNonNull;
33+
34+
import java.io.FileReader;
35+
import java.io.FileWriter;
36+
import java.io.IOException;
37+
import java.lang.reflect.Constructor;
38+
import java.lang.reflect.InvocationTargetException;
39+
import java.nio.file.Files;
40+
import java.nio.file.Path;
41+
import java.nio.file.Paths;
42+
import java.util.ArrayList;
43+
import java.util.List;
44+
import java.util.Objects;
45+
3546
import org.checkerframework.checker.index.qual.NonNegative;
47+
import org.json.JSONObject;
3648
import org.sirix.BinaryEncodingVersion;
3749
import org.sirix.access.trx.node.HashType;
3850
import org.sirix.exception.SirixIOException;
@@ -46,20 +58,11 @@
4658
import org.sirix.settings.VersioningType;
4759
import org.sirix.utils.OS;
4860

49-
import java.io.FileReader;
50-
import java.io.FileWriter;
51-
import java.io.IOException;
52-
import java.lang.reflect.Constructor;
53-
import java.lang.reflect.InvocationTargetException;
54-
import java.nio.file.Files;
55-
import java.nio.file.Path;
56-
import java.nio.file.Paths;
57-
import java.util.ArrayList;
58-
import java.util.List;
59-
import java.util.Objects;
61+
import com.google.common.base.MoreObjects;
62+
import com.google.gson.stream.JsonReader;
63+
import com.google.gson.stream.JsonWriter;
6064

61-
import static com.google.common.base.Preconditions.checkArgument;
62-
import static java.util.Objects.requireNonNull;
65+
import net.openhft.hashing.LongHashFunction;
6366

6467
/**
6568
* Holds the settings for a resource which acts as a base for session that can not change. This
@@ -165,6 +168,36 @@ public static int compareStructure(final Path file) {
165168
}
166169
}
167170

171+
public static final class AWSStorageInformation {
172+
private final String awsProfile;
173+
private final String awsRegion;
174+
private final String bucketName; //this should be same as the database name
175+
private final boolean shouldCreateBucketIfNotExists;
176+
177+
public AWSStorageInformation(String awsProfile, String awsRegion, String bucketName,
178+
boolean shouldCreateBucketIfNotExists) {
179+
this.awsProfile = awsProfile;
180+
this.awsRegion = awsRegion;
181+
this.bucketName = bucketName;
182+
this.shouldCreateBucketIfNotExists = shouldCreateBucketIfNotExists;
183+
}
184+
185+
public String getAwsProfile() {
186+
return awsProfile;
187+
}
188+
189+
public String getAwsRegion() {
190+
return awsRegion;
191+
}
192+
193+
public String getBucketName() {
194+
return bucketName;
195+
}
196+
197+
public boolean shouldCreateBucketIfNotExists() {
198+
return shouldCreateBucketIfNotExists;
199+
}
200+
}
168201
// FIXED STANDARD FIELDS
169202
/**
170203
* Standard storage.
@@ -297,6 +330,14 @@ public static int compareStructure(final Path file) {
297330

298331
// END MEMBERS FOR FIXED FIELDS
299332

333+
/*
334+
* Optional AWS Credentials
335+
* */
336+
/*
337+
* This could be improved in future to make it more sophisticated in terms setting the credentials
338+
* for creating the cloud client connection
339+
* */
340+
public final AWSStorageInformation awsStoreInfo;
300341
/**
301342
* Get a new builder instance.
302343
*
@@ -330,6 +371,7 @@ private ResourceConfiguration(final ResourceConfiguration.Builder builder) {
330371
customCommitTimestamps = builder.customCommitTimestamps;
331372
storeNodeHistory = builder.storeNodeHistory;
332373
binaryVersion = builder.binaryEncodingVersion;
374+
awsStoreInfo = builder.awsStoreInfo;
333375
}
334376

335377
public BinaryEncodingVersion getBinaryEncodingVersion() {
@@ -448,7 +490,7 @@ public boolean storeNodeHistory() {
448490
private static final String[] JSONNAMES =
449491
{ "binaryEncoding", "revisioning", "revisioningClass", "numbersOfRevisiontoRestore", "byteHandlerClasses",
450492
"storageKind", "hashKind", "hashFunction", "compression", "pathSummary", "resourceID", "deweyIDsStored",
451-
"persistenter", "storeDiffs", "customCommitTimestamps", "storeNodeHistory", "storeChildCount" };
493+
"persistenter", "storeDiffs", "customCommitTimestamps", "storeNodeHistory", "storeChildCount", "awsStoreInfo" };
452494

453495
/**
454496
* Serialize the configuration.
@@ -596,7 +638,21 @@ public static ResourceConfiguration deserialize(final Path file) throws SirixIOE
596638
name = jsonReader.nextName();
597639
assert name.equals(JSONNAMES[16]);
598640
final boolean storeChildCount = jsonReader.nextBoolean();
599-
641+
name = jsonReader.nextName();
642+
assert name.equals(JSONNAMES[17]);
643+
/*Begin object to read the nested json properties required aws connection*/
644+
jsonReader.beginObject();
645+
AWSStorageInformation awsStoreInfo=null;
646+
if(jsonReader.hasNext()) {
647+
final String awsProfile=jsonReader.nextString();
648+
final String awsRegion=jsonReader.nextString();
649+
final String bucketName=jsonReader.nextString();
650+
final boolean shouldCreateBucketIfNotExists=jsonReader.nextBoolean();
651+
awsStoreInfo = new AWSStorageInformation(awsProfile,
652+
awsRegion, bucketName, shouldCreateBucketIfNotExists);
653+
}
654+
jsonReader.endObject();
655+
/*End object to end reading the nested json properties*/
600656
jsonReader.endObject();
601657
jsonReader.close();
602658
fileReader.close();
@@ -619,7 +675,8 @@ public static ResourceConfiguration deserialize(final Path file) throws SirixIOE
619675
.storeDiffs(storeDiffs)
620676
.storeChildCount(storeChildCount)
621677
.customCommitTimestamps(customCommitTimestamps)
622-
.storeNodeHistory(storeNodeHistory);
678+
.storeNodeHistory(storeNodeHistory)
679+
.awsStoreInfo(awsStoreInfo);
623680

624681
// Deserialized instance.
625682
final ResourceConfiguration config = new ResourceConfiguration(builder);
@@ -713,6 +770,8 @@ public static final class Builder {
713770

714771
private BinaryEncodingVersion binaryEncodingVersion = BINARY_ENCODING_VERSION;
715772

773+
private AWSStorageInformation awsStoreInfo;
774+
716775
/**
717776
* Constructor, setting the mandatory fields.
718777
*
@@ -880,6 +939,12 @@ public Builder binaryEncodingVersion(BinaryEncodingVersion binaryEncodingVersion
880939
return this;
881940
}
882941

942+
/*Since this is an optional config parameter, null check is not needed*/
943+
public Builder awsStoreInfo(final AWSStorageInformation awsStoreInfo) {
944+
this.awsStoreInfo = awsStoreInfo;
945+
return this;
946+
}
947+
883948
@Override
884949
public String toString() {
885950
return MoreObjects.toStringHelper(this)

‎bundles/sirix-core/src/main/java/org/sirix/io/StorageType.java

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import com.github.benmanes.caffeine.cache.Caffeine;
2525
import org.sirix.access.ResourceConfiguration;
2626
import org.sirix.exception.SirixIOException;
27+
import org.sirix.io.cloud.amazon.AmazonS3Storage;
2728
import org.sirix.io.file.FileStorage;
2829
import org.sirix.io.filechannel.FileChannelStorage;
2930
import org.sirix.io.iouring.IOUringStorage;
@@ -113,6 +114,16 @@ public IOStorage getInstance(final ResourceConfiguration resourceConf) {
113114
storage.loadRevisionFileDataIntoMemory(cache);
114115
return storage;
115116
}
117+
},
118+
119+
CLOUD {
120+
@Override
121+
public IOStorage getInstance(final ResourceConfiguration resourceConf) {
122+
final AsyncCache<Integer, RevisionFileData> cache =
123+
getIntegerRevisionFileDataAsyncCache(resourceConf);
124+
final var storage = new AmazonS3Storage(resourceConf, cache);
125+
return storage;
126+
}
116127
};
117128

118129
public static final ConcurrentMap<Path, AsyncCache<Integer, RevisionFileData>> CACHE_REPOSITORY =

‎bundles/sirix-core/src/main/java/org/sirix/io/cloud/amazon/AmazonS3Storage.java

Lines changed: 13 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -54,12 +54,6 @@ public final class AmazonS3Storage implements ICloudStorage {
5454
*/
5555
private final Path file;
5656

57-
/**
58-
* S3 storage bucket name
59-
*
60-
*/
61-
private String bucketName;
62-
6357
private S3Client s3Client;
6458

6559
/** Logger. */
@@ -75,26 +69,24 @@ public final class AmazonS3Storage implements ICloudStorage {
7569
*/
7670
private final AsyncCache<Integer, RevisionFileData> cache;
7771

78-
private String awsProfile;
79-
private String region;
72+
private ResourceConfiguration.AWSStorageInformation awsStorageInfo;
8073

8174
private final AmazonS3StorageReader reader;
8275

76+
8377
/**
8478
* Support AWS authentication only with .aws credentials file with the required
8579
* profile name from the creds file
8680
*/
87-
public AmazonS3Storage(String bucketName, String awsProfile,
88-
String region,
89-
boolean shouldCreateBucketIfNotExists, final ResourceConfiguration resourceConfig,
81+
public AmazonS3Storage(final ResourceConfiguration resourceConfig,
9082
AsyncCache<Integer, RevisionFileData> cache) {
91-
this.bucketName = bucketName;
92-
this.awsProfile = awsProfile;
93-
this.region = region;
83+
this.awsStorageInfo = resourceConfig.awsStoreInfo;
9484
this.cache = cache;
9585
this.byteHandlerPipeline = resourceConfig.byteHandlePipeline;
9686
this.file = resourceConfig.resourcePath;
9787
this.s3Client = getS3Client(); //this client is needed for the below checks, so initialize it here only.
88+
String bucketName = awsStorageInfo.getBucketName();
89+
boolean shouldCreateBucketIfNotExists = awsStorageInfo.shouldCreateBucketIfNotExists();
9890
if(!isBucketExists(bucketName) && shouldCreateBucketIfNotExists) {
9991
createBucket(bucketName);
10092
}
@@ -105,7 +97,8 @@ public AmazonS3Storage(String bucketName, String awsProfile,
10597
new ByteHandlerPipeline(this.byteHandlerPipeline),
10698
SerializationType.DATA,
10799
new PagePersister(),
108-
cache.synchronous());
100+
cache.synchronous(),
101+
resourceConfig);
109102
}
110103

111104
void createBucket(String bucketName) {
@@ -140,31 +133,23 @@ boolean isBucketExists(String bucketName) {
140133

141134
S3Client getS3Client() {
142135
return this.s3Client==null ? S3Client.builder()
143-
.region(Region.of(region))
144-
.credentialsProvider(ProfileCredentialsProvider.create(awsProfile))
136+
.region(Region.of(awsStorageInfo.getAwsRegion()))
137+
.credentialsProvider(ProfileCredentialsProvider.create(awsStorageInfo.getAwsProfile()))
145138
.build() : this.s3Client;
146139
}
147140

148141
S3AsyncClient getAsyncS3Client() {
149142
return S3AsyncClient.builder()
150-
.region(Region.of(region))
151-
.credentialsProvider(ProfileCredentialsProvider.create(awsProfile))
143+
.region(Region.of(awsStorageInfo.getAwsRegion()))
144+
.credentialsProvider(ProfileCredentialsProvider.create(awsStorageInfo.getAwsProfile()))
152145
.build();
153146
}
154147

155148
@Override
156149
public Writer createWriter() {
157-
AmazonS3StorageReader reader = new AmazonS3StorageReader(bucketName,
158-
s3Client,
159-
getDataFilePath().toAbsolutePath().toString(),
160-
getRevisionFilePath().toAbsolutePath().toString(),
161-
new ByteHandlerPipeline(byteHandlerPipeline),
162-
SerializationType.DATA,
163-
new PagePersister(),
164-
cache.synchronous());
165150
return new AmazonS3StorageWriter (getDataFilePath().toAbsolutePath().toString(),
166151
getRevisionFilePath().toAbsolutePath().toString(),
167-
bucketName,
152+
awsStorageInfo.getBucketName(),
168153
SerializationType.DATA,new PagePersister(),
169154
cache,reader,
170155
this.getAsyncS3Client());

‎bundles/sirix-core/src/main/java/org/sirix/io/cloud/amazon/AmazonS3StorageReader.java

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package org.sirix.io.cloud.amazon;
22

33
import java.io.File;
4-
import java.io.FileNotFoundException;
54
import java.io.FileOutputStream;
65
import java.io.IOException;
76
import java.io.OutputStream;
@@ -11,6 +10,7 @@
1110
import java.time.Instant;
1211

1312
import org.checkerframework.checker.nullness.qual.Nullable;
13+
import org.sirix.access.ResourceConfiguration;
1414
import org.sirix.api.PageReadOnlyTrx;
1515
import org.sirix.io.Reader;
1616
import org.sirix.io.RevisionFileData;
@@ -39,10 +39,11 @@ public class AmazonS3StorageReader implements Reader {
3939
* S3 storage bucket name
4040
*
4141
*/
42-
private String bucketName;
42+
private final String bucketName;
4343

44-
private S3Client s3Client;
44+
private final S3Client s3Client;
4545

46+
private final ResourceConfiguration resourceConfig;
4647
/** Logger. */
4748
private static final LogWrapper LOGGER = new LogWrapper(LoggerFactory.getLogger(AmazonS3StorageReader.class));
4849

@@ -56,9 +57,11 @@ public AmazonS3StorageReader(String bucketName,
5657
final ByteHandler byteHandler,
5758
final SerializationType serializationType,
5859
final PagePersister pagePersister,
59-
final Cache<Integer, RevisionFileData> cache) {
60+
final Cache<Integer, RevisionFileData> cache,
61+
ResourceConfiguration resourceConfig) {
6062
this.bucketName = bucketName;
6163
this.s3Client = s3Client;
64+
this.resourceConfig = resourceConfig;
6265
Path dataFilePath = readObjectDataFromS3(dataFileKeyName);
6366
Path revisionOffsetFilePath = readObjectDataFromS3(revisionsOffsetFileKeyName);
6467
try {
@@ -93,13 +96,13 @@ protected Path readObjectDataFromS3(String keyName) {
9396
byte[] data = objectBytes.asByteArray();
9497
/*As the bucketName has to be same as the database name, it makes sense to use/create file on the local filesystem
9598
* instead of in the tmp partition*/
96-
String path = FileSystems.getDefault().getSeparator() + bucketName + FileSystems.getDefault().getSeparator() + keyName;
99+
Path path = resourceConfig.resourcePath;
97100
// Write the data to a local file.
98-
File myFile = new File(path);
101+
File myFile = path.toFile();
99102
try(OutputStream os = new FileOutputStream(myFile)){
100103
os.write(data);
101104
}
102-
return Path.of(path);
105+
return path;
103106
} catch (IOException ex) {
104107
ex.printStackTrace();
105108
} catch (S3Exception e) {

0 commit comments

Comments
 (0)