Skip to content

Commit 14f3579

Browse files
mcakircalimcakircali
committed
feat(S3): added S3ObjectPath
1 parent 3adb373 commit 14f3579

12 files changed

+185
-89
lines changed

‎src/eckit/CMakeLists.txt

+2
Original file line numberDiff line numberDiff line change
@@ -305,6 +305,8 @@ if( eckit_HAVE_AWS_S3 )
305305
io/s3/S3Name.h
306306
io/s3/S3ObjectName.cc
307307
io/s3/S3ObjectName.h
308+
io/s3/S3ObjectPath.cc
309+
io/s3/S3ObjectPath.h
308310
io/s3/S3Session.cc
309311
io/s3/S3Session.h
310312
io/s3/S3URIManager.cc

‎src/eckit/io/s3/S3BucketName.cc

+10-5
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
#include "eckit/io/s3/S3Exception.h"
2222
#include "eckit/io/s3/S3Name.h"
2323
#include "eckit/io/s3/S3ObjectName.h"
24+
#include "eckit/io/s3/S3ObjectPath.h"
2425
#include "eckit/log/Log.h"
2526
#include "eckit/net/Endpoint.h"
2627

@@ -34,15 +35,19 @@ namespace eckit {
3435

3536
//----------------------------------------------------------------------------------------------------------------------
3637

37-
S3BucketName::S3BucketName(const URI& uri) : S3Name(uri) {
38-
const auto pairs = parse(uri.name());
39-
if (pairs.empty()) { throw S3SeriousBug("Could not parse bucket name!", Here()); }
40-
bucket_ = pairs[0];
38+
auto S3BucketName::parse(const std::string& name) -> std::string {
39+
const auto parsed = S3Name::parse(name);
40+
if (parsed.size() != 1) { throw S3SeriousBug("Could not parse bucket from name: " + name, Here()); }
41+
return {parsed[0]};
4142
}
4243

44+
//----------------------------------------------------------------------------------------------------------------------
45+
4346
S3BucketName::S3BucketName(const net::Endpoint& endpoint, std::string bucket)
4447
: S3Name(endpoint), bucket_ {std::move(bucket)} { }
4548

49+
S3BucketName::S3BucketName(const URI& uri) : S3Name(uri), bucket_ {parse(uri.name())} { }
50+
4651
//----------------------------------------------------------------------------------------------------------------------
4752

4853
auto S3BucketName::uri() const -> URI {
@@ -63,7 +68,7 @@ void S3BucketName::print(std::ostream& out) const {
6368
//----------------------------------------------------------------------------------------------------------------------
6469

6570
auto S3BucketName::makeObject(const std::string& object) const -> std::unique_ptr<S3ObjectName> {
66-
return std::make_unique<S3ObjectName>(endpoint(), bucket_, object);
71+
return std::make_unique<S3ObjectName>(endpoint(), S3ObjectPath {bucket_, object});
6772
}
6873

6974
auto S3BucketName::exists() const -> bool {

‎src/eckit/io/s3/S3BucketName.h

+3
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,9 @@ class S3ObjectName;
3535
//----------------------------------------------------------------------------------------------------------------------
3636

3737
class S3BucketName : public S3Name {
38+
public: // methods
39+
static auto parse(const std::string& name) -> std::string;
40+
3841
public: // methods
3942
explicit S3BucketName(const URI& uri);
4043

‎src/eckit/io/s3/S3Client.h

+10-12
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
#pragma once
2121

2222
#include "eckit/io/s3/S3Config.h"
23+
#include "eckit/io/s3/S3ObjectPath.h"
2324

2425
#include <cstdint>
2526
#include <iosfwd>
@@ -47,6 +48,8 @@ class S3Client {
4748

4849
auto config() const -> const S3Config& { return config_; }
4950

51+
// bucket operations
52+
5053
virtual void createBucket(const std::string& bucket) const = 0;
5154

5255
virtual void emptyBucket(const std::string& bucket) const = 0;
@@ -57,26 +60,21 @@ class S3Client {
5760

5861
virtual auto listBuckets() const -> std::vector<std::string> = 0;
5962

60-
virtual auto putObject(const std::string& bucket,
61-
const std::string& object,
62-
const void* buffer,
63-
uint64_t length) const -> long long = 0;
63+
// object operations
64+
65+
virtual auto putObject(const S3ObjectPath& path, const void* buffer, uint64_t length) const -> long long = 0;
6466

65-
virtual auto getObject(const std::string& bucket,
66-
const std::string& object,
67-
void* buffer,
68-
uint64_t offset,
69-
uint64_t length) const -> long long = 0;
67+
virtual auto getObject(const S3ObjectPath& path, void* buffer, uint64_t offset, uint64_t length) const -> long long = 0;
7068

71-
virtual void deleteObject(const std::string& bucket, const std::string& object) const = 0;
69+
virtual void deleteObject(const S3ObjectPath& path) const = 0;
7270

7371
virtual void deleteObjects(const std::string& bucket, const std::vector<std::string>& objects) const = 0;
7472

7573
virtual auto listObjects(const std::string& bucket) const -> std::vector<std::string> = 0;
7674

77-
virtual auto objectExists(const std::string& bucket, const std::string& object) const -> bool = 0;
75+
virtual auto objectExists(const S3ObjectPath& path) const -> bool = 0;
7876

79-
virtual auto objectSize(const std::string& bucket, const std::string& object) const -> long long = 0;
77+
virtual auto objectSize(const S3ObjectPath& path) const -> long long = 0;
8078

8179
protected: // methods
8280
S3Client();

‎src/eckit/io/s3/S3Name.cc

+14-3
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,10 @@
1717

1818
#include "eckit/exception/Exceptions.h"
1919
#include "eckit/filesystem/URI.h"
20+
#include "eckit/io/s3/S3BucketName.h"
2021
#include "eckit/io/s3/S3Client.h"
22+
#include "eckit/io/s3/S3ObjectName.h"
23+
#include "eckit/io/s3/S3ObjectPath.h"
2124
#include "eckit/io/s3/S3Session.h"
2225
#include "eckit/utils/Tokenizer.h"
2326

@@ -34,11 +37,20 @@ auto S3Name::parse(const std::string& name) -> std::vector<std::string> {
3437
return Tokenizer("/").tokenize(name);
3538
}
3639

40+
auto S3Name::make(const net::Endpoint& endpoint, const std::string& name) -> std::unique_ptr<S3Name> {
41+
const auto names = parse(name);
42+
switch (names.size()) {
43+
case 1: return std::make_unique<S3BucketName>(endpoint, names[0]);
44+
case 2: return std::make_unique<S3ObjectName>(endpoint, S3ObjectPath {names[0], names[1]});
45+
default: throw SeriousBug("Could not parse S3 name: " + name, Here());
46+
}
47+
}
48+
3749
//----------------------------------------------------------------------------------------------------------------------
3850

3951
S3Name::S3Name(const net::Endpoint& endpoint) : endpoint_ {endpoint} { }
4052

41-
S3Name::S3Name(const URI& uri) : S3Name( {uri.host(), uri.port()}) {
53+
S3Name::S3Name(const URI& uri) : endpoint_ {uri.host(), uri.port()} {
4254
/// @todo is "s3://endpoint/bucket/object" a valid URI ?
4355
ASSERT(uri.scheme() == type);
4456
}
@@ -62,8 +74,7 @@ void S3Name::print(std::ostream& out) const {
6274
//----------------------------------------------------------------------------------------------------------------------
6375

6476
auto S3Name::client() const -> S3Client& {
65-
if (!client_) { client_ = S3Session::instance().getClient(endpoint_); }
66-
return *client_;
77+
return *S3Session::instance().getClient(endpoint_);
6778
}
6879

6980
//----------------------------------------------------------------------------------------------------------------------

‎src/eckit/io/s3/S3Name.h

+5-3
Original file line numberDiff line numberDiff line change
@@ -34,11 +34,13 @@ class S3Client;
3434
//----------------------------------------------------------------------------------------------------------------------
3535

3636
class S3Name {
37-
public: // types
37+
public: // statics
3838
static constexpr auto type = "s3";
3939

4040
static auto parse(const std::string& name) -> std::vector<std::string>;
4141

42+
static auto make(const net::Endpoint& endpoint, const std::string& name) -> std::unique_ptr<S3Name>;
43+
4244
public: // methods
4345
explicit S3Name(const net::Endpoint& endpoint);
4446

@@ -56,6 +58,8 @@ class S3Name {
5658

5759
// accessors
5860

61+
void endpoint(const net::Endpoint& endpoint) { endpoint_ = endpoint; }
62+
5963
auto endpoint() const -> const net::Endpoint& { return endpoint_; }
6064

6165
virtual auto uri() const -> URI;
@@ -73,8 +77,6 @@ class S3Name {
7377

7478
private: // members
7579
net::Endpoint endpoint_;
76-
77-
mutable std::shared_ptr<S3Client> client_;
7880
};
7981

8082
//----------------------------------------------------------------------------------------------------------------------

‎src/eckit/io/s3/S3ObjectName.cc

+20-16
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
#include "eckit/io/s3/S3Exception.h"
2121
#include "eckit/io/s3/S3Handle.h"
2222
#include "eckit/io/s3/S3Name.h"
23+
#include "eckit/io/s3/S3ObjectPath.h"
2324
#include "eckit/log/CodeLocation.h"
2425
#include "eckit/net/Endpoint.h"
2526

@@ -31,57 +32,60 @@ namespace eckit {
3132

3233
//----------------------------------------------------------------------------------------------------------------------
3334

34-
S3ObjectName::S3ObjectName(const URI& uri) : S3Name(uri) {
35-
const auto pairs = parse(uri.name());
36-
if (pairs.size() != 2) { throw S3SeriousBug("Could not parse bucket and object names!", Here()); }
37-
bucket_ = pairs[0];
38-
object_ = pairs[1];
35+
auto S3ObjectName::parse(const std::string& name) -> S3ObjectPath {
36+
const auto parsed = S3Name::parse(name);
37+
if (parsed.size() != 2) { throw S3SeriousBug("Could not parse bucket/object from name: " + name, Here()); }
38+
return {parsed[0], parsed[1]};
3939
}
4040

41-
S3ObjectName::S3ObjectName(const net::Endpoint& endpoint, std::string bucket, std::string object)
42-
: S3Name(endpoint), bucket_ {std::move(bucket)}, object_ {std::move(object)} { }
41+
//----------------------------------------------------------------------------------------------------------------------
42+
43+
S3ObjectName::S3ObjectName(const net::Endpoint& endpoint, S3ObjectPath path)
44+
: S3Name(endpoint), path_ {std::move(path)} { }
45+
46+
S3ObjectName::S3ObjectName(const URI& uri) : S3Name(uri), path_ {parse(uri.name())} { }
4347

4448
//----------------------------------------------------------------------------------------------------------------------
4549

4650
void S3ObjectName::print(std::ostream& out) const {
47-
out << "S3ObjectName[object=" << object_ << ",bucket=" << bucket_;
51+
out << "S3ObjectName[path=" << path_;
4852
S3Name::print(out);
4953
}
5054

5155
//----------------------------------------------------------------------------------------------------------------------
5256

5357
auto S3ObjectName::uri() const -> URI {
5458
auto uri = S3Name::uri();
55-
uri.path("/" + bucket_ + "/" + object_);
59+
uri.path(path_);
5660
return uri;
5761
}
5862

5963
auto S3ObjectName::asString() const -> std::string {
60-
return S3Name::asString() + "/" + bucket_ + "/" + object_;
64+
return S3Name::asString() + '/' + path_.asString();
6165
}
6266

6367
auto S3ObjectName::size() const -> long long {
64-
return client().objectSize(bucket_, object_);
68+
return client().objectSize(path_);
6569
}
6670

6771
auto S3ObjectName::exists() const -> bool {
68-
return client().objectExists(bucket_, object_);
72+
return client().objectExists(path_);
6973
}
7074

7175
auto S3ObjectName::bucketExists() const -> bool {
72-
return client().bucketExists(bucket_);
76+
return client().bucketExists(path_.bucket);
7377
}
7478

7579
void S3ObjectName::remove() {
76-
client().deleteObject(bucket_, object_);
80+
client().deleteObject(path_);
7781
}
7882

7983
auto S3ObjectName::put(const void* buffer, const long length) const -> long long {
80-
return client().putObject(bucket_, object_, buffer, length);
84+
return client().putObject(path_, buffer, length);
8185
}
8286

8387
auto S3ObjectName::get(void* buffer, const long offset, const long length) const -> long long {
84-
return client().getObject(bucket_, object_, buffer, offset, length);
88+
return client().getObject(path_, buffer, offset, length);
8589
}
8690

8791
auto S3ObjectName::dataHandle() -> DataHandle* {

‎src/eckit/io/s3/S3ObjectName.h

+11-7
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
#pragma once
2121

2222
#include "eckit/io/s3/S3Name.h"
23-
#include "eckit/net/Endpoint.h"
23+
#include "eckit/io/s3/S3ObjectPath.h"
2424

2525
#include <ostream>
2626
#include <string>
@@ -33,10 +33,13 @@ class DataHandle;
3333
//----------------------------------------------------------------------------------------------------------------------
3434

3535
class S3ObjectName : public S3Name {
36+
public: // helpers
37+
static auto parse(const std::string& name) -> S3ObjectPath;
38+
3639
public: // methods
37-
explicit S3ObjectName(const URI& uri);
40+
S3ObjectName(const net::Endpoint& endpoint, S3ObjectPath path);
3841

39-
S3ObjectName(const net::Endpoint& endpoint, std::string bucket, std::string object);
42+
explicit S3ObjectName(const URI& uri);
4043

4144
auto uri() const -> URI override;
4245

@@ -58,18 +61,19 @@ class S3ObjectName : public S3Name {
5861

5962
auto asString() const -> std::string override;
6063

61-
auto name() const -> const std::string& { return object_; }
64+
auto path() const -> const S3ObjectPath& { return path_; }
65+
66+
auto name() const -> const std::string& { return path_.object; }
6267

63-
auto bucket() const -> const std::string& { return bucket_; }
68+
auto bucket() const -> const std::string& { return path_.bucket; }
6469

6570
auto bucketExists() const -> bool;
6671

6772
private: // methods
6873
void print(std::ostream& out) const override;
6974

7075
private: // members
71-
std::string bucket_;
72-
std::string object_;
76+
S3ObjectPath path_;
7377
};
7478

7579
//----------------------------------------------------------------------------------------------------------------------

‎src/eckit/io/s3/S3ObjectPath.cc

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
/*
2+
* (C) Copyright 1996- ECMWF.
3+
*
4+
* This software is licensed under the terms of the Apache Licence Version 2.0
5+
* which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
6+
* In applying this licence, ECMWF does not waive the privileges and immunities
7+
* granted to it by virtue of its status as an intergovernmental organisation nor
8+
* does it submit to any jurisdiction.
9+
*/
10+
11+
/*
12+
* This software was developed as part of the EC H2020 funded project IO-SEA
13+
* (Project ID: 955811) iosea-project.eu
14+
*/
15+
16+
#include "eckit/io/s3/S3ObjectPath.h"
17+
18+
#include <ostream>
19+
20+
namespace eckit {
21+
22+
//----------------------------------------------------------------------------------------------------------------------
23+
24+
std::ostream& operator<<(std::ostream& out, const S3ObjectPath& path) {
25+
out << "object=" << path.object << ", bucket=" << path.bucket;
26+
return out;
27+
}
28+
29+
//----------------------------------------------------------------------------------------------------------------------
30+
31+
} // namespace eckit
32+

‎src/eckit/io/s3/S3ObjectPath.h

+40
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
/*
2+
* (C) Copyright 1996- ECMWF.
3+
*
4+
* This software is licensed under the terms of the Apache Licence Version 2.0
5+
* which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
6+
* In applying this licence, ECMWF does not waive the privileges and immunities
7+
* granted to it by virtue of its status as an intergovernmental organisation nor
8+
* does it submit to any jurisdiction.
9+
*/
10+
11+
/*
12+
* This software was developed as part of the EC H2020 funded project IO-SEA
13+
* (Project ID: 955811) iosea-project.eu
14+
*/
15+
16+
/// @file S3ObjectPath.h
17+
/// @author Metin Cakircali
18+
/// @date Dec 2024
19+
20+
#pragma once
21+
22+
#include <ostream>
23+
#include <string>
24+
25+
namespace eckit {
26+
27+
struct S3ObjectPath {
28+
std::string bucket;
29+
std::string object;
30+
31+
auto asString() const -> std::string { return bucket + '/' + object; }
32+
33+
operator std::string() const { return asString(); }
34+
35+
friend std::ostream& operator<<(std::ostream& out, const S3ObjectPath& path);
36+
};
37+
38+
//----------------------------------------------------------------------------------------------------------------------
39+
40+
} // namespace eckit

0 commit comments

Comments
 (0)