diff --git a/models/extended/searchBucket.smithy b/models/extended/searchBucket.smithy new file mode 100644 index 00000000..28bb4e88 --- /dev/null +++ b/models/extended/searchBucket.smithy @@ -0,0 +1,58 @@ +$version: "2.0" + +namespace cloudserver.s3extended + +@http(method: "GET", uri: "/{Bucket}") +@readonly +operation SearchBucket { + input := { + @required + @httpLabel + Bucket: String + + @required + @httpQuery("search") + Query: String + + @httpQuery("delimiter") + Delimiter: String + + @httpQuery("encoding-type") + EncodingType: String + + @httpQuery("marker") + Marker: String + + @httpQuery("max-keys") + MaxKeys: Integer + + @httpQuery("prefix") + Prefix: String + + @httpHeader("x-amz-request-payer") + RequestPayer: String + } + output := @xmlName("ListBucketResult") { + IsTruncated: Boolean + + Marker: String + + NextMarker: String + + @xmlFlattened + Contents: ObjectList + + Name: String + + Prefix: String + + Delimiter: String + + MaxKeys: Integer + + @xmlFlattened + CommonPrefixes: CommonPrefixList + + EncodingType: String + } +} diff --git a/models/extended/searchBucketV2.smithy b/models/extended/searchBucketV2.smithy new file mode 100644 index 00000000..e2c21c8e --- /dev/null +++ b/models/extended/searchBucketV2.smithy @@ -0,0 +1,97 @@ +$version: "2.0" + +namespace cloudserver.s3extended + +@http(method: "GET", uri: "/{Bucket}?list-type=2") +@readonly +operation SearchBucketV2 { + input := { + @required + @httpLabel + Bucket: String + + @required + @httpQuery("search") + Query: String + + @httpQuery("delimiter") + Delimiter: String + + @httpQuery("encoding-type") + EncodingType: String + + @httpQuery("max-keys") + MaxKeys: Integer + + @httpQuery("prefix") + Prefix: String + + @httpQuery("continuation-token") + ContinuationToken: String + + @httpQuery("fetch-owner") + FetchOwner: Boolean + + @httpQuery("start-after") + StartAfter: String + + @httpHeader("x-amz-request-payer") + RequestPayer: String + + @httpHeader("x-amz-expected-bucket-owner") + ExpectedBucketOwner: String + } + output := { + IsTruncated: Boolean + + @xmlFlattened + Contents: ObjectList + + Name: String + + Prefix: String + + Delimiter: String + + MaxKeys: Integer + + @xmlFlattened + CommonPrefixes: CommonPrefixList + + EncodingType: String + + KeyCount: Integer + + ContinuationToken: String + + NextContinuationToken: String + + StartAfter: String + } +} + +list ObjectList { + member: S3Object +} + +structure S3Object { + Key: String + LastModified: Timestamp + ETag: String + Size: Integer + StorageClass: String + Owner: Owner +} + +structure Owner { + DisplayName: String + ID: String +} + +list CommonPrefixList { + member: CommonPrefix +} + +structure CommonPrefix { + Prefix: String +} diff --git a/models/extended/searchBucketVersions.smithy b/models/extended/searchBucketVersions.smithy new file mode 100644 index 00000000..2e326803 --- /dev/null +++ b/models/extended/searchBucketVersions.smithy @@ -0,0 +1,108 @@ +$version: "2.0" + +namespace cloudserver.s3extended + +@http(method: "GET", uri: "/{Bucket}?versions=true") +@readonly +operation SearchBucketVersions { + input := { + @required + @httpLabel + Bucket: String + + @required + @httpQuery("search") + Query: String + + @httpQuery("delimiter") + Delimiter: String + + @httpQuery("encoding-type") + EncodingType: String + + @httpQuery("key-marker") + KeyMarker: String + + @httpQuery("max-keys") + MaxKeys: Integer + + @httpQuery("prefix") + Prefix: String + + @httpHeader("x-amz-expected-bucket-owner") + ExpectedBucketOwner: String + + @httpQuery("version-id-marker") + VersionIdMarker: String + } + output := @xmlName("ListVersionsResult") { + IsTruncated: Boolean + + KeyMarker: String + + VersionIdMarker: String + + NextKeyMarker: String + + NextVersionIdMarker: String + + @xmlFlattened + Version: VersionList + + @xmlFlattened + DeleteMarker: DeleteMarkerList + + Name: String + + Prefix: String + + Delimiter: String + + MaxKeys: Integer + + @xmlFlattened + CommonPrefixes: CommonPrefixList + + EncodingType: String + } +} + +list VersionList { + member: Version +} + +structure Version { + Key: String + + @xmlName("IsLatest") + IsLatest: Boolean + + LastModified: Timestamp + + ETag: String + + Size: Integer + + StorageClass: String + + Owner: Owner + + VersionId: String +} + +list DeleteMarkerList { + member: DeleteMarker +} + +structure DeleteMarker { + Key: String + + @xmlName("IsLatest") + IsLatest: Boolean + + LastModified: Timestamp + + Owner: Owner + + VersionId: String +} diff --git a/package.json b/package.json index 04b98d91..d6bc3af3 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,8 @@ "files": [ "dist", "build/smithy/cloudserver/typescript-codegen", - "build/smithy/cloudserverBucketQuota/typescript-codegen" + "build/smithy/cloudserverBucketQuota/typescript-codegen", + "build/smithy/cloudserverS3Extended/typescript-codegen" ], "publishConfig": { "access": "public", @@ -31,8 +32,9 @@ "build:smithy": "smithy build", "build:generated": "cd build/smithy/cloudserver/typescript-codegen && yarn install && yarn build", "build:generated:bucketQuota": "cd build/smithy/cloudserverBucketQuota/typescript-codegen && yarn install && yarn build", + "build:generated:s3extended": "cd build/smithy/cloudserverS3Extended/typescript-codegen && yarn install && yarn build", "build:wrapper": "tsc", - "build": "yarn install && yarn clean:build && yarn build:smithy && yarn build:generated && yarn build:generated:bucketQuota && yarn build:wrapper", + "build": "yarn install && yarn clean:build && yarn build:smithy && yarn build:generated && yarn build:generated:bucketQuota && yarn build:generated:s3extended && yarn build:wrapper", "test": "jest", "test:indexes": "jest tests/testIndexesApis.test.ts", "test:error-handling": "jest tests/testErrorHandling.test.ts", @@ -42,7 +44,8 @@ "test:metadata": "jest tests/testMetadataApis.test.ts", "test:raft": "jest tests/testRaftApis.test.ts", "test:bucketQuotas": "jest tests/testQuotaApis.test.ts", - "test:mongo-backend": "yarn test:indexes && yarn test:error-handling && yarn test:multiple-backend && yarn test:bucketQuotas", + "test:s3Extended": "jest tests/testS3ExtendedApis.test.ts", + "test:mongo-backend": "yarn test:indexes && yarn test:error-handling && yarn test:multiple-backend && yarn test:bucketQuotas && yarn test:s3Extended", "test:metadata-backend": "yarn test:api && yarn test:lifecycle && yarn test:metadata && yarn test:raft", "lint": "eslint src tests", "typecheck": "tsc --noEmit" diff --git a/service/cloudserverS3Extended.smithy b/service/cloudserverS3Extended.smithy new file mode 100644 index 00000000..e24432f8 --- /dev/null +++ b/service/cloudserverS3Extended.smithy @@ -0,0 +1,19 @@ +$version: "2.0" + +namespace cloudserver.s3extended + +use aws.protocols#restXml +use aws.auth#sigv4 +use aws.api#service + +@restXml +@sigv4(name: "s3") +@service(sdkId: "cloudserverS3Extended") +service CloudserverS3Extended { + version: "2018-07-11", + operations: [ + SearchBucket, + SearchBucketV2, + SearchBucketVersions, + ] +} diff --git a/smithy-build.json b/smithy-build.json index 5a475dd5..49bd9ee3 100644 --- a/smithy-build.json +++ b/smithy-build.json @@ -25,6 +25,15 @@ "packageVersion": "1.0.0" } } + }, + "cloudserverS3Extended": { + "plugins": { + "typescript-codegen": { + "service": "cloudserver.s3extended#CloudserverS3Extended", + "package": "@scality/cloudserverclient-s3extended", + "packageVersion": "1.0.0" + } + } } } } diff --git a/src/clients/s3Extended.ts b/src/clients/s3Extended.ts new file mode 100644 index 00000000..1c94b3bc --- /dev/null +++ b/src/clients/s3Extended.ts @@ -0,0 +1,12 @@ +import { + CloudserverS3Extended, + CloudserverS3ExtendedClientConfig +} from '../../build/smithy/cloudserverS3Extended/typescript-codegen'; +import { CloudserverClientConfig } from '../../build/smithy/cloudserver/typescript-codegen'; + +export * from '../../build/smithy/cloudserverS3Extended/typescript-codegen'; +export class S3ExtendedClient extends CloudserverS3Extended { + constructor(config: CloudserverClientConfig | CloudserverS3ExtendedClientConfig) { + super(config); + } +} diff --git a/src/index.ts b/src/index.ts index 27edc3a6..1e1170d6 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,3 +1,4 @@ export * from './clients/cloudserver'; export { BucketQuotaClient } from './clients/bucketQuota'; +export { S3ExtendedClient } from './clients/s3Extended'; export * from './utils'; diff --git a/tests/testS3ExtendedApis.test.ts b/tests/testS3ExtendedApis.test.ts new file mode 100644 index 00000000..648b68fa --- /dev/null +++ b/tests/testS3ExtendedApis.test.ts @@ -0,0 +1,137 @@ +import { S3ExtendedClient } from '../src/index'; +import { + SearchBucketCommand, + SearchBucketV2Command, + SearchBucketVersionsCommand, +} from '../src/clients/s3Extended'; +import { S3Client, PutObjectCommand, DeleteObjectCommand } from '@aws-sdk/client-s3'; +import { createTestClient, testConfig } from './testSetup'; +import assert from 'assert'; + +describe('S3 Extended API Tests', () => { + let client: S3ExtendedClient; + let s3client: S3Client; + const key2ndObject = `${testConfig.objectKey}2nd`; + const body2ndObject = `${testConfig.objectData}2nd`; + + beforeAll(async () => { + const testClients = createTestClient(); + client = testClients.s3ExtendedClient; + s3client = testClients.s3client; + + const putObjectCommand = new PutObjectCommand({ + Bucket: testConfig.bucketName, + Key: key2ndObject, + Body: body2ndObject, + }); + await s3client.send(putObjectCommand); + }); + + it('should test SearchBucketv1', async () => { + const getCommand1= new SearchBucketCommand({ + Bucket: testConfig.bucketName, + Query: `content-length >= ${testConfig.objectData.length}`, + MaxKeys: 5 + }); + const getData1 = await client.send(getCommand1); + assert.strictEqual(getData1.Contents?.length, 2); + + const maxKey = 1; + const getCommand2= new SearchBucketCommand({ + Bucket: testConfig.bucketName, + Query: `content-length >= ${testConfig.objectData.length}`, + MaxKeys: maxKey + }); + const getData2 = await client.send(getCommand2); + assert.strictEqual(getData2.Contents?.length, maxKey); + assert.strictEqual(getData2.IsTruncated, true); + + const getCommand3 = new SearchBucketCommand({ + Bucket: testConfig.bucketName, + Query: `content-length >= ${body2ndObject.length}`, + MaxKeys: 5 + }); + const getData3 = await client.send(getCommand3); + assert.strictEqual(getData3.Contents?.length, 1); + assert.strictEqual(getData3.Contents[0].Key, key2ndObject); + + const getCommand4 = new SearchBucketCommand({ + Bucket: testConfig.bucketName, + Query: `key = ${key2ndObject}`, + MaxKeys: 5 + }); + const getData4 = await client.send(getCommand4); + assert.strictEqual(getData4.Contents?.length, 1); + assert.strictEqual(getData4.Contents[0].Key, key2ndObject); + + const getCommand5 = new SearchBucketCommand({ + Bucket: testConfig.bucketName, + Query: `key = iDontExists`, + MaxKeys: 5 + }); + const getData5 = await client.send(getCommand5); + assert.strictEqual(getData5.Contents, undefined); + }); + + it('should test SearchBucketv2', async () => { + const getCommand1= new SearchBucketV2Command({ + Bucket: testConfig.bucketName, + Query: `content-length >= ${testConfig.objectData.length}`, + MaxKeys: 5, + FetchOwner: true, + }); + const getData1 = await client.send(getCommand1); + assert.strictEqual(getData1.Contents?.length, 2); + assert.strictEqual(getData1.KeyCount, 2); + assert.strictEqual(getData1.Contents[0].Owner?.DisplayName, 'Bart'); + + const getCommand2 = new SearchBucketV2Command({ + Bucket: testConfig.bucketName, + Query: `content-length >= 0`, + MaxKeys: 5, + StartAfter: testConfig.objectKey, // Skip first object + }); + const getData2 = await client.send(getCommand2); + assert.strictEqual(getData2.Contents?.length, 1); + assert.strictEqual(getData2.Contents[0].Key, key2ndObject); + }); + + it('should test SearchBucketVersions', async () => { + const getCommand1 = new SearchBucketVersionsCommand({ + Bucket: testConfig.bucketName, + Query: `content-length >= 0`, + MaxKeys: 100, + }); + const getData1 = await client.send(getCommand1); + assert.strictEqual(getData1.Version?.length, 2); + assert.strictEqual(getData1.Version[0].IsLatest, true); + + // Delete one object to create a DeleteMarker + const deleteCommand = new DeleteObjectCommand({ + Bucket: testConfig.bucketName, + Key: key2ndObject, + }); + await s3client.send(deleteCommand); + + const getCommand2 = new SearchBucketVersionsCommand({ + Bucket: testConfig.bucketName, + Query: `content-length >= 0`, + MaxKeys: 100, + }); + const getData2 = await client.send(getCommand2); + assert.strictEqual(getData2.DeleteMarker?.[0].Key, key2ndObject); + assert.ok(getData2.DeleteMarker?.[0].VersionId); + + assert.ok(getData2.Version); + const firstVersion = getData2.Version[0]; + const getCommand3 = new SearchBucketVersionsCommand({ + Bucket: testConfig.bucketName, + Query: `content-length >= 0`, + KeyMarker: firstVersion.Key, + VersionIdMarker: firstVersion.VersionId, + MaxKeys: 100, + }); + const getData3 = await client.send(getCommand3); + assert.notStrictEqual(getData3.Version?.[0]?.Key, firstVersion.Key); + }); +}); diff --git a/tests/testSetup.ts b/tests/testSetup.ts index cebf7775..3a09c04e 100644 --- a/tests/testSetup.ts +++ b/tests/testSetup.ts @@ -4,6 +4,7 @@ import { CloudserverClient, CloudserverClientConfig } from '../src/clients/cloud import { S3Client, PutObjectCommand, CreateBucketCommand, PutBucketVersioningCommand } from '@aws-sdk/client-s3'; import { AwsCredentialIdentity, AwsCredentialIdentityProvider } from '@aws-sdk/types'; import { BucketQuotaClient } from '../src/clients/bucketQuota'; +import { S3ExtendedClient } from '../src/clients/s3Extended'; jest.setTimeout(30000); const credentialsProvider: AwsCredentialIdentityProvider = async (): Promise => ({ @@ -79,11 +80,13 @@ async function initBucketForTests() { export function createTestClient(): { client: CloudserverClient, bucketQuotaClient: BucketQuotaClient, + s3ExtendedClient: S3ExtendedClient, s3client: S3Client } { return { client: new CloudserverClient(config), bucketQuotaClient: new BucketQuotaClient(config), + s3ExtendedClient: new S3ExtendedClient(config), s3client, }; }