diff --git a/.github/docker/docker-compose.yaml b/.github/docker/docker-compose.yaml index f0d7ad01fb..6cfbcc6ca1 100644 --- a/.github/docker/docker-compose.yaml +++ b/.github/docker/docker-compose.yaml @@ -8,6 +8,7 @@ services: - ${HOME}/.aws/credentials:/root/.aws/credentials - /tmp/artifacts/${JOB_NAME}:/artifacts - /tmp/coverage/${JOB_NAME}:/coverage/test + - /logs:/logs environment: - CI=true - ENABLE_LOCAL_CACHE=true diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index ae044ac1b2..632c4f728d 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -313,11 +313,19 @@ jobs: MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }} CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}-testcoverage JOB_NAME: ${{ github.job }} + S3_SERVER_ACCESS_LOGS_MODE: ENABLED steps: - name: Checkout uses: actions/checkout@v4 - name: Setup CI environment uses: ./.github/actions/setup-ci + - name: Setup server access logs file and directory + shell: bash + run: | + set -exu + sudo mkdir -p /logs + sudo chmod 0777 /logs + sudo touch /logs/server-access.log && sudo chmod 0666 /logs/server-access.log - name: Setup CI services run: docker compose --profile mongo up -d working-directory: .github/docker @@ -366,11 +374,19 @@ jobs: MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }} CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}-testcoverage JOB_NAME: ${{ github.job }} + S3_SERVER_ACCESS_LOGS_MODE: ENABLED steps: - name: Checkout uses: actions/checkout@v4 - name: Setup CI environment uses: ./.github/actions/setup-ci + - name: Setup server access logs file and directory + shell: bash + run: | + set -exu + sudo mkdir -p /logs + sudo chmod 0777 /logs + sudo touch /logs/server-access.log && sudo chmod 0666 /logs/server-access.log - name: Setup CI services run: docker compose --profile mongo up -d working-directory: .github/docker @@ -426,6 +442,7 @@ jobs: MPU_TESTING: "yes" ENABLE_NULL_VERSION_COMPAT_MODE: "${{ matrix.enable-null-compat }}" JOB_NAME: ${{ matrix.job-name }} + S3_SERVER_ACCESS_LOGS_MODE: ENABLED steps: - name: Checkout uses: actions/checkout@v4 @@ -436,6 +453,13 @@ jobs: run: | set -exu mkdir -p /tmp/artifacts/${{ matrix.job-name }}/ + - name: Setup server access logs file and directory + shell: bash + run: | + set -exu + sudo mkdir -p /logs + sudo chmod 0777 /logs + sudo touch /logs/server-access.log && sudo chmod 0666 /logs/server-access.log - name: Setup CI services run: docker compose up -d working-directory: .github/docker @@ -499,6 +523,7 @@ jobs: VAULT_IMAGE: ghcr.io/scality/vault:7.76.0 S3_END_TO_END: true S3_TESTVAL_OWNERCANONICALID: 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be + S3_SERVER_ACCESS_LOGS_MODE: ENABLED steps: - name: Checkout uses: actions/checkout@v4 @@ -515,6 +540,13 @@ jobs: run: | set -exu mkdir -p /tmp/artifacts/${{ matrix.job-name }}/ + - name: Setup server access logs file and directory + shell: bash + run: | + set -exu + sudo mkdir -p /logs + sudo chmod 0777 /logs + sudo touch /logs/server-access.log && sudo chmod 0666 /logs/server-access.log - name: Modify md-config.json for vformat run: | sed -i 's/\("METADATA_NEW_BUCKETS_VFORMAT":\s*\)"[^"]*"/\1"${{ matrix.vformat }}"/' .github/docker/md-config.json @@ -830,6 +862,7 @@ jobs: CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}-testcoverage JOB_NAME: ${{ github.job }} ENABLE_NULL_VERSION_COMPAT_MODE: true # needed with mongodb backend + S3_SERVER_ACCESS_LOGS_MODE: ENABLED steps: - name: Checkout uses: actions/checkout@v4 @@ -841,6 +874,13 @@ jobs: password: ${{ github.token }} - name: Setup CI environment uses: ./.github/actions/setup-ci + - name: Setup server access logs file and directory + shell: bash + run: | + set -exu + sudo mkdir -p /logs + sudo chmod 0777 /logs + sudo touch /logs/server-access.log && sudo chmod 0666 /logs/server-access.log - uses: ruby/setup-ruby@v1 with: ruby-version: '3.2' diff --git a/lib/api/api.js b/lib/api/api.js index 4e2596234b..2a7a876371 100644 --- a/lib/api/api.js +++ b/lib/api/api.js @@ -182,6 +182,7 @@ const api = { }); } if (request.serverAccessLog) { + request.serverAccessLog.bucketName = request.bucketName; request.serverAccessLog.objectKey = request.objectKey; request.serverAccessLog.analyticsAction = actionLog; } diff --git a/lib/api/bucketGet.js b/lib/api/bucketGet.js index 8eef7694e4..d4e7e1b81b 100644 --- a/lib/api/bucketGet.js +++ b/lib/api/bucketGet.js @@ -292,10 +292,18 @@ function bucketGet(authInfo, request, log, callback) { log.addDefaultFields({ action: 'ListObjectsV2', }); + if (request.serverAccessLog) { + // eslint-disable-next-line no-param-reassign + request.serverAccessLog.analyticsAction = 'ListObjectsV2'; + } } else if (params.versions !== undefined) { log.addDefaultFields({ action: 'ListObjectVersions', }); + if (request.serverAccessLog) { + // eslint-disable-next-line no-param-reassign + request.serverAccessLog.analyticsAction = 'ListObjectVersions'; + } } log.debug('processing request', { method: 'bucketGet' }); const encoding = params['encoding-type']; diff --git a/lib/api/multiObjectDelete.js b/lib/api/multiObjectDelete.js index e62e6b756e..e251f5f4fa 100644 --- a/lib/api/multiObjectDelete.js +++ b/lib/api/multiObjectDelete.js @@ -502,6 +502,10 @@ function multiObjectDelete(authInfo, request, log, callback) { if (bucketShield(bucketMD, 'objectDelete')) { return next(errors.NoSuchBucket); } + if (request.serverAccessLog) { + // eslint-disable-next-line no-param-reassign + request.serverAccessLog.bucketOwner = bucketMD.getOwner(); + } // The implicit deny flag is ignored in the DeleteObjects API, as authorization only // affects the objects. if (!isBucketAuthorized(bucketMD, 'objectDelete', canonicalID, authInfo, log, request)) { diff --git a/lib/metadata/metadataUtils.js b/lib/metadata/metadataUtils.js index 648d9d105a..5c1f82b1b8 100644 --- a/lib/metadata/metadataUtils.js +++ b/lib/metadata/metadataUtils.js @@ -28,7 +28,6 @@ function storeServerAccessLogInfo(request, bucket, raftSessionId) { if (bucket) { request.serverAccessLog.bucketOwner = bucket.getOwner(); - request.serverAccessLog.bucketName = bucket.getName(); } if (bucket && bucket.getBucketLoggingStatus() && bucket.getBucketLoggingStatus().getLoggingEnabled()) { diff --git a/lib/server.js b/lib/server.js index 401c11ffb5..60a54b2a1f 100644 --- a/lib/server.js +++ b/lib/server.js @@ -409,7 +409,8 @@ class S3Server { } } - try { + try { + logger.info('ServerAccessLogger config', { config: _config.serverAccessLogs }); if (_config.serverAccessLogs.mode === serverAccessLogsModes.LOG_ONLY || _config.serverAccessLogs.mode === serverAccessLogsModes.ENABLED) { var serverAccessLogger = new ServerAccessLogger( diff --git a/lib/utilities/serverAccessLogger.js b/lib/utilities/serverAccessLogger.js index d83bbdbdd5..448e3a6195 100644 --- a/lib/utilities/serverAccessLogger.js +++ b/lib/utilities/serverAccessLogger.js @@ -181,78 +181,92 @@ function getRemoteIPFromRequest(request) { return remoteIP; } +// eslint-disable-next-line max-len +// https://github.com/awslabs/glue-extensions-for-iceberg/blob/52bdb2908216a85859fd76a45981d0326d016a2f/spark/src/main/scala/software/amazon/glue/s3a/audit/S3LogVerbs.java +// https://github.com/open-io/swift/blob/ff518e9907f74b5a2565973a260f36386b5d5cbf/etc/s3-default.cfg.in#L78 +// https://stackoverflow.com/questions/42707878/amazon-s3-logs-operation-definition +const methodToResType = Object.freeze({ + 'bucketDelete': 'BUCKET', + 'bucketDeleteCors': 'CORS', + 'bucketDeleteEncryption': 'ENCRYPTION', + 'bucketDeleteWebsite': 'WEBSITE', + 'bucketGet': 'BUCKET', + 'bucketGetACL': 'ACL', + 'bucketGetCors': 'CORS', + 'bucketGetObjectLock': 'OBJECT', + 'bucketGetVersioning': 'VERSIONING', + 'bucketGetWebsite': 'WEBSITE', + 'bucketGetLocation': 'LOCATION', + 'bucketGetEncryption': 'ENCRYPTION', + 'bucketHead': 'BUCKET', + 'bucketPut': 'BUCKET', + 'bucketPutACL': 'ACL', + 'bucketPutCors': 'CORS', + 'bucketPutVersioning': 'VERSIONING', + 'bucketPutTagging': 'TAGGING', + 'bucketDeleteTagging': 'TAGGING', + 'bucketGetTagging': 'TAGGING', + 'bucketPutWebsite': 'WEBSITE', + 'bucketPutReplication': 'REPLICATION', + 'bucketGetReplication': 'REPLICATION', + 'bucketDeleteReplication': 'REPLICATION', + 'bucketDeleteQuota': 'QUOTA', + 'bucketPutLifecycle': 'LIFECYCLE', + 'bucketUpdateQuota': 'QUOTA', + 'bucketGetLifecycle': 'LIFECYCLE', + 'bucketDeleteLifecycle': 'LIFECYCLE', + 'bucketPutPolicy': 'BUCKETPOLICY', + 'bucketGetPolicy': 'BUCKETPOLICY', + 'bucketGetQuota': 'QUOTA', + 'bucketDeletePolicy': 'BUCKETPOLICY', + 'bucketPutObjectLock': 'OBJECT', + 'bucketPutNotification': 'NOTIFICATION', + 'bucketGetNotification': 'NOTIFICATION', + 'bucketPutEncryption': 'ENCRYPTION', + 'bucketPutLogging': 'LOGGING_STATUS', + 'bucketGetLogging': 'LOGGING_STATUS', + // 'corsPreflight': '', + 'completeMultipartUpload': 'UPLOAD', + 'initiateMultipartUpload': 'UPLOADS', + 'listMultipartUploads': 'UPLOADS', + 'listParts': 'UPLOAD', + 'metadataSearch': 'OBJECT', + 'multiObjectDelete': 'OBJECT', + 'multipartDelete': 'UPLOAD', + 'objectDelete': 'OBJECT', + 'objectDeleteTagging': 'TAGGING', + 'objectGet': 'OBJECT', + 'objectGetACL': 'ACL', + 'objectGetLegalHold': 'LEGALHOLD', + 'objectGetRetention': 'OBJECT_LOCK_RETENTION', + 'objectGetTagging': 'TAGGING', + 'objectCopy': 'COPY', + 'objectHead': 'OBJECT', + 'objectPut': 'OBJECT', + 'objectPutACL': 'ACL', + 'objectPutLegalHold': 'LEGALHOLD', + 'objectPutTagging': 'TAGGING', + 'objectPutPart': 'PART', + 'objectPutCopyPart': 'COPY', + 'objectPutRetention': 'OBJECT_LOCK_RETENTION', + 'objectRestore': 'OBJECT', + 'serviceGet': 'SERVICE', // ListBuckets + 'websiteGet': 'WEBSITE', + 'websiteHead': 'WEBSITE', +}); + function getOperation(req) { - const methodToResType = Object.freeze({ - 'bucketDelete': 'BUCKET', - 'bucketDeleteCors': 'BUCKET', - 'bucketDeleteEncryption': 'BUCKET', - 'bucketDeleteWebsite': 'BUCKET', - 'bucketGet': 'BUCKET', - 'bucketGetACL': 'BUCKET', - 'bucketGetCors': 'BUCKET', - 'bucketGetObjectLock': 'BUCKET', - 'bucketGetVersioning': 'VERSIONING', - 'bucketGetWebsite': 'BUCKET', - 'bucketGetLocation': 'BUCKET', - 'bucketGetEncryption': 'BUCKET', - 'bucketHead': 'BUCKET', - 'bucketPut': 'BUCKET', - 'bucketPutACL': 'BUCKET', - 'bucketPutCors': 'BUCKET', - 'bucketPutVersioning': 'VERSIONING', - 'bucketPutTagging': 'BUCKET', - 'bucketDeleteTagging': 'BUCKET', - 'bucketGetTagging': 'BUCKET', - 'bucketPutWebsite': 'BUCKET', - 'bucketPutReplication': 'BUCKET', - 'bucketGetReplication': 'BUCKET', - 'bucketDeleteReplication': 'BUCKET', - 'bucketDeleteQuota': 'BUCKET', - 'bucketPutLifecycle': 'BUCKET', - 'bucketUpdateQuota': 'BUCKET', - 'bucketGetLifecycle': 'BUCKET', - 'bucketDeleteLifecycle': 'BUCKET', - 'bucketPutPolicy': 'BUCKETPOLICY', - 'bucketGetPolicy': 'BUCKETPOLICY', - 'bucketGetQuota': 'BUCKET', - 'bucketDeletePolicy': 'BUCKETPOLICY', - 'bucketPutObjectLock': 'BUCKET', - 'bucketPutNotification': 'BUCKET', - 'bucketGetNotification': 'BUCKET', - 'bucketPutEncryption': 'BUCKET', - 'bucketPutLogging': 'LOGGING_STATUS', - 'bucketGetLogging': 'LOGGING_STATUS', - // 'corsPreflight': '', - 'completeMultipartUpload': 'OBJECT', - 'initiateMultipartUpload': 'OBJECT', - 'listMultipartUploads': 'OBJECT', - 'listParts': 'OBJECT', - 'metadataSearch': 'OBJECT', - 'multiObjectDelete': 'OBJECT', - 'multipartDelete': 'OBJECT', - 'objectDelete': 'OBJECT', - 'objectDeleteTagging': 'OBJECT', - 'objectGet': 'OBJECT', - 'objectGetACL': 'OBJECT', - 'objectGetLegalHold': 'OBJECT', - 'objectGetRetention': 'OBJECT', - 'objectGetTagging': 'OBJECT', - 'objectCopy': 'OBJECT', - 'objectHead': 'OBJECT', - 'objectPut': 'OBJECT', - 'objectPutACL': 'OBJECT', - 'objectPutLegalHold': 'OBJECT', - 'objectPutTagging': 'OBJECT', - 'objectPutPart': 'OBJECT', - 'objectPutCopyPart': 'OBJECT', - 'objectPutRetention': 'OBJECT', - 'objectRestore': 'OBJECT', - // 'serviceGet': '', - // 'websiteGet': '', - // 'websiteHead': '', - }); - - return `REST.${req.method}.${methodToResType[req.apiMethod] ? methodToResType[req.apiMethod] : 'UNKNOWN'}`; + const resourceType = methodToResType[req.apiMethod]; + if (!resourceType) { + process.emitWarning('Unknown apiMethod for server access log', { + type: 'ServerAccessLogWarning', + code: 'UNKNOWN_API_METHOD', + detail: `apiMethod=${req.apiMethod}, method=${req.method}, url=${req.url}` + }); + return `REST.${req.method}.UNKNOWN`; + } + + return `REST.${req.method}.${resourceType}`; } function getRequester(authInfo) { @@ -284,33 +298,43 @@ function getURI(request) { return requestURI; } -function getObjectSize(request, response) { - const objectSizePutMethods = Object.freeze({ - 'objectPut': true, - 'objectPutPart': true, - }); +const objectSizePutMethods = Object.freeze({ + 'objectPut': true, + 'objectPutPart': true, +}); - const objectSizeGetMethods = Object.freeze({ - 'objectGet': true, - }); +const objectSizeGetMethods = Object.freeze({ + 'objectGet': true, +}); +function getObjectSize(request, response) { // If it is a PUT get the Content-Length from the request, if it is a GET get it from the response. if (request && response && objectSizeGetMethods[request.apiMethod]) { const len = response.getHeader('Content-Length'); - if (len === undefined || len === null) { + if (len === undefined || len === null || len === '') { return null; } - return len; + const numLen = Number(len); + if (isNaN(numLen)) { + return null; + } + + return numLen; } if (request && objectSizePutMethods[request.apiMethod]) { const len = request.headers['content-length']; - if (len === undefined || len === null) { + if (len === undefined || len === null || len === '') { + return null; + } + + const numLen = Number(len); + if (isNaN(numLen)) { return null; } - return len; + return numLen; } return null; @@ -387,50 +411,50 @@ function logServerAccess(req, res) { pid: process.pid, // Analytics - action: params.analyticsAction || null, - accountName: params.analyticsAccountName || null, + action: params.analyticsAction === undefined ? null : params.analyticsAction, + accountName: params.analyticsAccountName === undefined ? null : params.analyticsAccountName, accountDisplayName: authInfo ? authInfo.getAccountDisplayName() : null, - userName: params.analyticsUserName || null, - clientPort: req.socket.remotePort || null, - httpMethod: req.method || null, - bytesDeleted: params.analyticsBytesDeleted || params.analyticsBytesDeleted === 0 ? 0 : null, - bytesReceived: req.parsedContentLength || 0, - bodyLength: parseInt(req.headers['content-length'], 10) || 0, - contentLength: req.parsedContentLength || 0, + userName: params.analyticsUserName === undefined ? null : params.analyticsUserName, + clientPort: req.socket.remotePort === undefined ? null : req.socket.remotePort, + httpMethod: req.method === undefined ? null : req.method, + bytesDeleted: params.analyticsBytesDeleted === undefined ? null : params.analyticsBytesDeleted, + bytesReceived: req.parsedContentLength === undefined ? null : req.parsedContentLength, + bodyLength: req.headers['content-length'] === undefined ? null : parseInt(req.headers['content-length'], 10), + contentLength: req.parsedContentLength === undefined ? null : req.parsedContentLength, // eslint-disable-next-line camelcase elapsed_ms: calculateElapsedMS(params.startTime, params.onCloseEndTime), - httpURL: req.url || null, + httpURL: req.url === undefined ? null : req.url, // AWS access server logs fields https://docs.aws.amazon.com/AmazonS3/latest/userguide/LogFormat.html startTime: timestampToDateTime643(params.startTimeUnixMS), // AWS "Time" field requester: getRequester(authInfo), operation: getOperation(req), requestURI: getURI(req), - errorCode: errorCode || null, + errorCode: errorCode === undefined ? null : errorCode, objectSize: getObjectSize(req, res), totalTime: calculateTotalTime(params.startTime, params.onFinishEndTime), turnAroundTime: calculateTurnAroundTime(params.startTurnAroundTime, endTurnAroundTime), - referer: req.headers.referer || null, - userAgent: req.headers['user-agent'] || null, - versionID: req.query ? req.query.versionId || null : null, + referer: req.headers.referer === undefined ? null : req.headers.referer, + userAgent: req.headers['user-agent'] === undefined ? null : req.headers['user-agent'], + versionID: !req.query ? null : req.query.versionId === undefined ? null : req.query.versionId, signatureVersion: authInfo ? authInfo.getAuthVersion() : null, cipherSuite: req.socket.encrypted ? req.socket.getCipher()['standardName'] : null, authenticationType: authInfo ? authInfo.getAuthType() : null, - hostHeader: req.headers.host || null, + hostHeader: req.headers.host === undefined ? null : req.headers.host, tlsVersion: req.socket.encrypted ? req.socket.getCipher()['version'] : null, aclRequired: null, // TODO: CLDSRV-774 // hostID: null, // NOT IMPLEMENTED // accessPointARN: null, // NOT IMPLEMENTED // Shared between AWS access server logs and Analytics logs - bucketOwner: params.bucketOwner || null, - bucketName: params.bucketName || null, // AWS "Bucket" field + bucketOwner: params.bucketOwner === undefined ? null : params.bucketOwner, + bucketName: params.bucketName === undefined ? null : params.bucketName, // AWS "Bucket" field // eslint-disable-next-line camelcase - req_id: requestID || null, // AWS "Request ID" field + req_id: requestID === undefined ? null : requestID, // AWS "Request ID" field bytesSent: getBytesSent(res, bytesSent), clientIP: getRemoteIPFromRequest(req), // AWS 'Remote IP' field - httpCode: res.statusCode || null, // AWS "HTTP Status" field - objectKey: params.objectKey || null, // AWS "Key" field + httpCode: res.statusCode === undefined ? null : res.statusCode, // AWS "HTTP Status" field + objectKey: params.objectKey === undefined ? null : params.objectKey, // AWS "Key" field // Scality server access logs extra fields logFormatVersion: SERVER_ACCESS_LOG_FORMAT_VERSION, @@ -438,7 +462,7 @@ function logServerAccess(req, res) { loggingTargetBucket: params.loggingEnabled ? params.loggingEnabled.TargetBucket : null, loggingTargetPrefix: params.loggingEnabled ? params.loggingEnabled.TargetPrefix : null, awsAccessKeyID: authInfo ? authInfo.getAccessKey() : null, - raftSessionID: params.raftSessionID || null, + raftSessionID: params.raftSessionID === undefined ? null : params.raftSessionID, })}\n`); } diff --git a/package.json b/package.json index 4274f41d6c..4137bf3ac8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@zenko/cloudserver", - "version": "9.2.5", + "version": "9.2.6", "description": "Zenko CloudServer, an open-source Node.js implementation of a server handling the Amazon S3 protocol", "main": "index.js", "engines": { @@ -21,7 +21,7 @@ "dependencies": { "@azure/storage-blob": "^12.28.0", "@hapi/joi": "^17.1.1", - "arsenal": "git+https://github.com/scality/Arsenal#8.2.41", + "arsenal": "git+https://github.com/scality/Arsenal#8.2.42", "async": "2.6.4", "aws-sdk": "^2.1692.0", "bucketclient": "scality/bucketclient#8.2.7", diff --git a/schema/server_access_log.schema.json b/schema/server_access_log.schema.json new file mode 100644 index 0000000000..a191437b79 --- /dev/null +++ b/schema/server_access_log.schema.json @@ -0,0 +1,248 @@ +{ + "$id": "https://github.com/scality/cloudserver/tree/development/9.2/schema/lib/server_access_log.schema.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Server Access Log", + "description": "A server access log. AWS fields are documented in https://docs.aws.amazon.com/AmazonS3/latest/userguide/LogFormat.html.", + "type": "object", + "properties": { + "time": { + "description": "Epoch timestamp in seconds, recorded when the log record is created.", + "type": "integer", + "minimum": 0 + }, + "hostname": { + "description": "Hostname of the worker as returned by os.hostname().", + "type": "string" + }, + "pid": { + "description": "PID of the worker.", + "type": "integer", + "minimum": 0 + }, + "action": { + "description": "S3 API action name.", + "type": "string" + }, + "accountName": { + "description": "Requester account display name.", + "type": ["string", "null"] + }, + "accountDisplayName": { + "description": "Requester account display name.", + "type": ["string", "null"] + }, + "userName": { + "description": "Requester IAM display name, or null if the requester is not an IAM user.", + "type": ["string", "null"] + }, + "clientPort": { + "description": "Requester remote connection port.", + "type": "integer", + "maximum": 65535, + "minimum": 1 + }, + "httpMethod": { + "description": "Request HTTP method.", + "type": "string", + "enum": ["GET", "HEAD", "OPTIONS", "TRACE", "PUT", "DELETE", "POST", "PATCH", "CONNECT"] + }, + "bytesDeleted": { + "description": "For DeleteObject: size in bytes of the deleted object, for DeleteObjects: sum of all the deleted objects.", + "type": ["integer", "null"], + "minimum": 0 + }, + "bytesReceived": { + "description": "For PutObject and UploadPart: size of the object in bytes.", + "type": "integer", + "minimum": 0 + }, + "bodyLength": { + "description": "Size in bytes of the request body, copied from the HTTP Content-Length header.", + "type": "integer", + "minimum": 0 + }, + "contentLength": { + "description": "Size in bytes of the request content.", + "type": "integer", + "minimum": 0 + }, + "elapsed_ms": { + "description": "Total duration of the request in milliseconds. The timer starts when the server first routes the request and stops when the request completes or is closed prematurely.", + "type": "number", + "minimum": 0 + }, + "httpURL": { + "description": "Request URL string. This contains only the URL that is present in the actual HTTP request.", + "type": "string" + }, + "startTime": { + "description": "Timestamp formatted as: 'seconds.milliseconds', recorded when the server first routes the request. Represents the AWS server access log 'Time' field. String type compatible with Clickhouse DateTime64(3) type.", + "type": "string" + }, + "requester": { + "description": "AWS server access log 'Requester' field. From AWS 'The canonical user ID of the requester, or a - for unauthenticated requests. If the requester was an IAM user, this field returns the requester's IAM user name along with the AWS account that the IAM user belongs to. This identifier is the same one used for access control purposes.'. We don't use null instead of '-' when the requester is missing.", + "type": ["string", "null"] + }, + "operation": { + "description": "AWS server access log 'Operation' field. From AWS 'The operation listed here is declared as SOAP.operation, REST.HTTP_method.resource_type, WEBSITE.HTTP_method.resource_type, or BATCH.DELETE.OBJECT, or S3.action.resource_type for S3 Lifecycle and logging. For Compute checksum job requests, the operation is listed as S3.COMPUTE.OBJECT.CHECKSUM.'.", + "type": "string" + }, + "requestURI": { + "description": "AWS server access log 'Request URI' field. From AWS 'The Request-URI part of the HTTP request message.'.", + "type": "string" + }, + "errorCode": { + "description": "AWS server access log 'Error Code' field. From AWS 'The Amazon S3 Error responses , or - if no error occurred.'. We use null to signal no error.", + "type": ["string", "null"] + }, + "objectSize": { + "description": "AWS server access log 'Object Size' field.", + "type": ["integer", "null"] + }, + "totalTime": { + "description": "AWS server access log 'Total Time' field. From AWS 'The number of milliseconds that the request was in flight from the server's perspective. This value is measured from the time that your request is received to the time that the last byte of the response is sent. Measurements made from the client's perspective might be longer because of network latency.'.", + "type": "string" + }, + "turnAroundTime": { + "description": "AWS server access log 'Turn Around Time' field. From AWS 'The number of milliseconds that Amazon S3 spent processing your request. This value is measured from the time that the last byte of your request was received until the time that the first byte of the response was sent.'.", + "type": ["string", "null"] + }, + "referer": { + "description": "AWS server access log 'Referer' field. From AWS 'The value of the HTTP Referer header, if present. HTTP user-agents (for example, browsers) typically set this header to the URL of the linking or embedding page when making a request.'.", + "type": ["string", "null"] + }, + "userAgent": { + "description": "AWS server access log 'User Agent' field. From AWS 'The value of the HTTP User-Agent header.'.", + "type": ["string", "null"] + }, + "versionID": { + "description": "AWS server access log 'Version ID' field. From AWS 'The version ID of the object being copied, or - if the x-amz-copy-source header didn't specify a versionId parameter as part of the copy source.'. We use null to signal no versionId.", + "type": ["string", "null"] + }, + "signatureVersion": { + "description": "AWS server access log 'Signature Version' field. From AWS 'The signature version, SigV2 or SigV4, that was used to authenticate the request, or a - for unauthenticated requests.'. We use null for unauthenticated requests.", + "type": ["string", "null"] + }, + "cipherSuite": { + "description": "AWS server access log 'Cipher Suite' field. From AWS 'The Transport Layer Security (TLS) cipher that was negotiated for an HTTPS request, or a - for HTTP.'. We use null for HTTP.", + "type": ["string", "null"] + }, + "authenticationType": { + "description": "AWS server access log 'Authentication Type' field. From AWS 'The type of request authentication used: AuthHeader for authentication headers, QueryString for query strings (presigned URLs), or a - for unauthenticated requests.'. We use null for unauthenticated requests.", + "type": ["string", "null"] + }, + "hostHeader": { + "description": "AWS server access log 'Host Header' field. From AWS 'The endpoint that was used to connect to Amazon S3.'.", + "type": "string" + }, + "tlsVersion": { + "description": "AWS server access log 'TLS Version' field. From AWS 'The Transport Layer Security (TLS) version negotiated by the client. The value is one of following: TLSv1.1, TLSv1.2, TLSv1.3, or - if TLS wasn't used.'. We use null if TLS was not used.", + "type": ["string", "null"] + }, + "aclRequired": { + "description": "AWS server access log 'ACL Required' field.", + "type": ["null"] + }, + "bucketOwner": { + "description": "AWS server access log 'Bucket Owner' field. From AWS 'The canonical user ID of the owner of the source bucket. The canonical user ID is another form of the AWS account ID'.", + "type": ["string", "null"] + }, + "bucketName": { + "description": "AWS server access log 'Bucket' field. Null for ListBuckets.", + "type": ["string", "null"] + }, + "req_id": { + "description": "AWS server access log 'Request ID' field. Matches the req_id field in logs.", + "type": "string" + }, + "bytesSent": { + "description": "AWS server access log 'Bytes Sent' field. From AWS 'The number of response bytes sent, excluding HTTP protocol overhead, or - if zero.'. We 0 and null are possible values in our implementation.", + "type": ["integer", "null"] + }, + "clientIP": { + "description": "AWS server access log 'Remote IP' field. From AWS 'The apparent IP address of the requester. Intermediate proxies and firewalls might obscure the actual IP address of the machine that's making the request.'.", + "type": "string" + }, + "httpCode": { + "description": "AWS server access log 'HTTP Status' field. From AWS 'The numeric HTTP status code of the response.'.", + "type": "integer" + }, + "objectKey": { + "description": "AWS server access log 'Key' field. From AWS 'The key (object name) part of the request.'.", + "type": ["string", "null"] + }, + "logFormatVersion": { + "description": "Version of the server access log schema.", + "type": ["string"], + "const": "0" + }, + "loggingEnabled": { + "description": "True if the target bucket has bucket logging configured and enabled.", + "type": "boolean" + }, + "loggingTargetBucket": { + "description": "Target bucket where server access logs should be uploaded. https://docs.aws.amazon.com/AmazonS3/latest/API/API_LoggingEnabled.html", + "type": ["string", "null"] + }, + "loggingTargetPrefix": { + "description": "Prefix used when creating the log object in the Target bucket. https://docs.aws.amazon.com/AmazonS3/latest/API/API_LoggingEnabled.html", + "type": ["string", "null"] + }, + "awsAccessKeyID": { + "description": "Requester AWS access key ID.", + "type": ["string", "null"] + }, + "raftSessionID": { + "description": "Raft session ID.", + "type": ["integer", "null"] + } + }, + "additionalProperties": false, + "required": [ + "time", + "hostname", + "pid", + "action", + "accountName", + "accountDisplayName", + "userName", + "clientPort", + "httpMethod", + "bytesDeleted", + "bytesReceived", + "bodyLength", + "contentLength", + "elapsed_ms", + "httpURL", + "startTime", + "requester", + "operation", + "requestURI", + "errorCode", + "objectSize", + "totalTime", + "turnAroundTime", + "referer", + "userAgent", + "versionID", + "signatureVersion", + "cipherSuite", + "authenticationType", + "hostHeader", + "tlsVersion", + "aclRequired", + "bucketOwner", + "bucketName", + "req_id", + "bytesSent", + "clientIP", + "httpCode", + "objectKey", + "logFormatVersion", + "loggingEnabled", + "loggingTargetBucket", + "loggingTargetPrefix", + "awsAccessKeyID", + "raftSessionID" + ] +} diff --git a/tests/functional/aws-node-sdk/test/serverAccessLogs/testServerAccessLogFile.js b/tests/functional/aws-node-sdk/test/serverAccessLogs/testServerAccessLogFile.js new file mode 100644 index 0000000000..c1d51bedb5 --- /dev/null +++ b/tests/functional/aws-node-sdk/test/serverAccessLogs/testServerAccessLogFile.js @@ -0,0 +1,2652 @@ +const assert = require('assert'); +const fs = require('fs'); +const path = require('path'); +const tv4 = require('tv4'); + +const withV4 = require('../support/withV4'); +const BucketUtility = require('../../lib/utility/bucket-util'); +const { config, serverAccessLogsModes } = require('../../../../../lib/Config'); + +const TEST_CONFIG = { + MAX_LOG_WAIT_RETRIES: 50, + LOG_POLL_DELAY_MS: 100, +}; + +// Load the JSON schema +const schemaPath = path.join(__dirname, '../../../../../schema/server_access_log.schema.json'); +const schema = JSON.parse(fs.readFileSync(schemaPath, 'utf8')); + +function truncateLogFileIfExists(filePath) { + if (fs.existsSync(filePath)) { + fs.truncateSync(filePath, 0); + } +} + +async function waitForLogs(filePath, expectedLines, maxRetries, delayMs) { + for (let attempt = 0; attempt < maxRetries; attempt++) { + const logEntries = fs.readFileSync(filePath, 'utf8'); + const lines = logEntries.trim().split('\n').filter(line => line.length > 0); + if (lines.length >= expectedLines) { + try { + return lines.map(line => JSON.parse(line)); + } catch (err) { + // FIXME(CLDSRV-800): readFileSync may read partial lines making JSON.parse fail, so we need to retry. + if (attempt == maxRetries) { + throw new Error(`Failed to read log entries from ${filePath} after ${maxRetries} attempts: ${err}`); + } + } + } + await sleep(delayMs); + } + throw new Error(`Failed to read log entries from ${filePath} after ${maxRetries} attempts`); +} + +async function waitForAction(filePath, action, maxRetries, delayMs) { + for (let attempt = 0; attempt < maxRetries; attempt++) { + const logEntries = fs.readFileSync(filePath, 'utf8'); + const lines = logEntries.trim().split('\n').filter(line => line.length > 0); + for (const line of lines) { + try { + const obj = JSON.parse(line); + if (obj.action === action) { + return; + } + } catch (err) { + // FIXME(CLDSRV-800): readFileSync may read partial lines making JSON.parse fail, so we need to retry. + if (attempt == maxRetries) { + throw new Error(`Failed to read log entries from ${filePath} after ${maxRetries} attempts: ${err}`); + } + } + } + await sleep(delayMs); + } + throw new Error(`Failed to read log entries from ${filePath} after ${maxRetries} attempts`); +} + +function sleep(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +async function cleanupBuckets(s3) { + let lastAction = 'ListBuckets'; + const bucketsResponse = await s3.listBuckets().promise(); + for (const bucket of bucketsResponse.Buckets) { + const listMPUResponse = await s3.listMultipartUploads({ Bucket: bucket.Name }).promise(); + if (listMPUResponse.Uploads && listMPUResponse.Uploads.length > 0) { + await Promise.all(listMPUResponse.Uploads.map(upload => + s3.abortMultipartUpload({ + Bucket: bucket.Name, + Key: upload.Key, + UploadId: upload.UploadId, + }).promise(), + )); + } + + await bu.empty(bucket.Name, true); + await s3.deleteBucket({ Bucket: bucket.Name }).promise(); + lastAction = 'DeleteBucket'; + } + return lastAction; +} + +var bu; + +// TODO: +// - [ ] We skip websiteGet and websiteHead because they need to be tested with HTTP requests to the website endpoint. +// Cannot delete the locked objects in the bucket, so we cannot delete the bucket. +// - [ ] Cloudserver returns PutBucketNotification action for PutBucketNotificationConfiguration (same for the Get) +// - We skip QUOTA methods because they are not part of the AWS API. +// - We skip objectRestore because it is not supported in CloudServer. +describe('Server Access Logs - File Output', async () => { + withV4(async sigCfg => { + const bucketUtil = new BucketUtility('default', sigCfg); + bu = bucketUtil; + const s3 = bucketUtil.s3; + const logFilePath = config.serverAccessLogs.outputFile; + const bucketName = 'test-server-access-log-bucket'; + const objectKey = 'test-object-key'; + + // List of all the log properties. + // Commented properties are set by the test or not tested. + // UNKNOWN: The property is not accessible by the test or is non determistic. + // DYNAMIC: The propery is set depending on the operation. + // STATIC: Shared among all tests. + // TODO: The property need to be tested. + const commonProperties = { + // 'time': '', // UNKNOWN + // 'hostname': '', // UNKNOWN + // 'pid': '', // UNKNOWN + 'action': 'REQUIRED', // DYNAMIC + 'accountName': 'Bart', // STATIC + 'accountDisplayName': 'Bart', // STATIC + 'userName': null, // TODO: Add test with IAM user to get a non null userName. + // 'clientPort': '', // UNKNOWN + 'httpMethod': 'REQUIRED', // DYNAMIC + // 'bytesDeleted': '', // TODO + // 'bytesReceived': '', // TODO + // 'bodyLength': '', // TODO + // 'contentLength': '', // TODO + // 'elapsed_ms': '', // UNKNOWN + // 'httpURL': '', // TODO + // 'startTime': '', // UNKNOWN + 'requester': '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be', // STATIC + 'operation': 'REQUIRED', // DYNAMIC + // 'requestURI': '', // TODO + 'errorCode': null, // DYNAMIC + // 'objectSize': '', // TODO + // 'totalTime': '', // UNKNOWN + // 'turnAroundTime': '', // UNKNOWN + 'referer': null, // TODO: Add test that sets the referer. + // 'userAgent': // UNKNOWN + // 'versionID': '', // UNKNOWN + 'signatureVersion': 'SigV4', // STATIC + 'cipherSuite': null, // TODO: Add https tests. + 'authenticationType': 'AuthHeader', // STATIC + // 'hostHeader': '', // UNKNOWN + 'tlsVersion': null, // TODO: Add https tests. + 'aclRequired': null, // TODO: Add https tests. + 'bucketOwner': '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be', // DYNAMIC + bucketName, // DYNAMIC + // 'req_id': '', // UNKNOWN + // 'bytesSent': '', // TODO + // 'clientIP': '', // UNKNOWN + 'httpCode': 200, // DYNAMIC + 'objectKey': null, // DYNAMIC + 'logFormatVersion': '0', // STATIC + 'loggingEnabled': false, // DYNAMIC + 'loggingTargetBucket': null, // DYNAMIC + 'loggingTargetPrefix': null, // DYNAMIC + 'awsAccessKeyID': 'accessKey1', // STATIC + 'raftSessionID': null, // UNKNOWN + }; + + const operations = [ + (() => { + // This operation tests deleting a bucket and expects a server access log entry for the bucket deletion. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + await s3.deleteBucket({ Bucket: bucketName }).promise(); + }; + return { + method, + methodName: 'bucketDelete', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + bucketOwner: null, + action: 'CreateBucket', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.DELETE.BUCKET', + action: 'DeleteBucket', + httpCode: 204, + httpMethod: 'DELETE', + } + ], + }; + })(), + (() => { + // This operation tests deleting a bucket's CORS configuration + // and expects a log entry for that operation. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + // CORS must be set before it can be deleted + await s3.putBucketCors({ + Bucket: bucketName, + CORSConfiguration: { + CORSRules: [{ + AllowedHeaders: ['*'], + AllowedMethods: ['GET', 'PUT'], + AllowedOrigins: ['*'], + }] + } + }).promise(); + await s3.deleteBucketCors({ Bucket: bucketName }).promise(); + }; + return { + method, + methodName: 'bucketDeleteCors', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + bucketOwner: null, + action: 'CreateBucket', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.CORS', + action: 'PutBucketCors', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.DELETE.CORS', + action: 'DeleteBucketCors', + httpCode: 204, + httpMethod: 'DELETE', + } + ], + }; + })(), + (() => { + // This operation tests deleting a bucket's encryption configuration + // and expects a log entry for that operation. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + // Bucket encryption must be configured before it can be deleted + await s3.putBucketEncryption({ + Bucket: bucketName, + ServerSideEncryptionConfiguration: { + Rules: [ + { + ApplyServerSideEncryptionByDefault: { + SSEAlgorithm: 'AES256', + } + } + ] + } + }).promise(); + await s3.deleteBucketEncryption({ Bucket: bucketName }).promise(); + }; + return { + method, + methodName: 'bucketDeleteEncryption', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + bucketOwner: null, + action: 'CreateBucket', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.ENCRYPTION', + action: 'PutBucketEncryption', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.DELETE.ENCRYPTION', + action: 'DeleteBucketEncryption', + httpCode: 204, + httpMethod: 'DELETE', + } + ], + }; + })(), + (() => { + // This operation tests deleting a bucket's website configuration + // and expects a log entry for that operation. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + // Website configuration must be set before it can be deleted + await s3.putBucketWebsite({ + Bucket: bucketName, + WebsiteConfiguration: { + IndexDocument: { + Suffix: 'index.html', + }, + }, + }).promise(); + await s3.deleteBucketWebsite({ Bucket: bucketName }).promise(); + }; + return { + method, + methodName: 'bucketDeleteWebsite', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + bucketOwner: null, + action: 'CreateBucket', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.WEBSITE', + action: 'PutBucketWebsite', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.DELETE.WEBSITE', + action: 'DeleteBucketWebsite', + httpCode: 204, + httpMethod: 'DELETE', + } + ], + }; + })(), + (() => { + // This operation tests the ListBucketV2 API and expects a log entry for that operation. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + // Upload an object to ensure the bucket is not empty + await s3.putObject({ Bucket: bucketName, Key: objectKey, Body: 'Data' }).promise(); + // Issue the ListBucketV2 request + await s3.listObjectsV2({ Bucket: bucketName }).promise(); + }; + return { + method, + methodName: 'listObjectsV2', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + bucketOwner: null, + action: 'CreateBucket', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.OBJECT', + action: 'PutObject', + objectKey, + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.GET.BUCKET', + action: 'ListObjectsV2', + httpMethod: 'GET', + } + ], + }; + })(), + (() => { + // This operation tests the ListObjects (v1) API and expects a log entry for that operation. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + // Upload an object to ensure the bucket is not empty + await s3.putObject({ Bucket: bucketName, Key: objectKey, Body: 'Data' }).promise(); + // Issue the ListObjects request + await s3.listObjects({ Bucket: bucketName }).promise(); + }; + return { + method, + methodName: 'bucketGet', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + bucketOwner: null, + action: 'CreateBucket', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.OBJECT', + action: 'PutObject', + objectKey, + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.GET.BUCKET', + action: 'ListObjects', + httpMethod: 'GET', + } + ], + }; + })(), + (() => { + // This operation tests getting a bucket's ACL + // and expects log entries for bucket creation and getting the ACL. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + await s3.getBucketAcl({ Bucket: bucketName }).promise(); + }; + return { + method, + methodName: 'bucketGetACL', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + bucketOwner: null, + action: 'CreateBucket', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.GET.ACL', + action: 'GetBucketAcl', + httpMethod: 'GET', + } + ], + }; + })(), + (() => { + // This operation tests getting a bucket's CORS configuration + // and expects log entries for bucket creation, + // putting a CORS configuration, and getting the CORS configuration. + const corsConfig = { + CORSRules: [ + { + AllowedOrigins: ['*'], + AllowedMethods: ['GET', 'POST'], + } + ] + }; + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + await s3.putBucketCors({ Bucket: bucketName, CORSConfiguration: corsConfig }).promise(); + await s3.getBucketCors({ Bucket: bucketName }).promise(); + }; + return { + method, + methodName: 'bucketGetCors', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + bucketOwner: null, + action: 'CreateBucket', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.CORS', + action: 'PutBucketCors', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.GET.CORS', + action: 'GetBucketCors', + httpMethod: 'GET', + } + ], + }; + })(), + (() => { + // This operation tests getting a bucket's Object Lock configuration. + const method = async () => { + await s3.createBucket({ + Bucket: bucketName, + ObjectLockEnabledForBucket: true, + }).promise(); + await s3.getObjectLockConfiguration({ Bucket: bucketName }).promise(); + }; + return { + method, + methodName: 'bucketGetObjectLock', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + bucketOwner: null, + action: 'CreateBucket', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.GET.OBJECT', + action: 'GetObjectLockConfiguration', + httpMethod: 'GET', + } + ], + }; + })(), + (() => { + // This operation tests getting a bucket's versioning configuration. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + await s3.getBucketVersioning({ Bucket: bucketName }).promise(); + }; + return { + method, + methodName: 'bucketGetVersioning', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + bucketOwner: null, + action: 'CreateBucket', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.GET.VERSIONING', + action: 'GetBucketVersioning', + httpMethod: 'GET', + } + ], + }; + })(), + (() => { + // This operation tests getting a bucket's website configuration. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + await s3.putBucketWebsite({ + Bucket: bucketName, + WebsiteConfiguration: { + IndexDocument: { Suffix: 'index.html' }, + }, + }).promise(); + await s3.getBucketWebsite({ Bucket: bucketName }).promise(); + }; + return { + method, + methodName: 'bucketGetWebsite', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + bucketOwner: null, + action: 'CreateBucket', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.WEBSITE', + action: 'PutBucketWebsite', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.GET.WEBSITE', + action: 'GetBucketWebsite', + httpMethod: 'GET', + } + ], + }; + })(), + (() => { + // This operation tests getting a bucket's location. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + await s3.getBucketLocation({ Bucket: bucketName }).promise(); + }; + return { + method, + methodName: 'bucketGetLocation', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + bucketOwner: null, + action: 'CreateBucket', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.GET.LOCATION', + action: 'GetBucketLocation', + httpMethod: 'GET', + } + ], + }; + })(), + (() => { + // This operation tests getting a bucket's encryption configuration. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + await s3.putBucketEncryption({ + Bucket: bucketName, + ServerSideEncryptionConfiguration: { + Rules: [ + { + ApplyServerSideEncryptionByDefault: { + SSEAlgorithm: 'AES256', + } + } + ] + } + }).promise(); + await s3.getBucketEncryption({ Bucket: bucketName }).promise(); + }; + return { + method, + methodName: 'bucketGetEncryption', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + bucketOwner: null, + action: 'CreateBucket', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.ENCRYPTION', + action: 'PutBucketEncryption', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.GET.ENCRYPTION', + action: 'GetBucketEncryption', + httpMethod: 'GET', + } + ], + }; + })(), + (() => { + // This operation tests heading a bucket. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + await s3.headBucket({ Bucket: bucketName }).promise(); + }; + return { + method, + methodName: 'bucketHead', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + bucketOwner: null, + action: 'CreateBucket', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.HEAD.BUCKET', + action: 'HeadBucket', + httpMethod: 'HEAD', + } + ], + }; + })(), + (() => { + // This operation tests creating a bucket. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + }; + return { + method, + methodName: 'bucketPut', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + bucketOwner: null, + action: 'CreateBucket', + httpMethod: 'PUT', + } + ], + }; + })(), + (() => { + // This operation tests putting a bucket ACL. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + await s3.putBucketAcl({ Bucket: bucketName, ACL: 'private' }).promise(); + }; + return { + method, + methodName: 'bucketPutACL', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + bucketOwner: null, + action: 'CreateBucket', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.ACL', + action: 'PutBucketAcl', + httpMethod: 'PUT', + } + ], + }; + })(), + (() => { + // This operation tests putting a bucket CORS configuration. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + await s3.putBucketCors({ + Bucket: bucketName, + CORSConfiguration: { + CORSRules: [{ + AllowedHeaders: ['*'], + AllowedMethods: ['GET', 'PUT'], + AllowedOrigins: ['*'], + }] + } + }).promise(); + }; + return { + method, + methodName: 'bucketPutCors', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + bucketOwner: null, + action: 'CreateBucket', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.CORS', + action: 'PutBucketCors', + httpMethod: 'PUT', + } + ], + }; + })(), + (() => { + // This operation tests putting bucket versioning configuration. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + await s3.putBucketVersioning({ + Bucket: bucketName, VersioningConfiguration: { Status: 'Enabled' }, + }).promise(); + }; + return { + method, + methodName: 'bucketPutVersioning', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + bucketOwner: null, + action: 'CreateBucket', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.VERSIONING', + action: 'PutBucketVersioning', + httpMethod: 'PUT', + } + ], + }; + })(), + (() => { + // This operation tests putting bucket tagging. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + await s3.putBucketTagging({ + Bucket: bucketName, + Tagging: { + TagSet: [{ Key: 'testKey', Value: 'testValue' }] + } + }).promise(); + }; + return { + method, + methodName: 'bucketPutTagging', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + bucketOwner: null, + action: 'CreateBucket', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.TAGGING', + action: 'PutBucketTagging', + httpMethod: 'PUT', + } + ], + }; + })(), + (() => { + // This operation tests deleting bucket tagging. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + await s3.putBucketTagging({ + Bucket: bucketName, + Tagging: { + TagSet: [{ Key: 'testKey', Value: 'testValue' }] + } + }).promise(); + await s3.deleteBucketTagging({ Bucket: bucketName }).promise(); + }; + return { + method, + methodName: 'bucketDeleteTagging', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + bucketOwner: null, + action: 'CreateBucket', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.TAGGING', + action: 'PutBucketTagging', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.DELETE.TAGGING', + action: 'DeleteBucketTagging', + httpCode: 204, + httpMethod: 'DELETE', + } + ], + }; + })(), + (() => { + // This operation tests getting bucket tagging. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + await s3.putBucketTagging({ + Bucket: bucketName, + Tagging: { + TagSet: [{ Key: 'testKey', Value: 'testValue' }] + } + }).promise(); + await s3.getBucketTagging({ Bucket: bucketName }).promise(); + }; + return { + method, + methodName: 'bucketGetTagging', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + bucketOwner: null, + action: 'CreateBucket', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.TAGGING', + action: 'PutBucketTagging', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.GET.TAGGING', + action: 'GetBucketTagging', + httpMethod: 'GET', + } + ], + }; + })(), + (() => { + // This operation tests putting bucket replication. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + await s3.putBucketVersioning({ + Bucket: bucketName, VersioningConfiguration: { Status: 'Enabled' }, + }).promise(); + await s3.putBucketReplication({ + Bucket: bucketName, + ReplicationConfiguration: { + Role: 'arn:aws:iam::123456789012:role/src-role,arn:aws:iam::123456789012:role/dest-role', + Rules: [{ + ID: 'rule1', + Status: 'Enabled', + Priority: 1, + Filter: { Prefix: '' }, + Destination: { + Bucket: 'arn:aws:s3:::destination-bucket' + } + }] + } + }).promise(); + }; + return { + method, + methodName: 'bucketPutReplication', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + bucketOwner: null, + action: 'CreateBucket', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.VERSIONING', + action: 'PutBucketVersioning', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.REPLICATION', + action: 'PutBucketReplication', + httpMethod: 'PUT', + } + ], + }; + })(), + (() => { + // This operation tests getting bucket replication. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + await s3.putBucketVersioning({ + Bucket: bucketName, VersioningConfiguration: { Status: 'Enabled' }, + }).promise(); + await s3.putBucketReplication({ + Bucket: bucketName, + ReplicationConfiguration: { + Role: 'arn:aws:iam::123456789012:role/src-role,arn:aws:iam::123456789012:role/dest-role', + Rules: [{ + ID: 'rule1', + Status: 'Enabled', + Priority: 1, + Filter: { Prefix: '' }, + Destination: { + Bucket: 'arn:aws:s3:::destination-bucket' + } + }] + } + }).promise(); + await s3.getBucketReplication({ Bucket: bucketName }).promise(); + }; + return { + method, + methodName: 'bucketGetReplication', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + bucketOwner: null, + action: 'CreateBucket', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.VERSIONING', + action: 'PutBucketVersioning', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.REPLICATION', + action: 'PutBucketReplication', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.GET.REPLICATION', + action: 'GetBucketReplication', + httpMethod: 'GET', + } + ], + }; + })(), + (() => { + // This operation tests deleting bucket replication. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + await s3.putBucketVersioning({ + Bucket: bucketName, VersioningConfiguration: { Status: 'Enabled' }, + }).promise(); + await s3.putBucketReplication({ + Bucket: bucketName, + ReplicationConfiguration: { + Role: 'arn:aws:iam::123456789012:role/src-role,arn:aws:iam::123456789012:role/dest-role', + Rules: [{ + ID: 'rule1', + Status: 'Enabled', + Priority: 1, + Filter: { Prefix: '' }, + Destination: { + Bucket: 'arn:aws:s3:::destination-bucket' + } + }] + } + }).promise(); + await s3.deleteBucketReplication({ Bucket: bucketName }).promise(); + }; + return { + method, + methodName: 'bucketDeleteReplication', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + bucketOwner: null, + action: 'CreateBucket', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.VERSIONING', + action: 'PutBucketVersioning', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.REPLICATION', + action: 'PutBucketReplication', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.DELETE.REPLICATION', + action: 'DeleteBucketReplication', + httpCode: 204, + httpMethod: 'DELETE', + } + ], + }; + })(), + (() => { + // This operation tests putting bucket lifecycle. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + await s3.putBucketLifecycleConfiguration({ + Bucket: bucketName, + LifecycleConfiguration: { + Rules: [{ + ID: 'rule1', + Status: 'Enabled', + Filter: { Prefix: 'documents/' }, + Expiration: { Days: 365 } + }] + } + }).promise(); + }; + return { + method, + methodName: 'bucketPutLifecycle', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + bucketOwner: null, + action: 'CreateBucket', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.LIFECYCLE', + action: 'PutBucketLifecycleConfiguration', + httpMethod: 'PUT', + } + ], + }; + })(), + (() => { + // This operation tests getting bucket lifecycle. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + await s3.putBucketLifecycleConfiguration({ + Bucket: bucketName, + LifecycleConfiguration: { + Rules: [{ + ID: 'rule1', + Status: 'Enabled', + Filter: { Prefix: 'documents/' }, + Expiration: { Days: 365 } + }] + } + }).promise(); + await s3.getBucketLifecycleConfiguration({ Bucket: bucketName }).promise(); + }; + return { + method, + methodName: 'bucketGetLifecycle', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + bucketOwner: null, + action: 'CreateBucket', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.LIFECYCLE', + action: 'PutBucketLifecycleConfiguration', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.GET.LIFECYCLE', + action: 'GetBucketLifecycleConfiguration', + httpMethod: 'GET', + } + ], + }; + })(), + (() => { + // This operation tests deleting bucket lifecycle. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + await s3.putBucketLifecycleConfiguration({ + Bucket: bucketName, + LifecycleConfiguration: { + Rules: [{ + ID: 'rule1', + Status: 'Enabled', + Filter: { Prefix: 'documents/' }, + Expiration: { Days: 365 } + }] + } + }).promise(); + await s3.deleteBucketLifecycle({ Bucket: bucketName }).promise(); + }; + return { + method, + methodName: 'bucketDeleteLifecycle', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + bucketOwner: null, + action: 'CreateBucket', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.LIFECYCLE', + action: 'PutBucketLifecycleConfiguration', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.DELETE.LIFECYCLE', + action: 'DeleteBucketLifecycle', + httpCode: 204, + httpMethod: 'DELETE', + } + ], + }; + })(), + (() => { + // This operation tests putting bucket policy. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + await s3.putBucketPolicy({ + Bucket: bucketName, + Policy: JSON.stringify({ + Version: '2012-10-17', + Statement: [{ + Effect: 'Allow', + Principal: '*', + Action: 's3:GetObject', + Resource: `arn:aws:s3:::${bucketName}/*` + }] + }) + }).promise(); + }; + return { + method, + methodName: 'bucketPutPolicy', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + bucketOwner: null, + action: 'CreateBucket', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.BUCKETPOLICY', + action: 'PutBucketPolicy', + httpMethod: 'PUT', + } + ], + }; + })(), + (() => { + // This operation tests getting bucket policy. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + await s3.putBucketPolicy({ + Bucket: bucketName, + Policy: JSON.stringify({ + Version: '2012-10-17', + Statement: [{ + Effect: 'Allow', + Principal: '*', + Action: 's3:GetObject', + Resource: `arn:aws:s3:::${bucketName}/*` + }] + }) + }).promise(); + await s3.getBucketPolicy({ Bucket: bucketName }).promise(); + }; + return { + method, + methodName: 'bucketGetPolicy', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + bucketOwner: null, + action: 'CreateBucket', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.BUCKETPOLICY', + action: 'PutBucketPolicy', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.GET.BUCKETPOLICY', + action: 'GetBucketPolicy', + httpMethod: 'GET', + } + ], + }; + })(), + (() => { + // This operation tests deleting bucket policy. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + await s3.putBucketPolicy({ + Bucket: bucketName, + Policy: JSON.stringify({ + Version: '2012-10-17', + Statement: [{ + Effect: 'Allow', + Principal: '*', + Action: 's3:GetObject', + Resource: `arn:aws:s3:::${bucketName}/*` + }] + }) + }).promise(); + await s3.deleteBucketPolicy({ Bucket: bucketName }).promise(); + }; + return { + method, + methodName: 'bucketDeletePolicy', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + bucketOwner: null, + action: 'CreateBucket', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.BUCKETPOLICY', + action: 'PutBucketPolicy', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.DELETE.BUCKETPOLICY', + action: 'DeleteBucketPolicy', + httpCode: 204, + httpMethod: 'DELETE', + } + ], + }; + })(), + (() => { + // This operation tests putting bucket object lock configuration. + const method = async () => { + await s3.createBucket({ + Bucket: bucketName, + ObjectLockEnabledForBucket: true, + }).promise(); + await s3.putObjectLockConfiguration({ + Bucket: bucketName, + ObjectLockConfiguration: { + ObjectLockEnabled: 'Enabled', + Rule: { + DefaultRetention: { + Mode: 'GOVERNANCE', + Days: 1 + } + } + } + }).promise(); + }; + return { + method, + methodName: 'bucketPutObjectLock', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + bucketOwner: null, + action: 'CreateBucket', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.OBJECT', + action: 'PutObjectLockConfiguration', + httpMethod: 'PUT', + } + ], + }; + })(), + (() => { + // This operation tests putting bucket notification configuration. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + await s3.putBucketNotificationConfiguration({ + Bucket: bucketName, + NotificationConfiguration: {} + }).promise(); + }; + return { + method, + methodName: 'bucketPutNotification', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + bucketOwner: null, + action: 'CreateBucket', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.NOTIFICATION', + action: 'PutBucketNotification', + httpMethod: 'PUT', + } + ], + }; + })(), + (() => { + // This operation tests getting bucket notification configuration. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + await s3.getBucketNotificationConfiguration({ Bucket: bucketName }).promise(); + }; + return { + method, + methodName: 'bucketGetNotification', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + bucketOwner: null, + action: 'CreateBucket', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.GET.NOTIFICATION', + action: 'GetBucketNotification', + httpMethod: 'GET', + } + ], + }; + })(), + (() => { + // This operation tests putting bucket encryption. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + await s3.putBucketEncryption({ + Bucket: bucketName, + ServerSideEncryptionConfiguration: { + Rules: [ + { + ApplyServerSideEncryptionByDefault: { + SSEAlgorithm: 'AES256', + } + } + ] + } + }).promise(); + }; + return { + method, + methodName: 'bucketPutEncryption', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + bucketOwner: null, + action: 'CreateBucket', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.ENCRYPTION', + action: 'PutBucketEncryption', + httpMethod: 'PUT', + } + ], + }; + })(), + (() => { + // This operation tests putting bucket logging. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + await s3.putBucketLogging({ + Bucket: bucketName, + BucketLoggingStatus: {} + }).promise(); + }; + return { + method, + methodName: 'bucketPutLogging', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + bucketOwner: null, + action: 'CreateBucket', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.LOGGING_STATUS', + action: 'PutBucketLogging', + httpMethod: 'PUT', + } + ], + }; + })(), + (() => { + // This operation tests getting bucket logging. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + await s3.putBucketLogging({ + Bucket: bucketName, + BucketLoggingStatus: { + LoggingEnabled: { TargetBucket: bucketName, TargetPrefix: 'prefix' }, + }, + }).promise(); + await s3.getBucketLogging({ Bucket: bucketName }).promise(); + await s3.putBucketLogging({ + Bucket: bucketName, + BucketLoggingStatus: {} + }).promise(); + }; + return { + method, + methodName: 'bucketGetLogging', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + bucketOwner: null, + action: 'CreateBucket', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.LOGGING_STATUS', + action: 'PutBucketLogging', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.GET.LOGGING_STATUS', + action: 'GetBucketLogging', + loggingEnabled: true, + loggingTargetBucket: bucketName, + loggingTargetPrefix: 'prefix', + httpMethod: 'GET', + }, + { + ...commonProperties, + operation: 'REST.PUT.LOGGING_STATUS', + action: 'PutBucketLogging', + loggingEnabled: true, + loggingTargetBucket: bucketName, + loggingTargetPrefix: 'prefix', + httpMethod: 'PUT', + } + ], + }; + })(), + (() => { + // This operation tests completing a multipart upload. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + const uploadId = + (await s3.createMultipartUpload({ Bucket: bucketName, Key: objectKey }).promise()).UploadId; + const uploadPartResponse = await s3.uploadPart({ + Bucket: bucketName, + Key: objectKey, + PartNumber: 1, + UploadId: uploadId, + Body: 'test data' + }).promise(); + await s3.completeMultipartUpload({ + Bucket: bucketName, + Key: objectKey, + UploadId: uploadId, + MultipartUpload: { + Parts: [{ + ETag: uploadPartResponse.ETag, + PartNumber: 1 + }] + } + }).promise(); + }; + return { + method, + methodName: 'completeMultipartUpload', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + bucketOwner: null, + action: 'CreateBucket', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.POST.UPLOADS', + action: 'CreateMultipartUpload', + objectKey, + httpMethod: 'POST', + }, + { + ...commonProperties, + operation: 'REST.PUT.PART', + action: 'UploadPart', + bucketOwner: null, + objectKey, + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.POST.UPLOAD', + action: 'CompleteMultipartUpload', + objectKey, + httpMethod: 'POST', + } + ], + }; + })(), + (() => { + // This operation tests initiating a multipart upload. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + await s3.createMultipartUpload({ Bucket: bucketName, Key: objectKey }).promise(); + }; + return { + method, + methodName: 'initiateMultipartUpload', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + bucketOwner: null, + action: 'CreateBucket', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.POST.UPLOADS', + action: 'CreateMultipartUpload', + objectKey, + httpMethod: 'POST', + } + ], + }; + })(), + (() => { + // This operation tests listing multipart uploads. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + await s3.createMultipartUpload({ Bucket: bucketName, Key: objectKey }).promise(); + await s3.listMultipartUploads({ Bucket: bucketName }).promise(); + }; + return { + method, + methodName: 'listMultipartUploads', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + bucketOwner: null, + action: 'CreateBucket', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.POST.UPLOADS', + action: 'CreateMultipartUpload', + objectKey, + httpMethod: 'POST', + }, + { + ...commonProperties, + operation: 'REST.GET.UPLOADS', + action: 'ListMultipartUploads', + httpMethod: 'GET', + } + ], + }; + })(), + (() => { + // This operation tests listing parts of a multipart upload. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + const uploadId = + (await s3.createMultipartUpload({ Bucket: bucketName, Key: objectKey }).promise()).UploadId; + await s3.uploadPart({ + Bucket: bucketName, + Key: objectKey, + PartNumber: 1, + UploadId: uploadId, + Body: 'test data' + }).promise(); + await s3.listParts({ Bucket: bucketName, Key: objectKey, UploadId: uploadId }).promise(); + }; + return { + method, + methodName: 'listParts', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + bucketOwner: null, + action: 'CreateBucket', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.POST.UPLOADS', + action: 'CreateMultipartUpload', + objectKey, + httpMethod: 'POST', + }, + { + ...commonProperties, + operation: 'REST.PUT.PART', + action: 'UploadPart', + bucketOwner: null, + objectKey, + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.GET.UPLOAD', + action: 'ListParts', + objectKey, + httpMethod: 'GET', + } + ], + }; + })(), + (() => { + // This operation tests deleting multiple objects. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + await s3.putObject({ Bucket: bucketName, Key: objectKey, Body: 'test data' }).promise(); + await s3.putObject({ Bucket: bucketName, Key: `${objectKey}2`, Body: 'test data 2' }).promise(); + await s3.deleteObjects({ + Bucket: bucketName, + Delete: { + Objects: [ + { Key: objectKey }, + { Key: `${objectKey}2` } + ] + } + }).promise(); + }; + return { + method, + methodName: 'multiObjectDelete', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + bucketOwner: null, + action: 'CreateBucket', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.OBJECT', + action: 'PutObject', + objectKey, + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.OBJECT', + action: 'PutObject', + objectKey: `${objectKey}2`, + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.POST.OBJECT', + action: 'DeleteObjects', + httpMethod: 'POST', + } + ], + }; + })(), + (() => { + // This operation tests aborting a multipart upload. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + const uploadId = + (await s3.createMultipartUpload({ Bucket: bucketName, Key: objectKey }).promise()).UploadId; + await s3.abortMultipartUpload({ Bucket: bucketName, Key: objectKey, UploadId: uploadId }).promise(); + }; + return { + method, + methodName: 'multipartDelete', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + bucketOwner: null, + action: 'CreateBucket', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.POST.UPLOADS', + action: 'CreateMultipartUpload', + objectKey, + httpMethod: 'POST', + }, + { + ...commonProperties, + operation: 'REST.DELETE.UPLOAD', + action: 'AbortMultipartUpload', + httpCode: 204, + objectKey, + httpMethod: 'DELETE', + } + ], + }; + })(), + (() => { + // This operation tests deleting an object. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + await s3.putObject({ Bucket: bucketName, Key: objectKey, Body: 'test data' }).promise(); + await s3.deleteObject({ Bucket: bucketName, Key: objectKey }).promise(); + }; + return { + method, + methodName: 'objectDelete', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + action: 'CreateBucket', + bucketOwner: null, + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.OBJECT', + action: 'PutObject', + objectKey, + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.DELETE.OBJECT', + action: 'DeleteObject', + httpCode: 204, + objectKey, + httpMethod: 'DELETE', + } + ], + }; + })(), + (() => { + // This operation tests deleting object tagging. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + await s3.putObject({ Bucket: bucketName, Key: objectKey, Body: 'test data' }).promise(); + await s3.putObjectTagging({ + Bucket: bucketName, + Key: objectKey, + Tagging: { + TagSet: [{ Key: 'testKey', Value: 'testValue' }] + } + }).promise(); + await s3.deleteObjectTagging({ Bucket: bucketName, Key: objectKey }).promise(); + }; + return { + method, + methodName: 'objectDeleteTagging', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + action: 'CreateBucket', + bucketOwner: null, + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.OBJECT', + action: 'PutObject', + objectKey, + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.TAGGING', + action: 'PutObjectTagging', + objectKey, + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.DELETE.TAGGING', + action: 'DeleteObjectTagging', + httpCode: 204, + objectKey, + httpMethod: 'DELETE', + } + ], + }; + })(), + (() => { + // This operation tests getting an object. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + await s3.putObject({ Bucket: bucketName, Key: objectKey, Body: 'test data' }).promise(); + await s3.getObject({ Bucket: bucketName, Key: objectKey }).promise(); + }; + return { + method, + methodName: 'objectGet', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + action: 'CreateBucket', + bucketOwner: null, + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.OBJECT', + action: 'PutObject', + objectKey, + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.GET.OBJECT', + action: 'GetObject', + objectKey, + httpMethod: 'GET', + } + ], + }; + })(), + (() => { + // This operation tests getting object ACL. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + await s3.putObject({ Bucket: bucketName, Key: objectKey, Body: 'test data' }).promise(); + await s3.getObjectAcl({ Bucket: bucketName, Key: objectKey }).promise(); + }; + return { + method, + methodName: 'objectGetACL', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + action: 'CreateBucket', + bucketOwner: null, + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.OBJECT', + action: 'PutObject', + objectKey, + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.GET.ACL', + action: 'GetObjectAcl', + objectKey, + httpMethod: 'GET', + } + ], + }; + })(), + (() => { + // This operation tests getting object legal hold. + const method = async () => { + await s3.createBucket({ + Bucket: bucketName, + ObjectLockEnabledForBucket: true, + }).promise(); + await s3.putObject({ Bucket: bucketName, Key: objectKey, Body: 'test data' }).promise(); + await s3.putObjectLegalHold({ + Bucket: bucketName, + Key: objectKey, + LegalHold: { Status: 'ON' } + }).promise(); + await s3.getObjectLegalHold({ Bucket: bucketName, Key: objectKey }).promise(); + await s3.putObjectLegalHold({ + Bucket: bucketName, + Key: objectKey, + LegalHold: { Status: 'OFF' } + }).promise(); + }; + return { + method, + methodName: 'objectGetLegalHold', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + action: 'CreateBucket', + bucketOwner: null, + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.OBJECT', + action: 'PutObject', + objectKey, + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.LEGALHOLD', + action: 'PutObjectLegalHold', + objectKey, + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.GET.LEGALHOLD', + action: 'GetObjectLegalHold', + objectKey, + httpMethod: 'GET', + }, + { + ...commonProperties, + operation: 'REST.PUT.LEGALHOLD', + action: 'PutObjectLegalHold', + objectKey, + httpMethod: 'PUT', + } + ], + }; + })(), + (() => { + // This operation tests getting object retention. + const method = async () => { + await s3.createBucket({ + Bucket: bucketName, + ObjectLockEnabledForBucket: true, + }).promise(); + await s3.putObject({ Bucket: bucketName, Key: objectKey, Body: 'test data' }).promise(); + const retainUntilDate = new Date(); + retainUntilDate.setDate(retainUntilDate.getDate() + 1); + await s3.putObjectRetention({ + Bucket: bucketName, + Key: objectKey, + Retention: { + Mode: 'GOVERNANCE', + RetainUntilDate: retainUntilDate + } + }).promise(); + await s3.getObjectRetention({ Bucket: bucketName, Key: objectKey }).promise(); + }; + return { + method, + methodName: 'objectGetRetention', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + action: 'CreateBucket', + bucketOwner: null, + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.OBJECT', + action: 'PutObject', + objectKey, + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.OBJECT_LOCK_RETENTION', + action: 'PutObjectRetention', + objectKey, + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.GET.OBJECT_LOCK_RETENTION', + action: 'GetObjectRetention', + objectKey, + httpMethod: 'GET', + } + ], + }; + })(), + (() => { + // This operation tests getting object tagging. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + await s3.putObject({ Bucket: bucketName, Key: objectKey, Body: 'test data' }).promise(); + await s3.getObjectTagging({ Bucket: bucketName, Key: objectKey }).promise(); + }; + return { + method, + methodName: 'objectGetTagging', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + action: 'CreateBucket', + bucketOwner: null, + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.OBJECT', + action: 'PutObject', + objectKey, + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.GET.TAGGING', + action: 'GetObjectTagging', + objectKey, + httpMethod: 'GET', + } + ], + }; + })(), + (() => { + // This operation tests copying an object. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + await s3.putObject({ Bucket: bucketName, Key: objectKey, Body: 'test data' }).promise(); + await s3.copyObject({ + Bucket: bucketName, + CopySource: `${bucketName}/${objectKey}`, + Key: `${objectKey}-copy` + }).promise(); + }; + return { + method, + methodName: 'objectCopy', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + action: 'CreateBucket', + bucketOwner: null, + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.OBJECT', + action: 'PutObject', + objectKey, + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.COPY', + action: 'CopyObject', + objectKey: `${objectKey}-copy`, + httpMethod: 'PUT', + } + ], + }; + })(), + (() => { + // This operation tests putting an object ACL. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + await s3.putObject({ Bucket: bucketName, Key: objectKey, Body: 'test data' }).promise(); + await s3.putObjectAcl({ Bucket: bucketName, Key: objectKey, ACL: 'private' }).promise(); + }; + return { + method, + methodName: 'objectPutACL', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + action: 'CreateBucket', + bucketOwner: null, + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.OBJECT', + action: 'PutObject', + objectKey, + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.ACL', + action: 'PutObjectAcl', + objectKey, + httpMethod: 'PUT', + } + ], + }; + })(), + (() => { + // This operation tests putting object legal hold. + const method = async () => { + await s3.createBucket({ + Bucket: bucketName, + ObjectLockEnabledForBucket: true, + }).promise(); + await s3.putObject({ Bucket: bucketName, Key: objectKey, Body: 'test data' }).promise(); + await s3.putObjectLegalHold({ + Bucket: bucketName, + Key: objectKey, + LegalHold: { Status: 'ON' } + }).promise(); + await s3.putObjectLegalHold({ + Bucket: bucketName, + Key: objectKey, + LegalHold: { Status: 'OFF' } + }).promise(); + }; + return { + method, + methodName: 'objectPutLegalHold', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + action: 'CreateBucket', + bucketOwner: null, + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.OBJECT', + action: 'PutObject', + objectKey, + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.LEGALHOLD', + action: 'PutObjectLegalHold', + objectKey, + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.LEGALHOLD', + action: 'PutObjectLegalHold', + objectKey, + httpMethod: 'PUT', + } + ], + }; + })(), + (() => { + // This operation tests putting object tagging. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + await s3.putObject({ Bucket: bucketName, Key: objectKey, Body: 'test data' }).promise(); + await s3.putObjectTagging({ + Bucket: bucketName, + Key: objectKey, + Tagging: { + TagSet: [{ Key: 'testKey', Value: 'testValue' }] + } + }).promise(); + }; + return { + method, + methodName: 'objectPutTagging', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + action: 'CreateBucket', + bucketOwner: null, + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.OBJECT', + action: 'PutObject', + objectKey, + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.TAGGING', + action: 'PutObjectTagging', + objectKey, + httpMethod: 'PUT', + } + ], + }; + })(), + (() => { + // This operation tests uploading a part in a multipart upload. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + const uploadId = + (await s3.createMultipartUpload({ Bucket: bucketName, Key: objectKey }).promise()).UploadId; + await s3.uploadPart({ + Bucket: bucketName, + Key: objectKey, + PartNumber: 1, + UploadId: uploadId, + Body: 'test data' + }).promise(); + }; + return { + method, + methodName: 'objectPutPart', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + action: 'CreateBucket', + bucketOwner: null, + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.POST.UPLOADS', + action: 'CreateMultipartUpload', + objectKey, + httpMethod: 'POST', + }, + { + ...commonProperties, + operation: 'REST.PUT.PART', + action: 'UploadPart', + bucketOwner: null, + objectKey, + httpMethod: 'PUT', + } + ], + }; + })(), + (() => { + // This operation tests uploading a part copy in a multipart upload. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + await s3.putObject({ Bucket: bucketName, Key: objectKey, Body: 'test data for copy' }).promise(); + const uploadId = + (await s3.createMultipartUpload({ Bucket: bucketName, Key: `${objectKey}-mpu` }).promise()) + .UploadId; + await s3.uploadPartCopy({ + Bucket: bucketName, + Key: `${objectKey}-mpu`, + PartNumber: 1, + UploadId: uploadId, + CopySource: `${bucketName}/${objectKey}` + }).promise(); + }; + return { + method, + methodName: 'objectPutCopyPart', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + action: 'CreateBucket', + bucketOwner: null, + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.OBJECT', + action: 'PutObject', + objectKey, + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.POST.UPLOADS', + action: 'CreateMultipartUpload', + objectKey: `${objectKey}-mpu`, + httpMethod: 'POST', + }, + { + ...commonProperties, + operation: 'REST.PUT.COPY', + action: 'UploadPartCopy', + objectKey: `${objectKey}-mpu`, + httpMethod: 'PUT', + } + ], + }; + })(), + (() => { + // This operation tests putting object retention. + const method = async () => { + await s3.createBucket({ + Bucket: bucketName, + ObjectLockEnabledForBucket: true, + }).promise(); + await s3.putObject({ Bucket: bucketName, Key: objectKey, Body: 'test data' }).promise(); + const retainUntilDate = new Date(); + retainUntilDate.setDate(retainUntilDate.getDate() + 1); + await s3.putObjectRetention({ + Bucket: bucketName, + Key: objectKey, + Retention: { + Mode: 'GOVERNANCE', + RetainUntilDate: retainUntilDate + } + }).promise(); + }; + return { + method, + methodName: 'objectPutRetention', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + bucketOwner: null, + action: 'CreateBucket', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.OBJECT', + action: 'PutObject', + objectKey, + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.OBJECT_LOCK_RETENTION', + action: 'PutObjectRetention', + objectKey, + httpMethod: 'PUT', + } + ], + }; + })(), + // Note: objectRestore can only be called on objects in GLACIER, DEEP_ARCHIVE, or + // GLACIER_IR storage classes. Since CloudServer only supports STANDARD storage class + // by default, this operation returns "InvalidObjectState" error and cannot be tested. + // This test is commented out until archive storage class support is added. + // (() => { + // // This operation tests the restore object API call. + // const method = async () => { + // await s3.createBucket({ Bucket: bucketName }).promise(); + // await s3.putObject({ + // Bucket: bucketName, + // Key: objectKey, + // Body: 'test data', + // StorageClass: 'GLACIER' // Not supported in CloudServer + // }).promise(); + // await s3.restoreObject({ + // Bucket: bucketName, + // Key: objectKey, + // RestoreRequest: { + // Days: 1 + // } + // }).promise(); + // }; + // const expectedOperations = ['REST.PUT.BUCKET','REST.PUT.OBJECT', 'REST.POST.OBJECT']; + // return { method, methodName: 'objectRestore', expectedOperations }; + // })(), + (() => { + const testBody = 'Hello, Server Access Logs!'; + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + await s3.putObject({ Bucket: bucketName, Key: objectKey, Body: testBody }).promise(); + }; + return { + method, + methodName: 'putObject', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + bucketOwner: null, + action: 'CreateBucket', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.OBJECT', + action: 'PutObject', + objectKey, + httpMethod: 'PUT', + } + ], + }; + })(), + (() => { + const testBody = 'Hello, Server Access Logs!'; + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + await s3.putObject({ Bucket: bucketName, Key: objectKey, Body: testBody }).promise(); + await s3.headObject({ Bucket: bucketName, Key: objectKey }).promise(); + }; + return { + method, + methodName: 'headObject', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + bucketOwner: null, + action: 'CreateBucket', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.PUT.OBJECT', + action: 'PutObject', + objectKey, + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.HEAD.OBJECT', + action: 'HeadObject', + objectKey, + httpMethod: 'HEAD', + } + ], + }; + })(), + (() => { + // This operation tests listing all buckets. + const method = async () => { + await s3.createBucket({ Bucket: bucketName }).promise(); + await s3.listBuckets().promise(); + }; + return { + method, + methodName: 'listBuckets', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.BUCKET', + bucketOwner: null, + action: 'CreateBucket', + httpMethod: 'PUT', + }, + { + ...commonProperties, + operation: 'REST.GET.SERVICE', + action: 'ListBuckets', + bucketOwner: null, + bucketName: null, + httpMethod: 'GET', + } + ], + }; + })(), + (() => { + // Test errorCode is set. + const method = async () => { + try { + await s3.deleteBucket({ Bucket: 'xxx'}).promise(); + } catch { + return; + } + }; + return { + method, + methodName: 'bucketDeleteError', + expected: [ + { + ...commonProperties, + operation: 'REST.DELETE.BUCKET', + action: 'DeleteBucket', + httpCode: 404, + errorCode: 'NoSuchBucket', + bucketOwner: null, + bucketName: 'xxx', + httpMethod: 'DELETE', + } + ], + }; + })(), + (() => { + // Test errorCode is set for PutObject. + const method = async () => { + try { + await s3.putObject({ Bucket: 'xxx', Key: 'key', Body: 'test' }).promise(); + } catch { + return; + } + }; + return { + method, + methodName: 'putObjectError', + expected: [ + { + ...commonProperties, + operation: 'REST.PUT.OBJECT', + action: 'PutObject', + httpCode: 404, + errorCode: 'NoSuchBucket', + bucketOwner: null, + bucketName: 'xxx', + httpMethod: 'PUT', + objectKey: 'key', + } + ], + }; + })(), + (() => { + // Test errorCode is set for GetObject. + const method = async () => { + try { + await s3.getObject({ Bucket: 'xxx', Key: 'key' }).promise(); + } catch { + return; + } + }; + return { + method, + methodName: 'getObjectError', + expected: [ + { + ...commonProperties, + operation: 'REST.GET.OBJECT', + action: 'GetObject', + httpCode: 404, + errorCode: 'NoSuchBucket', + bucketOwner: null, + bucketName: 'xxx', + httpMethod: 'GET', + objectKey: 'key', + } + ], + }; + })() + // TODO: CLDSRV-799 + // (() => { + // // Test errorCode is set. + // const method = async () => { + // try { + // await s3.deleteBucket({ Bucket: 'UPPERCASE'}).promise(); + // } catch { + // return; + // } + // }; + // return { + // method, + // methodName: 'bucketDeleteError', + // expected: [ + // { + // ...commonProperties, + // operation: 'REST.DELETE.BUCKET', + // action: 'DeleteBucket', + // httpCode: 404, + // errorCode: 'NoSuchBucket', + // bucketOwner: null, + // bucketName: 'xxx', + // httpMethod: 'DELETE', + // } + // ], + // }; + // })() + ]; + + before(async function () { + if (config.serverAccessLogs.mode === serverAccessLogsModes.DISABLED) { + this.skip(); + } + if (process.env.AWS_ON_AIR) { + this.skip(); + } + if (!fs.existsSync(path.dirname(logFilePath))) { + throw new Error('Logs directory does not exist'); + } + }); + + after(async () => { + truncateLogFileIfExists(logFilePath); + }); + + beforeEach(async () => { + truncateLogFileIfExists(logFilePath); + }); + + afterEach(async () => { + const lastAction = await cleanupBuckets(s3, bucketName); + await waitForAction(logFilePath, lastAction, + TEST_CONFIG.MAX_LOG_WAIT_RETRIES, TEST_CONFIG.LOG_POLL_DELAY_MS); + truncateLogFileIfExists(logFilePath); + }); + + for (const operation of operations) { + it(`should log correct ${operation.methodName} operation with all required fields`, async () => { + await operation.method(); + const logEntries = await waitForLogs(logFilePath, operation.expected.length, + TEST_CONFIG.MAX_LOG_WAIT_RETRIES, TEST_CONFIG.LOG_POLL_DELAY_MS); + assert.strictEqual(logEntries.length, operation.expected.length, + `Expected ${operation.expected.length} log entries, got ${logEntries.length}`); + + for (let logEntryIdx = 0, operationIdx = 0; + operationIdx < operation.expected.length; + logEntryIdx++, operationIdx++) { + const result = tv4.validateResult(logEntries[logEntryIdx], schema); + assert.strictEqual(result.valid, true, + `Log entry should match schema: ${JSON.stringify(result.error)}`); + + const properties = operation.expected[operationIdx]; + for (const [key, val] of Object.entries(properties)) { + assert.strictEqual(logEntries[logEntryIdx][key], val, + `Invalid value for ${key}, action ${properties.action}`); + } + } + }); + } + }); +}); diff --git a/tests/unit/utils/serverAccessLogger.js b/tests/unit/utils/serverAccessLogger.js index 3ac930669d..e2895101f7 100644 --- a/tests/unit/utils/serverAccessLogger.js +++ b/tests/unit/utils/serverAccessLogger.js @@ -180,10 +180,10 @@ describe('serverAccessLogger utility functions', () => { assert.strictEqual(result, 'REST.GET.LOGGING_STATUS'); }); - it('should return REST.POST.OBJECT for completeMultipartUpload', () => { + it('should return REST.POST.UPLOAD for completeMultipartUpload', () => { const req = { method: 'POST', apiMethod: 'completeMultipartUpload' }; const result = getOperation(req); - assert.strictEqual(result, 'REST.POST.OBJECT'); + assert.strictEqual(result, 'REST.POST.UPLOAD'); }); it('should return REST.method.UNKNOWN for unknown apiMethod', () => { @@ -328,7 +328,7 @@ describe('serverAccessLogger utility functions', () => { getHeader: name => name === 'Content-Length' ? '12345' : null, }; const result = getObjectSize(request, response); - assert.strictEqual(result, '12345'); + assert.strictEqual(result, 12345); }); it('should return Content-Length from request for objectPut', () => { @@ -340,7 +340,7 @@ describe('serverAccessLogger utility functions', () => { getHeader: () => null, }; const result = getObjectSize(request, response); - assert.strictEqual(result, '54321'); + assert.strictEqual(result, 54321); }); it('should return Content-Length from request for objectPutPart', () => { @@ -352,7 +352,7 @@ describe('serverAccessLogger utility functions', () => { getHeader: () => null, }; const result = getObjectSize(request, response); - assert.strictEqual(result, '67890'); + assert.strictEqual(result, 67890); }); it('should handle Content-Length of 0 for objectGet', () => { @@ -361,7 +361,7 @@ describe('serverAccessLogger utility functions', () => { getHeader: name => name === 'Content-Length' ? '0' : null, }; const result = getObjectSize(request, response); - assert.strictEqual(result, '0'); + assert.strictEqual(result, 0); }); it('should handle Content-Length of number 0 for objectGet', () => { @@ -382,7 +382,7 @@ describe('serverAccessLogger utility functions', () => { getHeader: () => null, }; const result = getObjectSize(request, response); - assert.strictEqual(result, '0'); + assert.strictEqual(result, 0); }); it('should handle Content-Length of number 0 for objectPut', () => { @@ -821,7 +821,7 @@ describe('serverAccessLogger utility functions', () => { assert.strictEqual(loggedData.operation, 'REST.GET.OBJECT'); assert.strictEqual(loggedData.requestURI, 'GET /test-bucket/test-key.txt HTTP/1.1'); assert.strictEqual(loggedData.errorCode, null); - assert.strictEqual(loggedData.objectSize, '2048'); + assert.strictEqual(loggedData.objectSize, 2048); assert.strictEqual(loggedData.totalTime, '9'); assert.strictEqual(loggedData.turnAroundTime, '1'); assert.strictEqual(loggedData.referer, 'https://example.com'); @@ -992,7 +992,7 @@ describe('serverAccessLogger utility functions', () => { assert.strictEqual(mockLogger.write.callCount, 1); const loggedData = JSON.parse(mockLogger.write.firstCall.args[0].trim()); assert.strictEqual(loggedData.operation, 'REST.PUT.OBJECT'); - assert.strictEqual(loggedData.objectSize, '5000'); + assert.strictEqual(loggedData.objectSize, 5000); assert.strictEqual(loggedData.bytesReceived, 5000); assert.strictEqual(loggedData.totalTime, '2'); }); diff --git a/yarn.lock b/yarn.lock index 0410f62c14..1e5869e684 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1495,9 +1495,9 @@ arraybuffer.prototype.slice@^1.0.4: optionalDependencies: ioctl "^2.0.2" -"arsenal@git+https://github.com/scality/Arsenal#8.2.41": - version "8.2.41" - resolved "git+https://github.com/scality/Arsenal#0914598961fa2ca352a336a26b05b82ddc481749" +"arsenal@git+https://github.com/scality/Arsenal#8.2.42": + version "8.2.42" + resolved "git+https://github.com/scality/Arsenal#e73ddc7efdb046389c4d2f533189530c7f5af452" dependencies: "@azure/identity" "^4.13.0" "@azure/storage-blob" "^12.28.0"