From 4f1ca9ab71e6e4fc12e5f19c8320efa88bcf8b89 Mon Sep 17 00:00:00 2001 From: Electra Chong Date: Tue, 17 Oct 2017 12:18:39 -0700 Subject: [PATCH] ft: enable aws backend object copy w/ versioning --- lib/api/objectCopy.js | 52 +-- lib/data/external/AwsClient.js | 10 +- lib/data/multipleBackendGateway.js | 4 +- lib/data/wrapper.js | 12 +- .../multipleBackend/objectCopy/objectCopy.js | 28 -- .../objectCopy/objectCopyAwsVersioning.js | 419 ++++++++++++++++++ .../test/multipleBackend/utils.js | 3 +- .../test/versioning/objectCopy.js | 9 +- tests/multipleBackend/objectCopy.js | 107 +++++ 9 files changed, 576 insertions(+), 68 deletions(-) create mode 100644 tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/objectCopyAwsVersioning.js create mode 100644 tests/multipleBackend/objectCopy.js diff --git a/lib/api/objectCopy.js b/lib/api/objectCopy.js index f48f8107db..5132be214b 100644 --- a/lib/api/objectCopy.js +++ b/lib/api/objectCopy.js @@ -23,6 +23,7 @@ const { config } = require('../Config'); const versionIdUtils = versioning.VersionID; const locationHeader = constants.objectLocationConstraintHeader; +const versioningNotImplBackends = constants.versioningNotImplBackends; const externalVersioningErrorMessage = 'We do not currently support putting ' + 'a versioned object to a location-constraint of type AWS or Azure.'; @@ -150,6 +151,7 @@ function _prepMetadata(request, sourceObjMD, headers, sourceIsDestination, contentDisposition: headersToStoreSource['content-disposition'], contentEncoding: removeAWSChunked(headersToStoreSource['content-encoding']), + dataStoreName: destLocationConstraintName, expires: headersToStoreSource.expires, overrideMetadata, lastModifiedDate: new Date().toJSON(), @@ -166,7 +168,7 @@ function _prepMetadata(request, sourceObjMD, headers, sourceIsDestination, storeMetadataParams.contentType = sourceObjMD['content-type']; } return { storeMetadataParams, sourceLocationConstraintName, - backendInfoObjDest }; + backendInfoDest: backendInfoObjDest.backendInfo }; } /** @@ -278,7 +280,7 @@ function objectCopy(authInfo, request, sourceBucket, return next(errors.PreconditionFailed, destBucketMD); } const { storeMetadataParams, error: metadataError, - sourceLocationConstraintName, backendInfoObjDest } = + sourceLocationConstraintName, backendInfoDest } = _prepMetadata(request, sourceObjMD, request.headers, sourceIsDestination, authInfo, destObjectKey, sourceBucketMD, destBucketMD, log); @@ -311,20 +313,23 @@ function objectCopy(authInfo, request, sourceBucket, } return next(null, storeMetadataParams, dataLocator, destBucketMD, destObjMD, sourceLocationConstraintName, - backendInfoObjDest); + backendInfoDest); }); }, function goGetData(storeMetadataParams, dataLocator, destBucketMD, - destObjMD, sourceLocationConstraintName, backendInfoObjDest, next) { + destObjMD, sourceLocationConstraintName, backendInfoDest, next) { const serverSideEncryption = destBucketMD.getServerSideEncryption(); + const vcfg = destBucketMD.getVersioningConfiguration(); + const isVersionedObj = vcfg && vcfg.Status === 'Enabled'; + const destLocationConstraintName = + storeMetadataParams.dataStoreName; - const backendInfoDest = backendInfoObjDest.backendInfo; - const destLocationConstraintName = backendInfoObjDest.controllingLC; - - // skip if source and dest and location constraint the same + // skip if source and dest and location constraint the same and + // versioning is not enabled // still send along serverSideEncryption info so algo // and masterKeyId stored properly in metadata - if (sourceIsDestination && storeMetadataParams.locationMatch) { + if (sourceIsDestination && storeMetadataParams.locationMatch + && !isVersionedObj) { return next(null, storeMetadataParams, dataLocator, destObjMD, serverSideEncryption, destBucketMD); } @@ -334,18 +339,13 @@ function objectCopy(authInfo, request, sourceBucket, // metadata to backend const destLocationConstraintType = config.getLocationConstraintType(destLocationConstraintName); - // NOTE: remove the following when we will support putting a - // versioned object to a location-constraint of type AWS or Azure. - if (constants.externalBackends[destLocationConstraintType]) { - const vcfg = destBucketMD.getVersioningConfiguration(); - const isVersionedObj = vcfg && vcfg.Status === 'Enabled'; - if (isVersionedObj) { - log.debug(externalVersioningErrorMessage, - { method: 'multipleBackendGateway', - error: errors.NotImplemented }); - return next(errors.NotImplemented.customizeDescription( - externalVersioningErrorMessage), destBucketMD); - } + if (versioningNotImplBackends[destLocationConstraintType] + && isVersionedObj) { + log.debug(externalVersioningErrorMessage, + { method: 'multipleBackendGateway', + error: errors.NotImplemented }); + return next(errors.NotImplemented.customizeDescription( + externalVersioningErrorMessage), destBucketMD); } if (dataLocator.length === 0) { if (!storeMetadataParams.locationMatch && @@ -372,7 +372,6 @@ function objectCopy(authInfo, request, sourceBucket, return next(null, storeMetadataParams, dataLocator, destObjMD, serverSideEncryption, destBucketMD); } - return data.copyObject(request, sourceLocationConstraintName, storeMetadataParams, dataLocator, dataStoreContext, backendInfoDest, serverSideEncryption, log, @@ -422,9 +421,12 @@ function objectCopy(authInfo, request, sourceBucket, // object with same name, so long as the source is not // the same as the destination if (!sourceIsDestination && dataToDelete) { - data.batchDelete(dataToDelete, request.method, null, - logger.newRequestLoggerFromSerializedUids( - log.getSerializedUids())); + const newDataStoreName = + storeMetadataParams.dataStoreName; + data.batchDelete(dataToDelete, request.method, + newDataStoreName, + logger.newRequestLoggerFromSerializedUids( + log.getSerializedUids())); } const sourceObjSize = storeMetadataParams.size; const destObjPrevSize = (destObjMD && diff --git a/lib/data/external/AwsClient.js b/lib/data/external/AwsClient.js index 57e83a4811..31573460e8 100644 --- a/lib/data/external/AwsClient.js +++ b/lib/data/external/AwsClient.js @@ -444,7 +444,7 @@ class AwsClient { CopySource: `${sourceAwsBucketName}/${sourceKey}`, Metadata: metaHeaders, MetadataDirective: metadataDirective, - }, err => { + }, (err, copyResult) => { if (err) { if (err.code === 'AccessDenied') { logHelper(log, 'error', 'Unable to access ' + @@ -462,7 +462,13 @@ class AwsClient { `AWS: ${err.message}`) ); } - return callback(null, destAwsKey); + if (!copyResult.VersionId) { + logHelper(log, 'error', 'missing version id for data ' + + 'backend object', missingVerIdInternalError, + this._dataStoreName); + return callback(missingVerIdInternalError); + } + return callback(null, destAwsKey, copyResult.VersionId); }); } uploadPartCopy(request, awsSourceKey, sourceLocationConstraintName, diff --git a/lib/data/multipleBackendGateway.js b/lib/data/multipleBackendGateway.js index fa329dcd5a..fb76d0101f 100644 --- a/lib/data/multipleBackendGateway.js +++ b/lib/data/multipleBackendGateway.js @@ -247,11 +247,13 @@ const multipleBackendGateway = { const client = clients[destLocationConstraintName]; if (client.copyObject) { return client.copyObject(request, externalSourceKey, - sourceLocationConstraintName, log, (err, key) => { + sourceLocationConstraintName, log, (err, key, + dataStoreVersionId) => { const dataRetrievalInfo = { key, dataStoreName: destLocationConstraintName, dataStoreType: client.clientType, + dataStoreVersionId, }; cb(err, dataRetrievalInfo); }); diff --git a/lib/data/wrapper.js b/lib/data/wrapper.js index 3f7ea290bd..4b31b8574f 100644 --- a/lib/data/wrapper.js +++ b/lib/data/wrapper.js @@ -245,11 +245,6 @@ const data = { if (_shouldSkipDelete(locations, requestMethod, newObjDataStoreName)) { return; } - log.trace('initiating batch delete', { - keys: locations, - implName, - method: 'batchDelete', - }); async.eachLimit(locations, 5, (loc, next) => { data.delete(loc, log, next); }, @@ -333,7 +328,7 @@ const data = { dataStoreContext, destBackendInfo, serverSideEncryption, log, cb) => { if (config.backends.data === 'multiple') { const destLocationConstraintName = - destBackendInfo.getControllingLocationConstraint(); + storeMetadataParams.dataStoreName; if (utils.externalBackendCopy(sourceLocationConstraintName, destLocationConstraintName)) { const objectGetInfo = dataLocator[0]; @@ -350,6 +345,8 @@ const data = { dataStoreName, dataStoreType: objectRetrievalInfo. dataStoreType, + dataStoreVersionId: + objectRetrievalInfo.dataStoreVersionId, size: storeMetadataParams.size, dataStoreETag: objectGetInfo.dataStoreETag, start: objectGetInfo.start, @@ -359,7 +356,6 @@ const data = { }); } } - // dataLocator is an array. need to get and put all parts // For now, copy 1 part at a time. Could increase the second // argument here to increase the number of parts @@ -448,6 +444,8 @@ const data = { dataStoreName: partRetrievalInfo. dataStoreName, dataStoreETag: part.dataStoreETag, + dataStoreVersionId: partRetrievalInfo. + dataStoreVersionId, start: part.start, size: part.size, }; diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/objectCopy.js b/tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/objectCopy.js index ea2515772d..7015a17d39 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/objectCopy.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/objectCopy.js @@ -17,7 +17,6 @@ const body = Buffer.from('I am a body', 'utf8'); const correctMD5 = 'be747eb4b75517bf6b3cf7c5fbb62f3a'; const emptyMD5 = 'd41d8cd98f00b204e9800998ecf8427e'; const locMetaHeader = constants.objectLocationConstraintHeader.substring(11); -const { versioningEnabled } = require('../../../lib/utility/versioning-util'); let bucketUtil; let s3; @@ -316,33 +315,6 @@ function testSuite() { }); }); - it('should return NotImplemented copying an object from mem to a ' + - 'versioning enable AWS bucket', done => { - putSourceObj(memLocation, false, bucket, key => { - const copyKey = `copyKey-${Date.now()}`; - const copyParams = { - Bucket: bucket, - Key: copyKey, - CopySource: `/${bucket}/${key}`, - MetadataDirective: 'REPLACE', - Metadata: { - 'scal-location-constraint': awsLocation }, - }; - s3.putBucketVersioning({ - Bucket: bucket, - VersioningConfiguration: versioningEnabled, - }, err => { - assert.equal(err, null, 'putBucketVersioning: ' + - `Expected success, got error ${err}`); - process.stdout.write('Copying object\n'); - s3.copyObject(copyParams, err => { - assert.strictEqual(err.code, 'NotImplemented'); - done(); - }); - }); - }); - }); - it('should copy an object from AWS to mem with "COPY" ' + 'directive and aws location metadata', done => { diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/objectCopyAwsVersioning.js b/tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/objectCopyAwsVersioning.js new file mode 100644 index 0000000000..e807678d4b --- /dev/null +++ b/tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/objectCopyAwsVersioning.js @@ -0,0 +1,419 @@ +const assert = require('assert'); +const async = require('async'); +const withV4 = require('../../support/withV4'); +const BucketUtility = require('../../../lib/utility/bucket-util'); +const { + describeSkipIfNotMultiple, + awsS3, + awsBucket, + memLocation, + fileLocation, + awsLocation, + enableVersioning, + suspendVersioning, + putToAwsBackend, + awsGetLatestVerId, + getAndAssertResult, +} = require('../utils'); + +const sourceBucketName = 'buckettestobjectcopyawsversioning-source'; +const destBucketName = 'buckettestobjectcopyawsversioning-dest'; + +const someBody = Buffer.from('I am a body', 'utf8'); +const wrongVersionBody = 'this is not the content you wanted'; +const correctMD5 = 'be747eb4b75517bf6b3cf7c5fbb62f3a'; +const emptyMD5 = 'd41d8cd98f00b204e9800998ecf8427e'; +const testMetadata = { 'test-header': 'copyme' }; + +let bucketUtil; +let s3; + +function _getCreateBucketParams(bucket, location) { + return { + Bucket: bucket, + CreateBucketConfiguration: { + LocationConstraint: location, + }, + }; +} + +function createBuckets(testParams, cb) { + const { sourceBucket, sourceLocation, destBucket, destLocation } + = testParams; + const sourceParams = _getCreateBucketParams(sourceBucket, sourceLocation); + const destParams = _getCreateBucketParams(destBucket, destLocation); + if (sourceBucket === destBucket) { + return s3.createBucket(sourceParams, err => cb(err)); + } + return async.map([sourceParams, destParams], + (createParams, next) => s3.createBucket(createParams, next), + err => cb(err)); +} + +function putSourceObj(testParams, cb) { + const { sourceBucket, isEmptyObj } = testParams; + const sourceKey = `sourcekey-${Date.now()}`; + const sourceParams = { + Bucket: sourceBucket, + Key: sourceKey, + Metadata: testMetadata, + }; + if (!isEmptyObj) { + sourceParams.Body = someBody; + } + s3.putObject(sourceParams, (err, result) => { + assert.strictEqual(err, null, + `Error putting source object: ${err}`); + if (isEmptyObj) { + assert.strictEqual(result.ETag, `"${emptyMD5}"`); + } else { + assert.strictEqual(result.ETag, `"${correctMD5}"`); + } + Object.assign(testParams, { + sourceKey, + sourceVersionId: result.VersionId, + }); + cb(); + }); +} + +function copyObject(testParams, cb) { + const { sourceBucket, sourceKey, sourceVersionId, sourceVersioningState, + destBucket, directive, destVersioningState, isEmptyObj } + = testParams; + const destKey = `destkey-${Date.now()}`; + const copyParams = { + Bucket: destBucket, + Key: destKey, + CopySource: `/${sourceBucket}/${sourceKey}`, + MetadataDirective: directive, + }; + if (sourceVersionId) { + copyParams.CopySource = + `${copyParams.CopySource}?versionId=${sourceVersionId}`; + } else if (sourceVersioningState === 'Suspended') { + copyParams.CopySource = + `${copyParams.CopySource}?versionId=null`; + } + s3.copyObject(copyParams, (err, data) => { + assert.strictEqual(err, null, + `Error copying object to destination: ${err}`); + if (destVersioningState === 'Enabled') { + assert.notEqual(data.VersionId, undefined); + } else { + assert.strictEqual(data.VersionId, undefined); + } + const expectedBody = isEmptyObj ? '' : someBody; + return awsGetLatestVerId(destKey, expectedBody, (err, awsVersionId) => { + Object.assign(testParams, { + destKey, + destVersionId: data.VersionId, + awsVersionId, + }); + if (!data.VersionId && destVersioningState === 'Suspended') { + // eslint-disable-next-line no-param-reassign + testParams.destVersionId = 'null'; + } + cb(); + }); + }); +} + +function assertGetObjects(testParams, cb) { + const { + sourceBucket, + sourceKey, + sourceVersionId, + destBucket, + destKey, + destVersionId, + awsVersionId, + isEmptyObj, + directive, + } = testParams; + const sourceGetParams = { Bucket: sourceBucket, Key: sourceKey, + VersionId: sourceVersionId }; + const destGetParams = { Bucket: destBucket, Key: destKey, + VersionId: destVersionId }; + const awsParams = { Bucket: awsBucket, Key: destKey, + VersionId: awsVersionId }; + + async.series([ + cb => s3.getObject(sourceGetParams, cb), + cb => s3.getObject(destGetParams, cb), + cb => awsS3.getObject(awsParams, cb), + ], (err, results) => { + assert.strictEqual(err, null, `Error in assertGetObjects: ${err}`); + const [sourceRes, destRes, awsRes] = results; + if (isEmptyObj) { + assert.strictEqual(sourceRes.ETag, `"${emptyMD5}"`); + assert.strictEqual(destRes.ETag, `"${emptyMD5}"`); + assert.strictEqual(awsRes.ETag, `"${emptyMD5}"`); + } else { + assert.strictEqual(sourceRes.ETag, `"${correctMD5}"`); + assert.strictEqual(destRes.ETag, `"${correctMD5}"`); + assert.deepStrictEqual(sourceRes.Body, destRes.Body); + assert.strictEqual(awsRes.ETag, `"${correctMD5}"`); + assert.deepStrictEqual(sourceRes.Body, awsRes.Body); + } + if (directive === 'COPY') { + assert.deepStrictEqual(sourceRes.Metadata, testMetadata); + assert.deepStrictEqual(sourceRes.Metadata, destRes.Metadata); + assert.deepStrictEqual(sourceRes.Metadata, awsRes.Metadata); + } else if (directive === 'REPLACE') { + assert.deepStrictEqual(destRes.Metadata, {}); + assert.deepStrictEqual(awsRes.Metadata, {}); + } + assert.strictEqual(sourceRes.ContentLength, destRes.ContentLength); + cb(); + }); +} + +describeSkipIfNotMultiple('AWS backend object copy with versioning', +function testSuite() { + this.timeout(250000); + withV4(sigCfg => { + bucketUtil = new BucketUtility('default', sigCfg); + s3 = bucketUtil.s3; + + afterEach(() => bucketUtil.empty(sourceBucketName) + .then(() => bucketUtil.deleteOne(sourceBucketName)) + .catch(err => { + process.stdout.write('Error deleting source bucket ' + + `in afterEach: ${err}\n`); + throw err; + }) + .then(() => bucketUtil.empty(destBucketName)) + .then(() => bucketUtil.deleteOne(destBucketName)) + .catch(err => { + if (err.code === 'NoSuchBucket') { + process.stdout.write('Warning: did not find dest bucket ' + + 'for deletion'); + // we do not throw err since dest bucket may not exist + // if we are using source as dest + } else { + process.stdout.write('Error deleting dest bucket ' + + `in afterEach: ${err}\n`); + throw err; + } + }) + ); + + [{ + directive: 'REPLACE', + isEmptyObj: true, + }, { + directive: 'REPLACE', + isEmptyObj: false, + }, { + directive: 'COPY', + isEmptyObj: false, + }].forEach(testParams => { + Object.assign(testParams, { + sourceBucket: sourceBucketName, + sourceLocation: awsLocation, + destBucket: destBucketName, + destLocation: awsLocation, + }); + const { isEmptyObj, directive } = testParams; + it(`should copy ${isEmptyObj ? 'an empty' : ''} object from AWS ` + + 'backend non-versioned bucket to AWS backend versioned bucket ' + + `with ${directive} directive`, done => { + Object.assign(testParams, { + sourceVersioningState: undefined, + destVersioningState: 'Enabled', + }); + async.waterfall([ + next => createBuckets(testParams, next), + next => putSourceObj(testParams, next), + next => enableVersioning(s3, testParams.destBucket, next), + next => copyObject(testParams, next), + // put another version to test and make sure version id from + // copy was stored to get the right version + next => putToAwsBackend(s3, destBucketName, + testParams.destKey, wrongVersionBody, () => next()), + next => assertGetObjects(testParams, next), + ], done); + }); + + it(`should copy ${isEmptyObj ? 'an empty ' : ''}version from one ` + + `AWS backend versioned bucket to another on ${directive} directive`, + done => { + Object.assign(testParams, { + sourceVersioningState: 'Enabled', + destVersioningState: 'Enabled', + }); + async.waterfall([ + next => createBuckets(testParams, next), + next => enableVersioning(s3, testParams.sourceBucket, next), + next => putSourceObj(testParams, next), + next => enableVersioning(s3, testParams.destBucket, next), + next => copyObject(testParams, next), + // put another version to test and make sure version id from + // copy was stored to get the right version + next => putToAwsBackend(s3, destBucketName, + testParams.destKey, wrongVersionBody, () => next()), + next => assertGetObjects(testParams, next), + ], done); + }); + + it(`should copy ${isEmptyObj ? 'an empty ' : ''}null version ` + + 'from one AWS backend versioning suspended bucket to another ' + + `versioning suspended bucket with ${directive} directive`, + done => { + Object.assign(testParams, { + sourceVersioningState: 'Suspended', + destVersioningState: 'Suspended', + }); + async.waterfall([ + next => createBuckets(testParams, next), + next => suspendVersioning(s3, testParams.sourceBucket, + next), + next => putSourceObj(testParams, next), + next => suspendVersioning(s3, testParams.destBucket, next), + next => copyObject(testParams, next), + next => enableVersioning(s3, testParams.destBucket, next), + // put another version to test and make sure version id from + // copy was stored to get the right version + next => putToAwsBackend(s3, destBucketName, + testParams.destKey, wrongVersionBody, () => next()), + next => assertGetObjects(testParams, next), + ], done); + }); + + it(`should copy ${isEmptyObj ? 'an empty ' : ''}version from a ` + + 'AWS backend versioned bucket to a versioned-suspended one with ' + + `${directive} directive`, done => { + Object.assign(testParams, { + sourceVersioningState: 'Enabled', + destVersioningState: 'Suspended', + }); + async.waterfall([ + next => createBuckets(testParams, next), + next => enableVersioning(s3, testParams.sourceBucket, next), + next => putSourceObj(testParams, next), + next => suspendVersioning(s3, testParams.destBucket, next), + next => copyObject(testParams, next), + // put another version to test and make sure version id from + // copy was stored to get the right version + next => enableVersioning(s3, testParams.destBucket, next), + next => putToAwsBackend(s3, destBucketName, + testParams.destKey, wrongVersionBody, () => next()), + next => assertGetObjects(testParams, next), + ], done); + }); + }); + + it('versioning not configured: if copy object to a pre-existing ' + + 'object on AWS backend, metadata should be overwritten but data of ' + + 'previous version in AWS should not be deleted', function itF(done) { + const destKey = `destkey-${Date.now()}`; + const testParams = { + sourceBucket: sourceBucketName, + sourceLocation: awsLocation, + sourceVersioningState: undefined, + destBucket: sourceBucketName, + destLocation: awsLocation, + destVersioningState: undefined, + isEmptyObj: true, + directive: 'REPLACE', + }; + async.waterfall([ + next => createBuckets(testParams, next), + next => putToAwsBackend(s3, testParams.destBucket, destKey, + someBody, err => next(err)), + next => awsGetLatestVerId(destKey, someBody, next), + (awsVerId, next) => { + this.test.awsVerId = awsVerId; + next(); + }, + next => putSourceObj(testParams, next), + next => s3.copyObject({ + Bucket: testParams.destBucket, + Key: destKey, + CopySource: `/${testParams.sourceBucket}` + + `/${testParams.sourceKey}`, + MetadataDirective: testParams.directive, + Metadata: { + 'scal-location-constraint': testParams.destLocation, + }, + }, next), + (copyResult, next) => awsGetLatestVerId(destKey, '', + (err, awsVersionId) => { + testParams.destKey = destKey; + testParams.destVersionId = copyResult.VersionId; + testParams.awsVersionId = awsVersionId; + next(); + }), + next => s3.deleteObject({ Bucket: testParams.destBucket, + Key: testParams.destKey, VersionId: 'null' }, next), + (delData, next) => getAndAssertResult(s3, { bucket: + testParams.destBucket, key: testParams.destKey, + expectedError: 'NoSuchKey' }, next), + next => awsGetLatestVerId(testParams.destKey, someBody, next), + (awsVerId, next) => { + assert.strictEqual(awsVerId, this.test.awsVerId); + next(); + }, + ], done); + }); + + [{ + sourceLocation: memLocation, + directive: 'REPLACE', + isEmptyObj: true, + }, { + sourceLocation: fileLocation, + directive: 'REPLACE', + isEmptyObj: true, + }, { + sourceLocation: memLocation, + directive: 'COPY', + isEmptyObj: false, + }, { + sourceLocation: fileLocation, + directive: 'COPY', + isEmptyObj: false, + }].forEach(testParams => { + Object.assign(testParams, { + sourceBucket: sourceBucketName, + sourceVersioningState: 'Enabled', + destBucket: destBucketName, + destLocation: awsLocation, + destVersioningState: 'Enabled', + }); + const { sourceLocation, directive, isEmptyObj } = testParams; + + it(`should copy ${isEmptyObj ? 'empty ' : ''}object from ` + + `${sourceLocation} to bucket on AWS backend with ` + + `versioning with ${directive}`, done => { + async.waterfall([ + next => createBuckets(testParams, next), + next => putSourceObj(testParams, next), + next => enableVersioning(s3, testParams.destBucket, next), + next => copyObject(testParams, next), + next => assertGetObjects(testParams, next), + ], done); + }); + + it(`should copy ${isEmptyObj ? 'an empty ' : ''}version from ` + + `${sourceLocation} to bucket on AWS backend with ` + + `versioning with ${directive} directive`, done => { + async.waterfall([ + next => createBuckets(testParams, next), + next => enableVersioning(s3, testParams.sourceBucket, next), + // returns a version id which is added to testParams + // to be used in object copy + next => putSourceObj(testParams, next), + next => enableVersioning(s3, testParams.destBucket, next), + next => copyObject(testParams, next), + // put another version to test and make sure version id + // from copy was stored to get the right version + next => putToAwsBackend(s3, destBucketName, + testParams.destKey, wrongVersionBody, () => next()), + next => assertGetObjects(testParams, next), + ], done); + }); + }); + }); +}); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/utils.js b/tests/functional/aws-node-sdk/test/multipleBackend/utils.js index 40ab193318..03fd2e48a1 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/utils.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/utils.js @@ -234,7 +234,8 @@ utils.awsGetLatestVerId = (key, body, cb) => { `getting object from AWS, got error ${err}`); const resultMD5 = utils.expectedETag(result.Body, false); const expectedMD5 = utils.expectedETag(body, false); - assert.strictEqual(resultMD5, expectedMD5); + assert.strictEqual(resultMD5, expectedMD5, + 'expected different body'); return cb(null, result.VersionId); }); }; diff --git a/tests/functional/aws-node-sdk/test/versioning/objectCopy.js b/tests/functional/aws-node-sdk/test/versioning/objectCopy.js index 72035019eb..0b2e251da2 100644 --- a/tests/functional/aws-node-sdk/test/versioning/objectCopy.js +++ b/tests/functional/aws-node-sdk/test/versioning/objectCopy.js @@ -467,17 +467,18 @@ describe('Object Version Copy', () => { it('should copy a 0 byte object to same destination', done => { const emptyFileETag = '"d41d8cd98f00b204e9800998ecf8427e"'; s3.putObject({ Bucket: sourceBucketName, Key: sourceObjName, - Body: '' }, (err, res) => { + Body: '' }, (err, putRes) => { checkNoError(err); copySource = `${sourceBucketName}/${sourceObjName}` + - `?versionId=${res.VersionId}`; + `?versionId=${putRes.VersionId}`; s3.copyObject({ Bucket: sourceBucketName, Key: sourceObjName, CopySource: copySource, StorageClass: 'REDUCED_REDUNDANCY', }, - (err, res) => { + (err, copyRes) => { checkNoError(err); - assert.strictEqual(res.ETag, emptyFileETag); + assert.notEqual(copyRes.VersionId, putRes.VersionId); + assert.strictEqual(copyRes.ETag, emptyFileETag); s3.getObject({ Bucket: sourceBucketName, Key: sourceObjName }, (err, res) => { assert.deepStrictEqual(res.Metadata, diff --git a/tests/multipleBackend/objectCopy.js b/tests/multipleBackend/objectCopy.js new file mode 100644 index 0000000000..ccf278e6dc --- /dev/null +++ b/tests/multipleBackend/objectCopy.js @@ -0,0 +1,107 @@ +const assert = require('assert'); +const async = require('async'); + +const { bucketPut } = require('../../lib/api/bucketPut'); +const objectPut = require('../../lib/api/objectPut'); +const objectCopy = require('../../lib/api/objectCopy'); +const { metadata } = require('../../lib/metadata/in_memory/metadata'); +const DummyRequest = require('../unit/DummyRequest'); +const { cleanup, DummyRequestLogger, makeAuthInfo } + = require('../unit/helpers'); + +const log = new DummyRequestLogger(); +const canonicalID = 'accessKey1'; +const authInfo = makeAuthInfo(canonicalID); +const namespace = 'default'; +const destBucketName = 'destbucketname'; +const sourceBucketName = 'sourcebucketname'; +const memLocation = 'mem-test'; +const fileLocation = 'file-test'; + +function _createBucketPutRequest(bucketName, bucketLoc) { + const post = bucketLoc ? '' + + '' + + `${bucketLoc}` + + '' : ''; + return new DummyRequest({ + bucketName, + namespace, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: '/', + post, + }); +} + +function _createObjectCopyRequest(destBucketName, objectKey) { + const params = { + bucketName: destBucketName, + namespace, + objectKey, + headers: {}, + url: `/${destBucketName}/${objectKey}`, + }; + return new DummyRequest(params); +} + +function _createObjectPutRequest(bucketName, objectKey, body) { + const sourceObjPutParams = { + bucketName, + namespace, + objectKey, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: '/', + }; + return new DummyRequest(sourceObjPutParams, body); +} + +function copySetup(params, cb) { + const { sourceBucket, sourceLocation, sourceKey, destBucket, + destLocation, body } = params; + const putDestBucketRequest = + _createBucketPutRequest(destBucket, destLocation); + const putSourceBucketRequest = + _createBucketPutRequest(sourceBucket, sourceLocation); + const putSourceObjRequest = _createObjectPutRequest(sourceBucket, + sourceKey, body); + async.series([ + callback => bucketPut(authInfo, putDestBucketRequest, log, callback), + callback => bucketPut(authInfo, putSourceBucketRequest, log, callback), + callback => objectPut(authInfo, putSourceObjRequest, undefined, log, + callback), + ], err => cb(err)); +} + +describe('ObjectCopy API with multiple backends', () => { + before(() => { + cleanup(); + }); + + after(() => cleanup()); + + it('object metadata for newly stored object should have dataStoreName ' + + 'if copying to mem based on bucket location', done => { + const params = { + sourceBucket: sourceBucketName, + sourceKey: `sourcekey-${Date.now()}`, + sourceLocation: fileLocation, + body: 'testbody', + destBucket: destBucketName, + destLocation: memLocation, + }; + const destKey = `destkey-${Date.now()}`; + const testObjectCopyRequest = + _createObjectCopyRequest(destBucketName, destKey); + copySetup(params, err => { + assert.strictEqual(err, null, `Error setting up copy: ${err}`); + objectCopy(authInfo, testObjectCopyRequest, sourceBucketName, + params.sourceKey, undefined, log, err => { + assert.strictEqual(err, null, `Error copying: ${err}`); + const bucket = metadata.keyMaps.get(params.destBucket); + const objMd = bucket.get(destKey); + assert.strictEqual(objMd.dataStoreName, memLocation); + done(); + }); + }); + }); +});