8000 ft: enable aws backend object copy w/ versioning [S3C-1053] by electrachong · Pull Request #957 · scality/cloudserver · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

ft: enable aws backend object copy w/ versioning [S3C-1053] #957

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Oct 26, 2017
Merged

ft: enable aws backend object copy w/ versioning [S3C-1053] #957

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file 8000
Failed to load files.
Loading
Diff view
Diff view
52 changes: 27 additions & 25 deletions lib/api/objectCopy.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ const { config } = require('../Config');

const versionIdUtils = versioning.VersionID;
const locationHeader = constants.objectLocationConstraintHeader;
const versioningNotImplBackends = constants.versioningNotImplBackends;
const externalVersioningErrorMessage = 'We do not currently support putting ' +
'a versioned object to a location-constraint of type AWS or Azure.';

Expand Down Expand Up @@ -150,6 +151,7 @@ function _prepMetadata(request, sourceObjMD, headers, sourceIsDestination,
contentDisposition: headersToStoreSource['content-disposition'],
contentEncoding:
removeAWSChunked(headersToStoreSource['content-encoding']),
dataStoreName: destLocationConstraintName,
expires: headersToStoreSource.expires,
overrideMetadata,
lastModifiedDate: new Date().toJSON(),
Expand All @@ -166,7 +168,7 @@ function _prepMetadata(request, sourceObjMD, headers, sourceIsDestination,
storeMetadataParams.contentType = sourceObjMD['content-type'];
}
return { storeMetadataParams, sourceLocationConstraintName,
backendInfoObjDest };
backendInfoDest: backendInfoObjDest.backendInfo };
}

/**
Expand Down Expand Up @@ -278,7 +280,7 @@ function objectCopy(authInfo, request, sourceBucket,
return next(errors.PreconditionFailed, destBucketMD);
}
const { storeMetadataParams, error: metadataError,
sourceLocationConstraintName, backendInfoObjDest } =
sourceLocationConstraintName, backendInfoDest } =
_prepMetadata(request, sourceObjMD, request.headers,
sourceIsDestination, authInfo, destObjectKey,
sourceBucketMD, destBucketMD, log);
Expand Down Expand Up @@ -311,20 +313,23 @@ function objectCopy(authInfo, request, sourceBucket,
}
return next(null, storeMetadataParams, dataLocator,
destBucketMD, destObjMD, sourceLocationConstraintName,
backendInfoObjDest);
backendInfoDest);
});
},
function goGetData(storeMetadataParams, dataLocator, destBucketMD,
destObjMD, sourceLocationConstraintName, backendInfoObjDest, next) {
destObjMD, sourceLocationConstraintName, backendInfoDest, next) {
const serverSideEncryption = destBucketMD.getServerSideEncryption();
const vcfg = destBucketMD.getVersioningConfiguration();
const isVersionedObj = vcfg && vcfg.Status === 'Enabled';
const destLocationConstraintName =
storeMetadataParams.dataStoreName;

const backendInfoDest = backendInfoObjDest.backendInfo;
const destLocationConstraintName = backendInfoObjDest.controllingLC;

// skip if source and dest and location constraint the same
// skip if source and dest and location constraint the same and
// versioning is not enabled
// still send along serverSideEncryption info so algo
// and masterKeyId stored properly in metadata
if (sourceIsDestination && storeMetadataParams.locationMatch) {
if (sourceIsDestination && storeMetadataParams.locationMatch
&& !isVersionedObj) {
Copy link
Contributor

Choose a reason for hiding this comment

The reaso 8000 n will be displayed to describe this comment to others. Learn more.

@nicolas note this.

return next(null, storeMetadataParams, dataLocator, destObjMD,
serverSideEncryption, destBucketMD);
}
Expand All @@ -334,18 +339,13 @@ function objectCopy(authInfo, request, sourceBucket,
// metadata to backend
const destLocationConstraintType =
config.getLocationConstraintType(destLocationConstraintName);
// NOTE: remove the following when we will support putting a
// versioned object to a location-constraint of type AWS or Azure.
if (constants.externalBackends[destLocationConstraintType]) {
const vcfg = destBucketMD.getVersioningConfiguration();
const isVersionedObj = vcfg && vcfg.Status === 'Enabled';
if (isVersionedObj) {
log.debug(externalVersioningErrorMessage,
{ method: 'multipleBackendGateway',
error: errors.NotImplemented });
return next(errors.NotImplemented.customizeDescription(
externalVersioningErrorMessage), destBucketMD);
}
if (versioningNotImplBackends[destLocationConstraintType]
&& isVersionedObj) {
log.debug(externalVersioningErrorMessage,
{ method: 'multipleBackendGateway',
error: errors.NotImplemented });
return next(errors.NotImplemented.customizeDescription(
externalVersioningErrorMessage), destBucketMD);
}
if (dataLocator.length === 0) {
if (!storeMetadataParams.locationMatch &&
Expand All @@ -372,7 +372,6 @@ function objectCopy(authInfo, request, sourceBucket,
return next(null, storeMetadataParams, dataLocator, destObjMD,
serverSideEncryption, destBucketMD);
}

return data.copyObject(request, sourceLocationConstraintName,
storeMetadataParams, dataLocator, dataStoreContext,
backendInfoDest, serverSideEncryption, log,
Expand Down Expand Up @@ -422,9 +421,12 @@ function objectCopy(authInfo, request, sourceBucket,
// object with same name, so long as the source is not
// the same as the destination
if (!sourceIsDestination && dataToDelete) {
data.batchDelete(dataToDelete, request.method, null,
logger.newRequestLoggerFromSerializedUids(
log.getSerializedUids()));
const newDataStoreName =
storeMetadataParams.dataStoreName;
data.batchDelete(dataToDelete, request.method,
newDataStoreName,
logger.newRequestLoggerFromSerializedUids(
log.getSerializedUids()));
}
const sourceObjSize = storeMetadataParams.size;
const destObjPrevSize = (destObjMD &&
Expand Down
10 changes: 8 additions & 2 deletions lib/data/external/AwsClient.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -444,7 +444,7 @@ class AwsClient {
CopySource: `${sourceAwsBucketName}/${sourceKey}`,
Metadata: metaHeaders,
MetadataDirective: metadataDirective,
}, err => {
8000 }, (err, copyResult) => {
if (err) {
if (err.code === 'AccessDenied') {
logHelper(log, 'error', 'Unable to access ' +
Expand All @@ -462,7 +462,13 @@ class AwsClient {
`AWS: ${err.message}`)
);
}
return callback(null, destAwsKey);
if (!copyResult.VersionId) {
logHelper(log, 'error', 'missing version id for data ' +
'backend object', missingVerIdInternalError,
this._dataStoreName);
return callback(missingVerIdInternalError);
}
return callback(null, destAwsKey, copyResult.VersionId);
});
}
uploadPartCopy(request, awsSourceKey, sourceLocationConstraintName,
Expand Down
4 changes: 3 additions & 1 deletion lib/data/multipleBackendGateway.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -247,11 +247,13 @@ const multipleBackendGateway = {
const client = clients[destLocationConstraintName];
if (client.copyObject) {
return client.copyObject(request, externalSourceKey,
sourceLocationConstraintName, log, (err, key) => {
sourceLocationConstraintName, log, (err, key,
dataStoreVersionId) => {
const dataRetrievalInfo = {
key,
dataStoreName: destLocationConstraintName,
dataStoreType: client.clientType,
dataStoreVersionId,
};
cb(err, dataRetrievalInfo);
});
Expand Down
12 changes: 5 additions & 7 deletions lib/data/wrapper.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -245,11 +245,6 @@ const data = {
if (_shouldSkipDelete(locations, requestMethod, newObjDataStoreName)) {
return;
}
log.trace('initiating batch delete', {
keys: locations,
implName,
method: 'batchDelete',
});
async.eachLimit(locations, 5, (loc, next) => {
data.delete(loc, log, next);
},
Expand Down Expand Up @@ -333,7 +328,7 @@ const data = {
dataStoreContext, destBackendInfo, serverSideEncryption, log, cb) => {
if (config.backends.data === 'multiple') {
const destLocationConstraintName =
destBackendInfo.getControllingLocationConstraint();
storeMetadataParams.dataStoreName;
if (utils.externalBackendCopy(sourceLocationConstraintName,
destLocationConstraintName)) {
const objectGetInfo = dataLocator[0];
Expand All @@ -350,6 +345,8 @@ const data = {
dataStoreName,
dataStoreType: objectRetrievalInfo.
dataStoreType,
dataStoreVersionId:
objectRetrievalInfo.dataStoreVersionId,
size: storeMetadataParams.size,
dataStoreETag: objectGetInfo.dataStoreETag,
start: objectGetInfo.start,
Expand All @@ -359,7 +356,6 @@ const data = {
});
}
}

// dataLocator is an array. need to get and put all parts
// For now, copy 1 part at a time. Could increase the second
// argument here to increase the number of parts
Expand Down Expand Up @@ -448,6 +444,8 @@ const data = {
dataStoreName: partRetrievalInfo.
dataStoreName,
dataStoreETag: part.dataStoreETag,
dataStoreVersionId: partRetrievalInfo.
dataStoreVersionId,
start: part.start,
size: part.size,
};
Expand Down
28 changes: 0 additions & 28 deletions tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/objectCopy.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ const body = Buffer.from('I am a body', 'utf8');
const correctMD5 = 'be747eb4b75517bf6b3cf7c5fbb62f3a';
const emptyMD5 = 'd41d8cd98f00b204e9800998ecf8427e';
const locMetaHeader = constants.objectLocationConstraintHeader.substring(11);
const { versioningEnabled } = require('../../../lib/utility/versioning-util');

let bucketUtil;
let s3;
Expand Down Expand Up @@ -316,33 +315,6 @@ function testSuite() {
});
});

it('should return NotImplemented copying an object from mem to a ' +
'versioning enable AWS bucket', done => {
putSourceObj(memLocation, false, bucket, key => {
const copyKey = `copyKey-${Date.now()}`;
const copyParams = {
Bucket: bucket,
Key: copyKey,
CopySource: `/${bucket}/${key}`,
MetadataDirective: 'REPLACE',
Metadata: {
'scal-location-constraint': awsLocation },
};
s3.putBucketVersioning({
Bucket: bucket,
VersioningConfiguration: versioningEnabled,
}, err => {
assert.equal(err, null, 'putBucketVersioning: ' +
`Expected success, got error ${err}`);
process.stdout.write('Copying object\n');
s3.copyObject(copyParams, err => {
assert.strictEqual(err.code, 'NotImplemented');
done();
});
});
});
});

it('should copy an object from AWS to mem with "COPY" ' +
'directive and aws location metadata',
done => {
Expand Down
Loading
0