Feature/cldsrv 546 post object #5601
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,210 @@ | ||
| const { auth, errors } = require('arsenal'); | ||
| const busboy = require('@fastify/busboy'); | ||
| const writeContinue = require('../../../utilities/writeContinue'); | ||
| const fs = require('fs'); | ||
| const path = require('path'); | ||
| const os = require('os'); | ||
|
|
||
| /** @see doc: https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-HTTPPOSTForms.html#HTTPPOSTFormDeclaration */ | ||
| const MAX_FIELD_SIZE = 20 * 1024; // 20KB | ||
| /** @see doc: https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html */ | ||
| const MAX_KEY_SIZE = 1024; | ||
| const POST_OBJECT_OPTIONAL_FIELDS = [ | ||
| 'acl', | ||
| 'awsaccesskeyid', | ||
| 'bucket', | ||
| 'cache-control', | ||
| 'content-disposition', | ||
| 'content-encoding', | ||
| 'content-type', | ||
| 'expires', | ||
| 'policy', | ||
| 'redirect', | ||
| 'tagging', | ||
| 'success_action_redirect', | ||
| 'success_action_status', | ||
| 'x-amz-meta-', | ||
| 'x-amz-storage-class', | ||
| 'x-amz-security-token', | ||
| 'x-amz-signgnature', | ||
| 'x-amz-website-redirect-location', | ||
| ]; | ||
|
|
||
| async function authenticateRequest(request, requestContexts, log) { | ||
| return new Promise(resolve => { | ||
| // TODO RING-45960 remove ignore auth check for POST object here | ||
| auth.server.doAuth(request, log, (err, userInfo, authorizationResults, streamingV4Params) => | ||
| resolve({ userInfo, authorizationResults, streamingV4Params }), 's3', requestContexts); | ||
| }); | ||
| } | ||
|
|
||
| async function parseFormData(request, response, requestContexts, log) { | ||
| /* eslint-disable no-param-reassign */ | ||
| let formDataParser; | ||
| try { | ||
| formDataParser = busboy({ headers: request.headers }); | ||
|
There was a problem hiding this comment. Some more details about Content-TypeDocumentation for the Content-Type, with the boundary directive and example with body: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Type Again, That will unnecessarily read the whole body and trigger an error Content-DispositionThe documentation about In the body, a part will look like Here You can send this with:
You can also have You can send this with:
Notice the You can notice in AWS documentation about the
So you can either send:
Which means by default the Also by testing on AWS, it accepts other form fields like
So any form field can be emitted by either the Note that it could be interesting to replace the default behavior of busboy and defining When doing that there should also be some limits defined on busboy, if a form field other than
${filename} replacementAbout
Also the documentation defines how And validation on form fields, like Additional validationYou can receive a Also by testing on AWS, they seem to trim the It's also possible to have multiple parts with the same <input type="text" name="key">
<input type="text" name="key">
<input type="text" name="acl">
<input type="text" name="acl">
<input type="file" name="file" multiple>or We should ensure, just like AWS that we don't have multiple times the same form field, whether busboy emits it via Maybe the file should not be read if the parsing failed and an error was caught. It seems the There is a validation on The
You should find attached a javascript script to write the raw body form-data easier than using `nc`.
It allows to test invalid bodies that are usually well formated by lib like `form-data` or by `curl` or `postman`.
|
||
| } catch (err) { | ||
| log.trace('Error creating form data parser', { error: err.toString() }); | ||
| return Promise.reject(errors.PreconditionFailed | ||
| .customizeDescription('Bucket POST must be of the enclosure-type multipart/form-data')); | ||
| } | ||
|
|
||
| // formDataParser = busboy({ headers: request.headers }); | ||
| writeContinue(request, response); | ||
|
|
||
| return new Promise((resolve, reject) => { | ||
| request.formData = {}; | ||
| let totalFieldSize = 0; | ||
| let fileEventData = null; | ||
| let tempFileStream; | ||
| let tempFilePath; | ||
| let authResponse; | ||
| let fileWrittenPromiseResolve; | ||
| let formParserFinishedPromiseResolve; | ||
|
|
||
| const fileWrittenPromise = new Promise((res) => { fileWrittenPromiseResolve = res; }); | ||
| const formParserFinishedPromise = new Promise((res) => { formParserFinishedPromiseResolve = res; }); | ||
|
|
||
| formDataParser.on('field', (fieldname, val) => { | ||
| // Check if we have exceeded the max size allowed for all fields | ||
| totalFieldSize += Buffer.byteLength(val, 'utf8'); | ||
| if (totalFieldSize > MAX_FIELD_SIZE) { | ||
| return reject(errors.MaxPostPreDataLengthExceeded); | ||
| } | ||
|
|
||
| // validate the fieldname | ||
| const lowerFieldname = fieldname.toLowerCase(); | ||
| // special handling for key field | ||
| if (lowerFieldname === 'key') { | ||
| 10000 | if (val.length > MAX_KEY_SIZE) { | |
| return reject(errors.KeyTooLong); | ||
| } else if (val.length === 0) { | ||
| return reject(errors.InvalidArgument | ||
| .customizeDescription('User key must have a length greater than 0.')); | ||
| } | ||
| request.formData[lowerFieldname] = val; | ||
| } | ||
| // add only the recognized fields to the formData object | ||
| if (POST_OBJECT_OPTIONAL_FIELDS.some(field => lowerFieldname.startsWith(field))) { | ||
|
There was a problem hiding this comment.
|
||
| request.formData[lowerFieldname] = val; | ||
| } | ||
| return undefined; | ||
| }); | ||
|
|
||
| formDataParser.on('file', async (fieldname, file, filename, encoding, mimetype) => { | ||
| if (fileEventData) { | ||
| file.resume(); // Resume the stream to drain and discard the file | ||
| if (tempFilePath) { | ||
| fs.unlink(tempFilePath, unlinkErr => { | ||
| if (unlinkErr) { | ||
| log.error('Failed to delete temp file', { error: unlinkErr }); | ||
| } | ||
| }); | ||
| } | ||
| return reject(errors.InvalidArgument | ||
| .customizeDescription('POST requires exactly one file upload per request.')); | ||
| } | ||
|
|
||
| fileEventData = { fieldname, file, filename, encoding, mimetype }; | ||
| if (!('key' in request.formData)) { | ||
| return reject(errors.InvalidArgument | ||
| .customizeDescription('Bucket POST must contain a field named ' | ||
| + "'key'. If it is specified, please check the order of the fields.")); | ||
| } | ||
| // Replace `${filename}` with the actual filename | ||
| request.formData.key = request.formData.key.replace('${filename}', filename); | ||
| try { | ||
| // Authenticate request before streaming file | ||
| // TODO RING-45960 auth to be properly implemented | ||
| authResponse = await authenticateRequest(request, requestContexts, log); | ||
|
|
||
| // Create a temporary file to stream the file data | ||
| // This is to finalize validation on form data before storing the file | ||
| tempFilePath = path.join(os.tmpdir(), filename); | ||
| tempFileStream = fs.createWriteStream(tempFilePath); | ||
|
|
||
| file.pipe(tempFileStream); | ||
|
|
||
| tempFileStream.on('finish', () => { | ||
| request.fileEventData = { ...fileEventData, file: tempFilePath }; | ||
| fileWrittenPromiseResolve(); | ||
| }); | ||
|
|
||
| tempFileStream.on('error', (err) => { | ||
| log.trace('Error streaming file to temporary location', { error: err.toString() }); | ||
| reject(errors.InternalError); | ||
| }); | ||
|
|
||
| // Wait for both file writing and form parsing to finish | ||
| return Promise.all([fileWrittenPromise, formParserFinishedPromise]) | ||
| .then(() => resolve(authResponse)) | ||
| .catch(reject); | ||
| } catch (err) { | ||
| return reject(err); | ||
| } | ||
| }); | ||
|
|
||
| formDataParser.on('finish', () => { | ||
| if (!fileEventData) { | ||
| return reject(errors.InvalidArgument | ||
| .customizeDescription('POST requires exactly one file upload per request.')); | ||
| } | ||
| return formParserFinishedPromiseResolve(); | ||
| }); | ||
|
|
||
| formDataParser.on('error', (err) => { | ||
| log.trace('Error processing form data:', { error: err.toString() }); | ||
| request.unpipe(formDataParser); | ||
| // Following observed AWS behaviour | ||
| reject(errors.MalformedPOSTRequest); | ||
| }); | ||
|
|
||
| request.pipe(formDataParser); | ||
| return undefined; | ||
| }); | ||
| } | ||
|
|
||
| function getFileStat(filePath, log) { | ||
| return new Promise((resolve, reject) => { | ||
| fs.stat(filePath, (err, stats) => { | ||
| if (err) { | ||
| log.trace('Error getting file size', { error: err.toString() }); | ||
| return reject(errors.InternalError); | ||
| } | ||
| return resolve(stats); | ||
| }); | ||
| }); | ||
| } | ||
|
|
||
| async function processPostForm(request, response, requestContexts, log, callback) { | ||
| try { | ||
| const { userInfo, authorizationResults, streamingV4Params } = | ||
| await parseFormData(request, response, requestContexts, log); | ||
|
|
||
| const fileStat = await getFileStat(request.fileEventData.file, log); | ||
| request.parsedContentLength = fileStat.size; | ||
| request.fileEventData.file = fs.createReadStream(request.fileEventData.file); | ||
| if (request.formData['content-type']) { | ||
| request.headers['content-type'] = request.formData['content-type']; | ||
| } else { | ||
| request.headers['content-type'] = 'binary/octet-stream'; | ||
| } | ||
|
|
||
| const authNames = { accountName: userInfo.getAccountDisplayName() }; | ||
| if (userInfo.isRequesterAnIAMUser()) { | ||
| authNames.userName = userInfo.getIAMdisplayName(); | ||
| } | ||
| log.addDefaultFields(authNames); | ||
|
|
||
| return callback(null, userInfo, authorizationResults, streamingV4Params); | ||
| } catch (err) { | ||
| return callback(err); | ||
| } | ||
| } | ||
|
|
||
| module.exports = { | ||
| authenticateRequest, | ||
| parseFormData, | ||
| processPostForm, | ||
| getFileStat, | ||
| }; | ||
Uh oh!
There was an error while loading. Please reload this page.