From d5cd1aaaa9b243ae9f4865e3ea8049b3e90ec60b Mon Sep 17 00:00:00 2001 From: Davide Aquaro Date: Wed, 2 Nov 2022 15:17:00 +0000 Subject: [PATCH 1/3] MIG-143: extended partial object list and updated tests --- packages/mdctl-cli/tasks/study.js | 17 ++++++----- packages/mdctl-cli/test/tasks/study.js | 39 ++++++++++++++++++++++---- 2 files changed, 43 insertions(+), 13 deletions(-) diff --git a/packages/mdctl-cli/tasks/study.js b/packages/mdctl-cli/tasks/study.js index 7bb7ff04..8d532b20 100644 --- a/packages/mdctl-cli/tasks/study.js +++ b/packages/mdctl-cli/tasks/study.js @@ -85,7 +85,7 @@ class Study extends Task { try { let manifestJSON if (manifestObj) { - manifestJSON = this.validateManifest(manifestObj) + manifestJSON = this.validateManifest(manifestObj, studyTools.getAvailableObjectNames()) } const { manifest } = await studyTools.getStudyManifest(manifestJSON) @@ -196,7 +196,7 @@ class Study extends Task { } - validateManifest(manifestObject) { + validateManifest(manifestObject, validKeys) { let manifestJSON try { manifestJSON = JSON.parse(manifestObject) @@ -214,13 +214,16 @@ class Study extends Task { } } /* - Ignore any keys passed in other than Assignments and eConsents. - In future this will be removed but for now we will only support those 2 objects together + We are allowing to export only "parent" objects, i.e. objects + that have dependencies */ - manifestJSON = _.pick(manifestJSON, ['c_task', 'ec__document_template', 'object']) + manifestJSON = _.pick(manifestJSON, ['object', ...validKeys]) if (_.isEqual(manifestJSON, { object: 'manifest' })) { - // This means that the manifest passed does not contain Assignments or eConsents - throw Fault.create('kInvalidArgument', { reason: 'No Assignments or eConsents to export' }) + // This means that the manifest passed does not contain any object to export + throw Fault.create('kInvalidArgument', { reason: 'Nothing to export' }) + } + if (!manifestJSON.object || manifestJSON.object !== 'manifest') { + throw Fault.create('kInvalidArgument', { reason: 'Invalid manifest. Please make sure it contains the right key/value ("object": "manifest")' }) } return manifestJSON } diff --git a/packages/mdctl-cli/test/tasks/study.js b/packages/mdctl-cli/test/tasks/study.js index 8fcbbd97..c4a010bf 100644 --- a/packages/mdctl-cli/test/tasks/study.js +++ b/packages/mdctl-cli/test/tasks/study.js @@ -1,12 +1,15 @@ // eslint-disable-next-line import/no-extraneous-dependencies const { assert, expect } = require('chai'), + { StudyManifestTools } = require('@medable/mdctl-axon-tools'), Study = require('../../tasks/study') -let study +let study, + studyManifest describe('MIG-85 - Test partial migrations in study.js module', () => { before(() => { study = new Study() + studyManifest = new StudyManifestTools() }) it('Test validateManifest function - Invalid file path', () => { @@ -45,10 +48,34 @@ describe('MIG-85 - Test partial migrations in study.js module', () => { .equal(res.reason, 'The manifest is not a valid JSON') }) + it('Test validateManifest function - Invalid manifest', () => { + const manifest = `{ + "c_group_task": { + "includes": [ + "key-002" + ] + } + }` + + let res + try { + res = study.validateManifest(manifest, studyManifest.getAvailableObjectNames()) + } catch (err) { + res = err + } + + assert + .equal(res.errCode, 'mdctl.invalidArgument.unspecified') + assert + .equal(res.code, 'kInvalidArgument') + assert + .equal(res.reason, 'Invalid manifest. Please make sure it contains the right key/value ("object": "manifest")') + }) + it('Test validateManifest function - No entities to export', () => { const manifest = `{ "object": "manifest", - "c_group": { + "c_step": { "includes": [ "key-014" ] @@ -62,7 +89,7 @@ describe('MIG-85 - Test partial migrations in study.js module', () => { let res try { - res = study.validateManifest(manifest) + res = study.validateManifest(manifest, studyManifest.getAvailableObjectNames()) } catch (err) { res = err } @@ -72,11 +99,11 @@ describe('MIG-85 - Test partial migrations in study.js module', () => { assert .equal(res.code, 'kInvalidArgument') assert - .equal(res.reason, 'No Assignments or eConsents to export') + .equal(res.reason, 'Nothing to export') }) - it('Test validateManifest function - Removed entities other than Assignments and eConsents', () => { + it('Test validateManifest function - Removed not allowed entities', () => { const manifest = `{ "object": "manifest", "c_task": { @@ -90,7 +117,7 @@ describe('MIG-85 - Test partial migrations in study.js module', () => { ] } }`, - res = study.validateManifest(manifest) + res = study.validateManifest(manifest, studyManifest.getAvailableObjectNames()) expect(res) .to.deep.equal({ From 70ab56fe3cddc3e8d18612b7e1679af267ed3ef7 Mon Sep 17 00:00:00 2001 From: Davide Aquaro Date: Thu, 10 Nov 2022 12:28:41 +0000 Subject: [PATCH 2/3] enabled individual instances export on objects in the list --- .../mdctl-axon-tools/lib/StudyManifestTools.js | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/packages/mdctl-axon-tools/lib/StudyManifestTools.js b/packages/mdctl-axon-tools/lib/StudyManifestTools.js index 4e9698c5..91c115ad 100644 --- a/packages/mdctl-axon-tools/lib/StudyManifestTools.js +++ b/packages/mdctl-axon-tools/lib/StudyManifestTools.js @@ -691,7 +691,7 @@ class StudyManifestTools { // eslint-disable-next-line no-restricted-syntax for await (const key of manifestKeys) { // Determine whether queriying by c_study or c_key - const property = (study) ? 'c_study' : 'c_key', + const property = (study) ? 'c_study' : first((await org.objects.object.find({ name: key }).paths('uniqueKey').toArray()).map(({ uniqueKey }) => uniqueKey)), // Use the study ID or the entities inside the "includes" array in the manifest values = (study) ? [study._id] : manifestObject[key].includes @@ -710,7 +710,7 @@ class StudyManifestTools { case 'ec__document_template': { // Get the eConsents ID's from the study or the manifest // econsent template properties are namespaced ec__, rather than c_ - const ecProp = property === 'c_study' ? 'ec__study' : 'ec__key' + const ecProp = property === 'c_study' ? 'ec__study' : property ids = (await this.getObjectIDsArray(org, key, ecProp, values)).map(v => v._id) // Load the manifest for the current ID's and their dependencies objectAndDependencies = await this.getConsentManifestEntities(org, ids, orgReferenceProps) @@ -730,8 +730,12 @@ class StudyManifestTools { objectAndDependencies = await this.getGroupManifestEntities(org, ids, orgReferenceProps) break } - case 'c_site': - case 'c_anchor_date_template': { + // Altough c_fault has reference to c_study it will not be included in the below list since that reference is not used + // If we included it c_fault will not get exported in a full study export + case 'c_anchor_date_template': + case 'c_participant_schedule': + case 'c_patient_flag': + case 'c_site': { pluralName = this.mapObjectNameToPlural(key) // These objects seem not to have dependencies so we'll load them directly objectAndDependencies = await this.getExportObjects(org, pluralName, { [property]: { $in: values } }, orgReferenceProps) @@ -744,8 +748,9 @@ class StudyManifestTools { } catch (e) { pluralName = key } - // Export the entire entity (no 'where' filter) if present - objectAndDependencies = await this.getExportObjects(org, pluralName, null, orgReferenceProps) + const where = property !== 'c_study' ? { [property]: { $in: values } } : null + // Allow to export individual instances (if specified) or all of them + objectAndDependencies = await this.getExportObjects(org, pluralName, where, orgReferenceProps) break } } From 7df6fcb8759978adc7fbbe63007d52c95c0f4414 Mon Sep 17 00:00:00 2001 From: Davide Aquaro Date: Thu, 10 Nov 2022 12:54:36 +0000 Subject: [PATCH 3/3] fixed broken test --- .../mdctl-axon-tools/__tests__/MIG-85/MIG-85.test.js | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/mdctl-axon-tools/__tests__/MIG-85/MIG-85.test.js b/packages/mdctl-axon-tools/__tests__/MIG-85/MIG-85.test.js index 54db5873..3de6ced1 100644 --- a/packages/mdctl-axon-tools/__tests__/MIG-85/MIG-85.test.js +++ b/packages/mdctl-axon-tools/__tests__/MIG-85/MIG-85.test.js @@ -5,8 +5,7 @@ jest.mock('@medable/mdctl-api-driver/lib/cortex.object', () => ({ Object: class jest.mock('../../lib/mappings') const fs = require('fs'), - StudyManifestTools = require('../../lib/StudyManifestTools'), - study = require('../../../mdctl-cli/tasks/study') + StudyManifestTools = require('../../lib/StudyManifestTools') describe('MIG-85 - Test partial migrations in StudyManifestTools', () => { @@ -104,6 +103,13 @@ describe('MIG-85 - Test partial migrations in StudyManifestTools', () => { toArray: () => entities.filter(e => e.object === 'c_task') }) }) + }, + object: { + find: () => ({ + paths: () => ({ + toArray: () => [{ uniqueKey: 'c_key' }] + }) + }) } } } @@ -227,6 +233,7 @@ describe('MIG-85 - Test partial migrations in StudyManifestTools', () => { ingestTransform = fs.readFileSync(`${__dirname}/../../packageScripts/ingestTransform.js`).toString() jest.spyOn(StudyManifestTools.prototype, 'getOrgObjectInfo').mockImplementation(() => dummyReferences) + jest.spyOn(StudyManifestTools.prototype, 'getOrgAndReferences').mockImplementation(() => ({ org, dummyReferences })) jest.spyOn(StudyManifestTools.prototype, 'validateReferences').mockImplementation(() => entities) jest.spyOn(StudyManifestTools.prototype, 'createManifest').mockImplementation(() => manifest) jest.spyOn(StudyManifestTools.prototype, 'getObjectIDsArray').mockImplementation(() => entities.filter(o => o.object === 'c_task'))