Add Lectern Client to monorepo #208
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,6 +1,6 @@ | ||
| { | ||
| "compilerOptions": { | ||
| "target": "ESNext", | ||
| "lib": ["ESNext"], | ||
| "module": "CommonJS", | ||
| "moduleResolution": "node", | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,6 +1,6 @@ | ||
| { | ||
| "compilerOptions": { | ||
| "target": "ESNext", | ||
| "lib": ["ESNext"], | ||
| "module": "CommonJS", | ||
| "moduleResolution": "node", | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| # Lectern TypeScript Client | ||
|
|
||
| ## Features: | ||
| - Runs different restrictions validations: regex, range, scripts, required fields, type checks, etc. | ||
| - Transforms the data from string to their proper type. | ||
| - Report validation errors. | ||
| - Fetch dictionaries from the configured lectern service. | ||
| - Provide typed definitions for the dictionary object. | ||
| - Analyze dictionary versions diff. | ||
|
|
||
| ## Usage examples: | ||
| - icgc-argo/argo-clinical [https://github.com/icgc-argo/argo-clinical] |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,52 @@ | ||
| { | ||
| "name": "@overturebio-stack/lectern-client", | ||
joneubank marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
|
||
| "version": "1.5.0", | ||
| "files": [ | ||
| "dist/" | ||
| ], | ||
| "main": "./dist/index.js", | ||
| "typings": "./dist/index.d.ts", | ||
| "description": "TypeScript client to interact with Lectern servers and perform data validation versus Lectern dictionaries.", | ||
| "scripts": { | ||
| "build": "rimraf dist && tsc", | ||
| "test": "nyc mocha --exit --timeout 5000 -r ts-node/register test/**.spec.ts", | ||
| "lint": "tslint -c tslint.json -e node_modules -p tsconfig.json" | ||
| }, | ||
| "repository": { | ||
| "type": "git", | ||
| "url": "https://github.com/overture-stack/lectern.git" | ||
| }, | ||
| "publishConfig": { | ||
| "access": "public" | ||
| }, | ||
| "license": "AGPL-3.0", | ||
| "devDependencies": { | ||
| "@types/chai": "^4.2.16", | ||
| "@types/deep-freeze": "^0.1.2", | ||
| "@types/lodash": "^4.14.195", | ||
| "@types/mocha": "^8.2.2", | ||
| "@types/node": "^12.0.10", | ||
| "@types/node-fetch": "^2.5.10", | ||
| "chai": "^4.3.4", | ||
| "husky": "^6.0.0", | ||
| "mocha": "^8.3.2", | ||
| "prettier": "^2.2.1", | ||
| "pretty-quick": "^3.1.0", | ||
| "rimraf": "^3.0.2", | ||
| "ts-node": "^9.1.1", | ||
| "tslint": "^6.1.3", | ||
| "typedoc": "^0.17.7", | ||
| "typescript": "^5.1.6" | ||
| }, | ||
| "dependencies": { | ||
| "cd": "^0.3.3", | ||
| "common": "workspace:^", | ||
| "deep-freeze": "^0.0.1", | ||
| "lodash": "^4.17.21", | ||
| "node-fetch": "^2.6.1", | ||
| "node-worker-threads-pool": "^1.4.3", | ||
| "promise-tools": "^2.1.0", | ||
| "winston": "^3.3.3" | ||
| }, | ||
| "author": "Ontario Institute for Cancer Research" | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,254 @@ | ||
| /* | ||
| * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved | ||
| * | ||
| * This program and the accompanying materials are made available under the terms of | ||
| * the GNU Affero General Public License v3.0. You should have received a copy of the | ||
| * GNU Affero General Public License along with this program. | ||
| * If not, see <http://www.gnu.org/licenses/>. | ||
| * | ||
| * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY | ||
| * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES | ||
| * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT | ||
| * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, | ||
| * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED | ||
| * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; | ||
| * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER | ||
| * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN | ||
| * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | ||
| */ | ||
|
|
||
| import { restClient } from './schema-rest-client'; | ||
| import { | ||
| SchemasDictionaryDiffs, | ||
| FieldChanges, | ||
| FieldDiff, | ||
| Change, | ||
| ChangeAnalysis, | ||
| ChangeTypeName, | ||
| RestrictionChanges, | ||
| FieldDefinition, | ||
| } from './schema-entities'; | ||
|
|
||
| const isFieldChange = (obj: any): obj is Change => { | ||
| return obj.type !== undefined; | ||
| }; | ||
|
|
||
| const isNestedChange = (obj: any): obj is { [field: string]: FieldChanges } => { | ||
| return obj.type === undefined; | ||
| }; | ||
|
|
||
| const isRestrictionChange = (obj: any): obj is { [field: string]: FieldChanges } => { | ||
| return obj.type === undefined; | ||
| }; | ||
|
|
||
| export const fetchDiffAndAnalyze = async (serviceUrl: string, name: string, fromVersion: string, toVersion: string) => { | ||
| const changes = await restClient.fetchDiff(serviceUrl, name, fromVersion, toVersion); | ||
| return analyzeChanges(changes); | ||
| }; | ||
|
|
||
| export const analyzeChanges = (schemasDiff: SchemasDictionaryDiffs): ChangeAnalysis => { | ||
| const analysis: ChangeAnalysis = { | ||
| fields: { | ||
| addedFields: [], | ||
| renamedFields: [], | ||
| deletedFields: [], | ||
| }, | ||
| isArrayDesignationChanges: [], | ||
| restrictionsChanges: { | ||
| codeList: { | ||
| created: [], | ||
| deleted: [], | ||
| updated: [], | ||
| }, | ||
| regex: { | ||
| updated: [], | ||
| created: [], | ||
| deleted: [], | ||
| }, | ||
| required: { | ||
| updated: [], | ||
| created: [], | ||
| deleted: [], | ||
| }, | ||
| script: { | ||
| updated: [], | ||
| created: [], | ||
| deleted: [], | ||
| }, | ||
| range: { | ||
| updated: [], | ||
| created: [], | ||
| deleted: [], | ||
| }, | ||
| }, | ||
| metaChanges: { | ||
| core: { | ||
| changedToCore: [], | ||
| changedFromCore: [], | ||
| }, | ||
| }, | ||
| valueTypeChanges: [], | ||
| }; | ||
|
|
||
| for (const field of Object.keys(schemasDiff)) { | ||
| const fieldChange: FieldDiff = schemasDiff[field]; | ||
| if (fieldChange) { | ||
| const fieldDiff = fieldChange.diff; | ||
| // if we have type at first level then it's a field add/delete | ||
| if (isFieldChange(fieldDiff)) { | ||
| categorizeFieldChanges(analysis, field, fieldDiff); | ||
| } | ||
|
|
||
| if (isNestedChange(fieldDiff)) { | ||
| if (fieldDiff.meta) { | ||
| categorizeMetaChagnes(analysis, field, fieldDiff.meta); | ||
| } | ||
|
|
||
| if (fieldDiff.restrictions) { | ||
| categorizeRestrictionChanges(analysis, field, fieldDiff.restrictions, fieldChange.after); | ||
| } | ||
|
|
||
| if (fieldDiff.isArray) { | ||
| categorizeFieldArrayDesignationChange(analysis, field, fieldDiff.isArray); | ||
| } | ||
|
|
||
| if (fieldDiff.valueType) { | ||
| categorizerValueTypeChange(analysis, field, fieldDiff.valueType); | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| return analysis; | ||
| }; | ||
|
|
||
| const categorizeFieldArrayDesignationChange = ( | ||
| analysis: ChangeAnalysis, | ||
| field: string, | ||
| changes: { [field: string]: FieldChanges } | Change, | ||
| ) => { | ||
| // changing isArray designation is a relevant change for all cases except if it is created and set to false | ||
| if (!(changes.type === 'created' && changes.data === false)) { | ||
| analysis.isArrayDesignationChanges.push(field); | ||
| } | ||
| }; | ||
|
|
||
| const categorizerValueTypeChange = ( | ||
| analysis: ChangeAnalysis, | ||
| field: string, | ||
| changes: { [field: string]: FieldChanges } | Change, | ||
| ) => { | ||
| analysis.valueTypeChanges.push(field); | ||
| }; | ||
|
|
||
| const categorizeRestrictionChanges = ( | ||
| analysis: ChangeAnalysis, | ||
| field: string, | ||
| restrictionsChange: { [field: string]: FieldChanges } | Change, | ||
| fieldDefinitionAfter?: FieldDefinition, | ||
| ) => { | ||
| const restrictionsToCheck = ['regex', 'script', 'required', 'codeList', 'range']; | ||
|
|
||
| // additions or deletions of a restriction object as whole (i.e. contains 1 or many restrictions within the 'data') | ||
| if (restrictionsChange.type) { | ||
| const createOrAddChange = restrictionsChange as Change; | ||
| const restrictionsData = createOrAddChange.data as any; | ||
|
|
||
| for (const k of restrictionsToCheck) { | ||
| if (restrictionsData[k]) { | ||
| analysis.restrictionsChanges[k as keyof RestrictionChanges][restrictionsChange.type as ChangeTypeName].push({ | ||
| field: field, | ||
| definition: restrictionsData[k], | ||
| } as any); | ||
| } | ||
| } | ||
| return; | ||
| } | ||
|
|
||
| // in case 'restrictions' key was already there but we modified its contents | ||
| const restrictionUpdate = restrictionsChange as { [field: string]: FieldChanges }; | ||
| for (const k of restrictionsToCheck) { | ||
| if (restrictionUpdate[k]) { | ||
| const change = restrictionUpdate[k] as Change; | ||
| // we need the '|| change' in case of nested attributes like ranges | ||
| /* | ||
| "diff": { | ||
| "restrictions": { | ||
| "range": { | ||
| "exclusiveMin": { | ||
| "type": "deleted", | ||
| "data": 0 | ||
| }, | ||
| "max": { | ||
| "type": "updated", | ||
| "data": 200000 | ||
| }, | ||
| "min": { | ||
| "type": "created", | ||
| "data": 0 | ||
| } | ||
| } | ||
| } | ||
| } | ||
| */ | ||
| if (k == 'range' && !change.type) { | ||
| // if the change is nested (type is at min max level) then the boundries were updated only : ex: | ||
| /* | ||
| change = { | ||
| "max" : { | ||
| type: "updated" | ||
| data: "..." | ||
| }, | ||
| "exclusiveMin": { | ||
| type: "deleted" | ||
| data .. | ||
| } | ||
| } | ||
| */ | ||
| const def: any = {}; | ||
| if (Object.keys(change).some((k) => k == 'max' || k == 'min' || k == 'exclusiveMin' || k == 'exclusiveMax')) { | ||
| analysis.restrictionsChanges[k]['updated'].push({ | ||
| field: field, | ||
| // we push the whole range definition since it doesnt make sense to just | ||
| // push one boundary. | ||
| definition: fieldDefinitionAfter?.restrictions?.range, | ||
| }); | ||
| } | ||
| return; | ||
| } | ||
| const definition = change.data || change; | ||
| analysis.restrictionsChanges[k as keyof RestrictionChanges][change.type as ChangeTypeName].push({ | ||
| field: field, | ||
| definition, | ||
| } as any); | ||
| } | ||
| } | ||
| }; | ||
|
|
||
| const categorizeFieldChanges = (analysis: ChangeAnalysis, field: string, changes: Change) => { | ||
| const changeType = changes.type; | ||
| if (changeType == 'created') { | ||
| analysis.fields.addedFields.push({ | ||
| name: field, | ||
| definition: changes.data, | ||
| }); | ||
| } else if (changeType == 'deleted') { | ||
| analysis.fields.deletedFields.push(field); | ||
| } | ||
| }; | ||
|
|
||
| const categorizeMetaChagnes = ( | ||
| analysis: ChangeAnalysis, | ||
| field: string, | ||
| metaChanges: { [field: string]: FieldChanges } | Change, | ||
| ) => { | ||
| // **** meta changes - core *** | ||
| if (metaChanges?.data?.core === true) { | ||
| const changeType = metaChanges.type; | ||
| if (changeType === 'created' || changeType === 'updated') { | ||
| analysis.metaChanges?.core.changedToCore.push(field); | ||
| } else if (changeType === 'deleted') { | ||
| analysis.metaChanges?.core.changedFromCore.push(field); | ||
| } | ||
| } | ||
| }; |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
no need arguments -rf when using rimraf