diff --git a/.github/workflows/get_model_data.js b/.github/workflows/get_model_data.js index 132ad39d..69039126 100644 --- a/.github/workflows/get_model_data.js +++ b/.github/workflows/get_model_data.js @@ -6,123 +6,113 @@ const ignoreDirs = [ '.datalad', '.github', 'docs', 'images', 'weights', 'trained-models-template' ]; -// find all spec.yml files +// Corrected the getAllPaths function to properly join paths const getAllPaths = function(dirPath, arrayOfPaths) { - files = fs.readdirSync(dirPath); + const files = fs.readdirSync(dirPath); arrayOfPaths = arrayOfPaths || []; - + files.forEach(function(file) { - if (fs.statSync(dirPath + "/" + file).isDirectory() && !ignoreDirs.includes(file)) { - arrayOfPaths = getAllPaths(dirPath + "/" + file, arrayOfPaths); - } else if (file === 'spec.yaml') { - arrayOfPaths.push(path.join(dirPath, "/", file)); - } + if (fs.statSync(path.join(dirPath, file)).isDirectory() && !ignoreDirs.includes(file)) { + arrayOfPaths = getAllPaths(path.join(dirPath, file), arrayOfPaths); + } else if (file === 'model_card.yaml') { + arrayOfPaths.push(path.join(dirPath, file)); // Removed extra "/" + } }); - + return arrayOfPaths; -} +}; function findObj(name, arr) { - for (const obj of arr) { - if (obj.name === name) return obj; - } - return null; + for (const obj of arr) { + if (obj.name === name) return obj; + } + return null; } const paths = getAllPaths('.'); const names = []; -models = {}; - -paths.forEach(function(path) { - const doc = yaml.load(fs.readFileSync(path, 'utf8')); - - //create names.yml - const example = doc.model.example.split(' '); - let org; - let modelName; - let version; - let isLink; - let modelType; - for (let i=0; i < example.length; i++) { - const str = example[i]; - if (str.includes(doc.model.model_name)) { - const initCombinedName = str.split('/'); - org = initCombinedName[0]; - modelName = initCombinedName[1]; - version = initCombinedName[2]; - isLink = !(example[i+1] === '--model_type'); - modelType = !isLink ? example[i+2] : 'model'; - break; +const models = {}; // Made 'models' a constant + +paths.forEach(function(filePath) { + const doc = yaml.load(fs.readFileSync(filePath, 'utf8')); + + // Retrieve model details directly + const modelDetails = doc.Model_details; + + // Destructure the relevant fields from modelDetails + const { Organization: org, Model_version: version, Model_type: modelType, More_information: modelName } = modelDetails; + + let orgStruct = findObj(org, names); + if (orgStruct === null) { + orgStruct = { + name: org, + modelNames: [] + }; + names.push(orgStruct); } - } - let orgStruct = findObj(org, names); - if (orgStruct === null) { - orgStruct = { - name: org, - modelNames: [] - }; - names.push(orgStruct); - } - let modelNameStruct = findObj(modelName, orgStruct.modelNames); - if (modelNameStruct === null) { - modelNameStruct = { - name: modelName, - versions: [] - }; - orgStruct.modelNames.push(modelNameStruct); - } - let versionStruct = findObj(version, modelNameStruct.versions); - if (versionStruct === null) { - versionStruct = { - name: version, - modelTypes: [] - }; - versionStruct.isLink = isLink; - modelNameStruct.versions.push(versionStruct); - } - if (!isLink) { + + let modelNameStruct = findObj(modelName, orgStruct.modelNames); + if (modelNameStruct === null) { + modelNameStruct = { + name: modelName, + versions: [] + }; + orgStruct.modelNames.push(modelNameStruct); + } + + let versionStruct = findObj(version, modelNameStruct.versions); + if (versionStruct === null) { + versionStruct = { + name: version, + modelTypes: [] + }; + modelNameStruct.versions.push(versionStruct); + } + versionStruct.modelTypes.push({ - name: modelType + name: modelType }); - } - - // create models.yml - const combined_name = org + '_' + modelName + '_' + version + '_' + modelType; - models[combined_name] = doc.model; - modelCardFields = [ - 'model_details', 'intended_use', 'factors', 'metrics', 'eval_data', 'training_data', 'quant_analyses', 'ethical_considerations', 'caveats_recs' - ]; - for (const field of modelCardFields) { - if (models[combined_name][field] === '') { - models[combined_name][field] = 'Information not provided.'; + + // Create models.yml + const combined_name = [org, modelName, version, modelType].join('_'); + models[combined_name] = doc; + + // Updated modelCardFields to reflect the new structure + const modelCardFields = [ + 'Model_details', 'Intended_use', 'Factors', 'Metrics', 'Evaluation Data', 'Training Data', 'Quantitative Analyses', 'Ethical Considerations', 'Caveats and Recommendations' + ]; + + for (const field of modelCardFields) { + if (models[combined_name][field] === '') { + models[combined_name][field] = 'Information not provided.'; + } } - } - - // create model pages - const permalink = `/${org}/${modelName}/${version}/${modelType}/`; - const page = `--- - layout: model_card - permalink: ${permalink} - combined_name: ${combined_name} - org: ${org} - modelName: ${modelName} - version: ${version} - modelType: ${modelType} + + // Create model pages + const permalink = `/${org}/${modelName}/${version}/${modelType}/`; + const page = `--- +layout: model_card +permalink: ${permalink} +combined_name: ${combined_name} +org: ${org} +modelName: ${modelName} +version: ${version} +modelType: ${modelType} --- - `; - const filename = `./trained-models-template/docs/_pages/${combined_name}.markdown`; - fs.writeFile(filename, page, "utf8", err => { - if (err) console.log(err); - }); +`; + const filename = `./trained-models-template/docs/_pages/${combined_name}.markdown`; + fs.writeFile(filename, page, "utf8", err => { + if (err) console.log(err); + }); }); -// write to files -yamlNames = yaml.dump(names); +// Write to files +const yamlNames = yaml.dump(names); fs.writeFile("./trained-models-template/docs/_data/names.yml", yamlNames, "utf8", err => { - if (err) console.log(err); + if (err) console.log(err); }); -yamlModels = yaml.dump(models); +const yamlModels = yaml.dump(models); fs.writeFile("./trained-models-template/docs/_data/models.yml", yamlModels, "utf8", err => { if (err) console.log(err); -}); \ No newline at end of file +}); diff --git a/.github/workflows/get_model_data.yml b/.github/workflows/get_model_data.yml index 44cccd17..ccefece5 100644 --- a/.github/workflows/get_model_data.yml +++ b/.github/workflows/get_model_data.yml @@ -37,8 +37,8 @@ jobs: - name: Extract files from clone and remove clone run: | - mv ./trained-models-template/docs . - rm -r ./trained-models-template + mv ./trained-models-template/docs/* ./docs + rm -r ./trained-models-template/ - name: Use GitHub Actions' cache to shorten build times and decrease load on servers uses: actions/cache@v2