Skip to content

[DAPS-1515] Integrate repository type system into core endpoints #1543

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 7 commits into
base: feat-DAPS-1513-rust-js-design
Choose a base branch
from
Open

[DAPS-1515] Integrate repository type system into core endpoints #1543

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
1b8a7fc
[DAPS-1515] Refactor legacy Repo class to integrate new repository pa…
AronPerez Jul 29, 2025
4d37418
[DAPS-1515] Address comments
AronPerez Aug 7, 2025
5caa3bf
[DAPS-1515] Enhance endpoints with repository type system integration
AronPerez Jul 29, 2025
d117b5a
[DAPS-1515] Run prettier
AronPerez Jul 29, 2025
66bdb31
[DAPS-1515] Readd globus and metadata
AronPerez Aug 13, 2025
90b0a2a
[DAPS-1515] Make usage of globus and metadata files
AronPerez Aug 13, 2025
50be6a3
[DAPS-1515] Address pr comments
AronPerez Aug 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions core/database/foxx/api/data_router.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ const g_lib = require("./support");
const g_proc = require("./process");
const g_tasks = require("./tasks");
const { UserToken } = require("./lib/user_token");
const { validateRepositorySupportsDataOperations } = require("./repository/validation");

module.exports = router;

Expand Down Expand Up @@ -90,6 +91,13 @@ function recordCreate(client, record, result) {

if (!repo_alloc) throw [g_lib.ERR_NO_ALLOCATION, "No allocation available"];

// Check if repository supports data operations
validateRepositorySupportsDataOperations(
repo_alloc._to,
null,
"Data uploads not supported for metadata-only repository",
);

// Extension setting only apply to managed data
if (record.ext) {
obj.ext_auto = false;
Expand Down
57 changes: 34 additions & 23 deletions core/database/foxx/api/repo.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ const g_db = require("@arangodb").db;
const g_lib = require("./support");
const { errors } = require("@arangodb");
const pathModule = require("./posix_path");
const { RepositoryOps } = require("./repository/operations");
const { Result } = require("./repository/types");

/**
* All DataFed repositories have the following path structure on a POSIX file system
Expand Down Expand Up @@ -37,6 +39,10 @@ const PathType = {
UNKNOWN: "UNKNOWN",
};

/**
* Legacy Repo class for backward compatibility
* Internally uses new repository patterns but maintains old API
*/
class Repo {
// ERROR code
#error = null;
Expand All @@ -47,6 +53,8 @@ class Repo {
// The repo id simply the key prepended with 'repo/'
#repo_id = null;
#repo_key = null;
// Store the repository object using new patterns
#repository = null;

/**
* Constructs a Repo object and checks if the key exists in the database.
Expand All @@ -66,28 +74,19 @@ class Repo {
//
// Will return true if it does and false if it does not.
if (a_key && a_key !== "repo/") {
if (a_key.startsWith("repo/")) {
this.#repo_id = a_key;
this.#repo_key = a_key.slice("repo/".length);
// Use new repository operations to find the repo
const findResult = RepositoryOps.find(a_key);

if (findResult.ok) {
this.#exists = true;
this.#repository = findResult.value;
this.#repo_id = findResult.value.data._id;
this.#repo_key = findResult.value.data._key;
} else {
this.#repo_id = "repo/" + a_key;
this.#repo_key = a_key;
}

// Check if the repo document exists
try {
if (collection.exists(this.#repo_key)) {
this.#exists = true;
} else {
this.#exists = false;
this.#error = g_lib.ERR_NOT_FOUND;
this.#err_msg = "Invalid repo: (" + a_key + "). No record found.";
}
} catch (e) {
this.#exists = false;
this.#error = g_lib.ERR_INTERNAL_FAULT;
this.#err_msg = "Unknown error encountered.";
console.log(e);
this.#error =
findResult.error.code === 404 ? g_lib.ERR_NOT_FOUND : g_lib.ERR_INTERNAL_FAULT;
this.#err_msg = findResult.error.message;
}
}
}
Expand Down Expand Up @@ -126,6 +125,14 @@ class Repo {
return this.#err_msg;
}

/**
* Get the underlying repository object (new pattern)
* @returns {object|null} Repository object or null if not exists
*/
getRepository() {
return this.#repository;
}

/**
* Detect what kind of POSIX path has been provided
*
Expand All @@ -138,13 +145,17 @@ class Repo {
throw [g_lib.ERR_PERM_DENIED, "Repo does not exist " + this.#repo_id];
}

let repo = g_db._document(this.#repo_id);
if (!repo.path) {
const repoData = this.#repository.data;
if (!repoData.path) {
// Metadata-only repos don't have paths
if (repoData.type === "metadata_only") {
return PathType.UNKNOWN;
}
throw [g_lib.ERR_INTERNAL_FAULT, "Repo document is missing path: " + this.#repo_id];
}

// Get and sanitize the repo root path by removing the trailing slash if one exists
let repo_root_path = repo.path.replace(/\/$/, "");
let repo_root_path = repoData.path.replace(/\/$/, "");
let sanitized_path = a_path.replace(/\/$/, "");

// Check if the sanitized path is exactly the repo root path
Expand Down
185 changes: 123 additions & 62 deletions core/database/foxx/api/repo_router.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,20 @@ const joi = require("joi");
const g_db = require("@arangodb").db;
const g_lib = require("./support");
const g_tasks = require("./tasks");
const { validateGlobusConfig, validatePartialGlobusConfig } = require("./repository/validation");
const { RepositoryOps } = require("./repository/operations");

// Helper function to prepare repository data for saving
const prepareRepoData = (obj) => {
// Ensure paths end with / for saving
if (obj.path && !obj.path.endsWith("/")) {
obj.path += "/";
}
Comment on lines +16 to +18
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You are making sure that the .path ends with "/", in the original code, it is also checking to make sure that it begins with "/". Is that check being done somewhere else?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see it is in the validation.js file.

if (obj.exp_path && !obj.exp_path.endsWith("/")) {
obj.exp_path += "/";
}
return obj;
};

module.exports = router;

Expand Down Expand Up @@ -121,28 +135,31 @@ router
g_lib.procInputParam(req.body, "summary", false, obj);
g_lib.procInputParam(req.body, "domain", false, obj);

if (!obj.path.startsWith("/"))
throw [
g_lib.ERR_INVALID_PARAM,
"Repository path must be an absolute path file system path.",
];

if (!obj.path.endsWith("/")) obj.path += "/";

var idx = obj.path.lastIndexOf("/", obj.path.length - 2);
if (obj.path.substr(idx + 1, obj.path.length - idx - 2) != obj._key)
throw [
g_lib.ERR_INVALID_PARAM,
"Last part of repository path must be repository ID suffix (" +
obj._key +
")",
];
Comment on lines -133 to -139
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is contained in the validation.js file, in the validateRepositoryPath function.


if (req.body.exp_path) {
obj.exp_path = req.body.exp_path;
if (!obj.exp_path.endsWith("/")) obj.path += "/";
}

// Validate the configuration
const validationResult = validateGlobusConfig({
id: obj._key,
title: obj.title,
capacity: obj.capacity,
admins: req.body.admins,
pub_key: obj.pub_key,
address: obj.address,
endpoint: obj.endpoint,
domain: obj.domain,
path: obj.path,
exp_path: obj.exp_path,
});

if (!validationResult.ok) {
throw [validationResult.error.code, validationResult.error.message];
}

// Prepare repository data for saving
prepareRepoData(obj);

var repo = g_db.repo.save(obj, {
returnNew: true,
});
Expand Down Expand Up @@ -211,40 +228,41 @@ router
g_lib.procInputParam(req.body, "summary", true, obj);
g_lib.procInputParam(req.body, "domain", true, obj);

if (req.body.path) {
if (!req.body.path.startsWith("/"))
throw [
g_lib.ERR_INVALID_PARAM,
"Repository path must be an absolute path file system path.",
];

obj.path = req.body.path;
if (!obj.path.endsWith("/")) obj.path += "/";

// Last part of storage path MUST end with the repo ID
var idx = obj.path.lastIndexOf("/", obj.path.length - 2);
var key = req.body.id.substr(5);
if (obj.path.substr(idx + 1, obj.path.length - idx - 2) != key)
throw [
g_lib.ERR_INVALID_PARAM,
"Last part of repository path must be repository ID suffix (" +
key +
")",
];
}
if (req.body.path) obj.path = req.body.path;
if (req.body.exp_path) obj.exp_path = req.body.exp_path;
if (req.body.capacity) obj.capacity = req.body.capacity;
if (req.body.pub_key) obj.pub_key = req.body.pub_key;
if (req.body.address) obj.address = req.body.address;
if (req.body.endpoint) obj.endpoint = req.body.endpoint;

if (req.body.exp_path) {
obj.exp_path = req.body.exp_path;
if (!obj.exp_path.endsWith("/")) obj.exp_path += "/";
}
// Extract repo key from ID for validation
const key = req.body.id.substr(5);

if (req.body.capacity) obj.capacity = req.body.capacity;
// Validate the partial configuration
const updateConfig = {
title: req.body.title,
domain: req.body.domain,
path: req.body.path,
exp_path: req.body.exp_path,
capacity: req.body.capacity,
pub_key: req.body.pub_key,
address: req.body.address,
endpoint: req.body.endpoint,
admins: req.body.admins,
};

if (req.body.pub_key) obj.pub_key = req.body.pub_key;
// Remove undefined fields
Object.keys(updateConfig).forEach(
(k) => updateConfig[k] === undefined && delete updateConfig[k],
);

if (req.body.address) obj.address = req.body.address;
const validationResult = validatePartialGlobusConfig(updateConfig, key);
if (!validationResult.ok) {
throw [validationResult.error.code, validationResult.error.message];
}

if (req.body.endpoint) obj.endpoint = req.body.endpoint;
// Prepare repository data for saving
prepareRepoData(obj);

var repo = g_db._update(req.body.id, obj, {
returnNew: true,
Expand Down Expand Up @@ -672,15 +690,37 @@ router
subject_id = req.queryParams.subject;
else subject_id = g_lib.getUserFromClientID(req.queryParams.subject)._id;

var result = g_tasks.taskInitAllocCreate(
client,
req.queryParams.repo,
subject_id,
req.queryParams.data_limit,
req.queryParams.rec_limit,
// Find the repository using the new type system
const findResult = RepositoryOps.find(req.queryParams.repo);
if (!findResult.ok) {
throw [findResult.error.code, findResult.error.message];
}

const repository = findResult.value;

// Check permissions
const permResult = RepositoryOps.checkPermission(
repository,
client._id,
"admin",
);
if (!permResult.ok || !permResult.value) {
throw g_lib.ERR_PERM_DENIED;
}

res.send(result);
// Create allocation using the new system
const allocResult = RepositoryOps.createAllocation(repository, {
subject: subject_id,
size: req.queryParams.data_limit,
rec_limit: req.queryParams.rec_limit,
});

if (!allocResult.ok) {
throw [allocResult.error.code, allocResult.error.message];
}

// Return the new response format
res.send(allocResult.value);
},
});
} catch (e) {
Expand All @@ -702,7 +742,7 @@ router
)
.summary("Create user/project repo allocation")
.description(
"Create user repo/project allocation. Only repo admin can set allocations. Returns a task document.",
"Create user repo/project allocation. Only repo admin can set allocations. Returns either a task (for Globus repos) or direct result (for metadata-only repos).",
);

router
Expand All @@ -722,13 +762,34 @@ router
subject_id = req.queryParams.subject;
else subject_id = g_lib.getUserFromClientID(req.queryParams.subject)._id;

var result = g_tasks.taskInitAllocDelete(
client,
req.queryParams.repo,
subject_id,
);
// Find the repository using the new type system
var findResult = RepositoryOps.find(req.queryParams.repo);
if (!findResult.ok) {
throw [findResult.error.code, findResult.error.message];
}

var repository = findResult.value;

// Check permissions
var permResult = RepositoryOps.checkPermission(repository, client._id, "admin");
if (!permResult.ok || !permResult.value) {
throw g_lib.ERR_PERM_DENIED;
}

// Check if subject exists
if (!g_db._exists(subject_id)) {
throw [g_lib.ERR_NOT_FOUND, "Subject not found: " + subject_id];
Comment on lines +775 to +781
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It would be clearer if we include in the exeception, that the allocation delete request failed, for audit reasons, in cases around authorization it is really useful to know who attempted to do what.

}

// Delete allocation using the new system
var deleteResult = RepositoryOps.deleteAllocation(repository, subject_id);

if (!deleteResult.ok) {
throw [deleteResult.error.code, deleteResult.error.message];
}

res.send(result);
// Return the new response format
res.send(deleteResult.value);
},
});
} catch (e) {
Expand All @@ -740,7 +801,7 @@ router
.queryParam("repo", joi.string().required(), "Repo ID")
.summary("Delete user/project repo allocation")
.description(
"Delete user repo/project allocation. Only repo admin can set allocations. Returns a task document.",
"Delete user repo/project allocation. Only repo admin can set allocations. Returns either a task (for Globus repos) or direct result (for metadata-only repos).",
);

router
Expand Down
2 changes: 0 additions & 2 deletions core/database/foxx/api/repository/factory.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@ const g_lib = require("../support");
* @param {string} [config.pub_key] - Public key for ZeroMQ CURVE authentication (required for GLOBUS type)
* @param {string} [config.address] - Network address (required for GLOBUS type)
* @param {string} [config.exp_path] - Export path (optional for GLOBUS type)
* @param {string} [config.domain] - Domain name (required for GLOBUS type)
* @returns {{ok: boolean, error: *}|{ok: boolean, value: *}} Result object containing repository or error
* @see https://doc.rust-lang.org/book/ch06-02-match.html
*/
Expand Down Expand Up @@ -69,7 +68,6 @@ const createRepositoryByType = (config) => {
pub_key: config.pub_key,
address: config.address,
exp_path: config.exp_path,
domain: config.domain,
});

const repoData = createRepositoryData({
Expand Down
Loading