Skip to content

Add column for last commit date, allow for subset of columns to be shown, overhaul sorting #392

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
yarikoptic wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Add column for last commit date, allow for subset of columns to be shown, overhaul sorting #392

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7b48e79
Add last commit date column and configurable columns support
yarikoptic Sep 4, 2025
8d914f2
Refactor sorting system and remove _SORT_ATTRS duplication
yarikoptic Sep 5, 2025
88db67b
RF: no need for _DEFAULT_COLUMNS
yarikoptic Sep 5, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 68 additions & 21 deletions datalad_registry/overview.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,19 +14,39 @@
lgr = logging.getLogger(__name__)
bp = Blueprint("overview", __name__, url_prefix="/overview")

_SORT_ATTRS = {
"keys-asc": ("annex_key_count", "asc"),
"keys-desc": ("annex_key_count", "desc"),
"update-asc": ("last_update_dt", "asc"),
"update-desc": ("last_update_dt", "desc"),
"url-asc": ("url", "asc"),
"url-desc": ("url", "desc"),
"annexed_files_in_wt_count-asc": ("annexed_files_in_wt_count", "asc"),
"annexed_files_in_wt_count-desc": ("annexed_files_in_wt_count", "desc"),
"annexed_files_in_wt_size-asc": ("annexed_files_in_wt_size", "asc"),
"annexed_files_in_wt_size-desc": ("annexed_files_in_wt_size", "desc"),
"git_objects_kb-asc": ("git_objects_kb", "asc"),
"git_objects_kb-desc": ("git_objects_kb", "desc"),
# _SORT_ATTRS removed - now using _AVAILABLE_COLUMNS with db_field mapping

# Available columns with their metadata
_AVAILABLE_COLUMNS = {
"url": {"label": "URL", "sortable": True, "db_field": "url"},
"dataset": {"label": "Dataset", "sortable": False},
"commit": {"label": "Commit", "sortable": False},
"head_dt": {"label": "Last commit date", "sortable": True, "db_field": "head_dt"},
"keys": {
"label": "Annex keys",
"sortable": True,
"tooltip": "Number of annex keys",
"db_field": "annex_key_count",
},
"annexed_files_count": {
"label": "Nr of Annexed files",
"sortable": True,
"tooltip": "Number of annexed files in working tree",
"db_field": "annexed_files_in_wt_count",
},
"annexed_files_size": {
"label": "Size of Annexed files",
"sortable": True,
"tooltip": "Size of annexed files in working tree",
"db_field": "annexed_files_in_wt_size",
},
"update": {"label": "Last update", "sortable": True, "db_field": "last_update_dt"},
"git_objects": {
"label": "Size of .git/objects",
"sortable": True,
"db_field": "git_objects_kb",
},
"metadata": {"label": "Metadata", "sortable": False},
}


Expand All @@ -36,7 +56,8 @@

@bp.get("/")
def overview(): # No type hints due to mypy#7187.
default_sort_scheme = "update-desc"
default_sort_column = "update"
default_sort_direction = "desc"

base_select_stmt = select(RepoUrl)

Expand All @@ -54,12 +75,34 @@ def overview(): # No type hints due to mypy#7187.
else:
base_select_stmt = base_select_stmt.filter(criteria)

# Decipher sorting scheme
sort_by = request.args.get("sort", default_sort_scheme, type=str)
if sort_by not in _SORT_ATTRS:
lgr.debug("Ignoring unknown sort parameter: %s", sort_by)
sort_by = default_sort_scheme
col, sort_method = _SORT_ATTRS[sort_by]
# Handle configurable columns
columns_param = request.args.get("columns", None, type=str)
if columns_param:
# Parse comma-separated column names
requested_columns = [col.strip() for col in columns_param.split(",")]
# Filter out invalid column names
visible_columns = [
col for col in requested_columns if col in _AVAILABLE_COLUMNS
]
if not visible_columns:
visible_columns = list(_AVAILABLE_COLUMNS)
else:
visible_columns = list(_AVAILABLE_COLUMNS)

# Handle sorting using new system
sort_by_param = request.args.get("sort_by", default_sort_column, type=str)
sort_direction = request.args.get("sort", default_sort_direction, type=str)
# Validate sort_by parameter
if sort_by_param in _AVAILABLE_COLUMNS and _AVAILABLE_COLUMNS[sort_by_param].get(
"sortable"
):
col = _AVAILABLE_COLUMNS[sort_by_param]["db_field"]
sort_method = sort_direction if sort_direction in ["asc", "desc"] else "asc"
else:
lgr.debug("Ignoring unknown sort_by parameter: %s", sort_by_param)
sort_by_param = default_sort_column
col = _AVAILABLE_COLUMNS[default_sort_column]["db_field"]
sort_method = default_sort_direction

# Apply sorting
select_stmt = base_select_stmt.order_by(
Expand All @@ -76,7 +119,11 @@ def overview(): # No type hints due to mypy#7187.
"overview.html",
pagination=pagination,
stats=stats,
sort_by=sort_by,
sort_by_param=sort_by_param,
sort_direction=sort_method, # Use the validated sort_method
search_query=query,
search_error=search_error,
visible_columns=visible_columns,
available_columns=_AVAILABLE_COLUMNS,
columns_param=columns_param,
)
2 changes: 2 additions & 0 deletions datalad_registry/tasks/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
from datalad_registry.utils import StrEnum
from datalad_registry.utils.datalad_tls import (
clone,
get_head_commit_date,
get_head_describe,
get_origin_annex_key_count,
get_origin_annex_uuid,
Expand Down Expand Up @@ -100,6 +101,7 @@ def _update_dataset_url_info(dataset_url: RepoUrl, ds: Dataset) -> None:

dataset_url.head = ds.repo.get_hexsha("origin/HEAD")
dataset_url.head_describe = get_head_describe(ds)
dataset_url.head_dt = get_head_commit_date(ds)

dataset_url.branches = get_origin_branches(ds)

Expand Down
117 changes: 60 additions & 57 deletions datalad_registry/templates/overview.html
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{%- macro sort_href(default, other) -%}
{%- if sort_by == default -%}
{%- set new_sort = other -%}
{%- macro sort_href(column_name, current_sort_by, current_direction) -%}
{%- if current_sort_by == column_name -%}
{%- set new_direction = 'asc' if current_direction == 'desc' else 'desc' -%}
{%- else -%}
{%- set new_sort = default -%}
{%- set new_direction = 'desc' -%}
{%- endif -%}
href="{{ url_for('.overview', query=search_query, sort='{}'.format(new_sort)) }}"
href="{{ url_for('.overview', query=search_query, sort_by=column_name, sort=new_direction, columns=columns_param) }}"
{%- endmacro -%}

{% macro render_pagination_widget(pagination, endpoint) %}
Expand All @@ -16,7 +16,7 @@
{% for page in pagination.iter_pages() %}
{% if page %}
{% if page != pagination.page %}
<a href="{{ url_for(endpoint, page=page, per_page=pagination.per_page, sort=sort_by, query=search_query) }}">{{ page }}</a>
<a href="{{ url_for(endpoint, page=page, per_page=pagination.per_page, sort_by=sort_by_param, sort=sort_direction, query=search_query, columns=columns_param) }}">{{ page }}</a>
{% else %}
<strong>{{ page }}</strong>
{% endif %}
Expand Down Expand Up @@ -60,7 +60,11 @@
{%- endif -%}
placeholder='Search query'
/>
<input type="hidden" name="sort" value="{{ sort_by }}"/>
<input type="hidden" name="sort_by" value="{{ sort_by_param }}"/>
<input type="hidden" name="sort" value="{{ sort_direction }}"/>
{% if columns_param %}
<input type="hidden" name="columns" value="{{ columns_param }}"/>
{% endif %}
<input type='submit' value='Search'/>
{% if search_error -%}
<p><span class="error">ERROR: {{ search_error }}</span></p>
Expand All @@ -69,7 +73,11 @@
{% if search_query -%}
<!-- Clear search query via Button for a unified UI -->
<form action="{{ url_for('.overview') }}" method="get">
<input type="hidden" name="sort" value="{{ sort_by }}"/>
<input type="hidden" name="sort_by" value="{{ sort_by_param }}"/>
<input type="hidden" name="sort" value="{{ sort_direction }}"/>
{% if columns_param %}
<input type="hidden" name="columns" value="{{ columns_param }}"/>
{% endif %}
<input type='submit' value='Clear'/>
</form>
{%- endif %}
Expand Down Expand Up @@ -132,58 +140,53 @@ <h2>Search query syntax</h2>

<table class="list">
<tr>
<th><a {{ sort_href('url-desc', 'url-asc') }}>URL</a></th>
<th>Dataset</th>
<th>Commit</th>
<th><a {{ sort_href('keys-desc', 'keys-asc') }}>Annex keys</a></th>
<th class="tooltip">
<a {{ sort_href('annexed_files_in_wt_count-desc', 'annexed_files_in_wt_count-asc') }}>
Nr of Annexed files
</a>
<span class="tooltiptext">Number of annexed files in working tree</span>
</th>
<th class="tooltip">
<a {{ sort_href('annexed_files_in_wt_size-desc', 'annexed_files_in_wt_size-asc') }}>
Size of Annexed files
</a>
<span class="tooltiptext">Size of annexed files in working tree</span>
</th>
<th>
<a {{ sort_href('update-desc', 'update-asc') }}>Last update</a>
</th>
<th class="tooltip">
<a {{ sort_href('git_objects_kb-desc', 'git_objects_kb-asc') }}>
Size of .git/objects
</a>
</th>
<th>
Metadata
</th>
{%- for column in visible_columns -%}
<th{%- if available_columns[column].get('tooltip') %} class="tooltip"{%- endif %}>
{%- if available_columns[column]['sortable'] -%}
<a {{ sort_href(column, sort_by_param, sort_direction) }}>{{ available_columns[column]['label'] }}</a>
{%- else -%}
{{ available_columns[column]['label'] }}
{%- endif -%}
{%- if available_columns[column].get('tooltip') -%}
<span class="tooltiptext">{{ available_columns[column]['tooltip'] }}</span>
{%- endif -%}
</th>
{%- endfor -%}
</tr>
{%- for i in pagination -%}
<tr>
<td><a href="{{ i.url }}">{{ i.url }}</a></td>
<td class="mono">
{% if i.ds_id is not none %}
<a href="{{ url_for('.overview', query='ds_id:' + i.ds_id) }}">{{ i.ds_id }}</a>
{% endif %}
</td>
<td class="mono">
{{ i.head_describe if i.head_describe is not none }}
</td>
<td>{{ i.annex_key_count|intcomma if i.annex_key_count is not none }}</td>
<td>{{ i.annexed_files_in_wt_count|intcomma if i.annexed_files_in_wt_count is not none }}</td>
<td>{{ i.annexed_files_in_wt_size|filesizeformat if i.annexed_files_in_wt_size is not none }}</td>
<td>{{ i.last_update_dt.strftime("%Y-%m-%dT%H:%M:%S%z") if i.last_update_dt is not none }}</td>
<td>{{ (i.git_objects_kb * 1024)|filesizeformat if i.git_objects_kb is not none }}</td>
<td>
{%- for data in i.metadata_ -%}
<a href="{{ url_for('url_metadata_api.url_metadata', url_metadata_id=data.id) }}">
{{ data.extractor_name }}
</a>
{{ ", " if not loop.last else "" }}
{%- endfor -%}
</td>
{%- for column in visible_columns -%}
<td{%- if column in ['dataset', 'commit'] %} class="mono"{%- endif %}>
{%- if column == 'url' -%}
<a href="{{ i.url }}">{{ i.url }}</a>
{%- elif column == 'dataset' -%}
{% if i.ds_id is not none %}
<a href="{{ url_for('.overview', query='ds_id:' + i.ds_id, columns=columns_param) }}">{{ i.ds_id }}</a>
{% endif %}
{%- elif column == 'commit' -%}
{{ i.head_describe if i.head_describe is not none }}
{%- elif column == 'head_dt' -%}
{{ i.head_dt.strftime("%Y-%m-%dT%H:%M:%S%z") if i.head_dt is not none }}
{%- elif column == 'keys' -%}
{{ i.annex_key_count|intcomma if i.annex_key_count is not none }}
{%- elif column == 'annexed_files_count' -%}
{{ i.annexed_files_in_wt_count|intcomma if i.annexed_files_in_wt_count is not none }}
{%- elif column == 'annexed_files_size' -%}
{{ i.annexed_files_in_wt_size|filesizeformat if i.annexed_files_in_wt_size is not none }}
{%- elif column == 'update' -%}
{{ i.last_update_dt.strftime("%Y-%m-%dT%H:%M:%S%z") if i.last_update_dt is not none }}
{%- elif column == 'git_objects' -%}
{{ (i.git_objects_kb * 1024)|filesizeformat if i.git_objects_kb is not none }}
{%- elif column == 'metadata' -%}
{%- for data in i.metadata_ -%}
<a href="{{ url_for('url_metadata_api.url_metadata', url_metadata_id=data.id) }}">
{{ data.extractor_name }}
</a>
{{ ", " if not loop.last else "" }}
{%- endfor -%}
{%- endif -%}
</td>
{%- endfor -%}
</tr>
{%- endfor -%}
</table>
Expand Down
Loading
Loading