Skip to content

Commit 9c801a3

Browse files
yolileyolile
committed
feat: add initial component_check_relevant_all_registry component and template
Move field table calculation from cardinal to select_data_from_registry so that the field calculation can be done for multiple files in relevant_all
1 parent 8566670 commit 9c801a3

7 files changed

+1082
-124
lines changed

‎component_check_relevant_all_registry.ipynb

+45
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "markdown",
5+
"source": [
6+
"## Check which publications from the Data Registry passes or not the Relevant criterion\n",
7+
"\n",
8+
"Use this notebook to check which publications in the Data Registry passes the Relevant criterion"
9+
],
10+
"metadata": {
11+
"collapsed": false
12+
}
13+
},
14+
{
15+
"cell_type": "code",
16+
"execution_count": null,
17+
"outputs": [],
18+
"source": [],
19+
"metadata": {
20+
"collapsed": false
21+
}
22+
}
23+
],
24+
"metadata": {
25+
"kernelspec": {
26+
"display_name": "Python 3",
27+
"language": "python",
28+
"name": "python3"
29+
},
30+
"language_info": {
31+
"codemirror_mode": {
32+
"name": "ipython",
33+
"version": 2
34+
},
35+
"file_extension": ".py",
36+
"mimetype": "text/x-python",
37+
"name": "python",
38+
"nbconvert_exporter": "python",
39+
"pygments_lexer": "ipython2",
40+
"version": "2.7.6"
41+
}
42+
},
43+
"nbformat": 4,
44+
"nbformat_minor": 0
45+
}

‎component_select_data_from_registry.ipynb

+14-1
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
{
1616
"cell_type": "markdown",
1717
"source": [
18-
"## Select a publication to download from the [Data Registry](https://data.open-contracting.org/)"
18+
"## Select a publication to download from the [Data Registry](https://data.open-contracting.org/) and generates its field list"
1919
],
2020
"metadata": {
2121
"id": "vpPltvi1tjEs"
@@ -62,6 +62,19 @@
6262
"metadata": {
6363
"collapsed": false
6464
}
65+
},
66+
{
67+
"cell_type": "code",
68+
"execution_count": null,
69+
"outputs": [],
70+
"source": [
71+
"# @title Extract the list of available fields { display-mode: \"form\" }\n",
72+
"\n",
73+
"fields_table = cardinal_calculate_coverage(file_name)"
74+
],
75+
"metadata": {
76+
"collapsed": false
77+
}
6578
}
6679
]
6780
}

‎component_setup_cardinal.ipynb

-13
Original file line numberDiff line numberDiff line change
@@ -64,19 +64,6 @@
6464
" fields_table[\"path\"] = fields_table[\"path\"].str.replace(r\"[][]|^/\", \"\", regex=True)\n",
6565
" return fields_table"
6666
]
67-
},
68-
{
69-
"cell_type": "code",
70-
"execution_count": null,
71-
"outputs": [],
72-
"source": [
73-
"# @title Extract the list of available fields { display-mode: \"form\" }\n",
74-
"\n",
75-
"fields_table = cardinal_calculate_coverage(file_name)"
76-
],
77-
"metadata": {
78-
"collapsed": false
79-
}
8067
}
8168
]
8269
}

‎manage.py

+9-2
Original file line numberDiff line numberDiff line change
@@ -59,18 +59,18 @@
5959
"template_usability_checks_registry": [
6060
"component_environment",
6161
"component_setup_charts",
62+
"component_setup_cardinal",
6263
"component_setup_download_data_from_registry",
6364
"component_select_data_from_registry",
64-
"component_setup_cardinal",
6565
"component_setup_usability",
6666
"component_check_usability_external",
6767
],
6868
"template_relevant_checks_registry": [
6969
"component_environment",
70+
"component_setup_cardinal",
7071
"component_setup_usability",
7172
"component_setup_download_data_from_registry",
7273
"component_select_data_from_registry",
73-
"component_setup_cardinal",
7474
"component_check_relevant",
7575
],
7676
"template_relevant_checks_fieldlist": [
@@ -90,6 +90,13 @@
9090
"component_scope_usability",
9191
"component_check_relevant",
9292
],
93+
"template_relevant_checks_registry_all": [
94+
"component_environment",
95+
"component_setup_cardinal",
96+
"component_setup_usability",
97+
"component_setup_download_data_from_registry",
98+
"component_check_relevant_all_registry",
99+
],
93100
}
94101

95102
BASEDIR = Path(__file__).resolve().parent

‎template_relevant_checks_registry.ipynb

+54-54
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,59 @@
9696
"# set_light_mode()"
9797
]
9898
},
99+
{
100+
"cell_type": "markdown",
101+
"metadata": {
102+
"id": "25lVf3PwsmmV"
103+
},
104+
"source": [
105+
"## Setup Cardinal"
106+
]
107+
},
108+
{
109+
"cell_type": "markdown",
110+
"metadata": {
111+
"id": "6fzwlrSLOF3w"
112+
},
113+
"source": [
114+
"### Install Cardinal"
115+
]
116+
},
117+
{
118+
"cell_type": "markdown",
119+
"metadata": {
120+
"id": "1H6SWBCTDq5G"
121+
},
122+
"source": [
123+
"This notebook uses [Cardinal](https://cardinal.readthedocs.io/en/latest/), a Rust package to calculate red flags and the coverage of OCDS data."
124+
]
125+
},
126+
{
127+
"cell_type": "code",
128+
"execution_count": null,
129+
"metadata": {
130+
"id": "6syz0fkkEdgj"
131+
},
132+
"outputs": [],
133+
"source": [
134+
"# @title Install { display-mode: \"form\" }\n",
135+
"\n",
136+
"! curl -sSOL https://github.com/open-contracting/cardinal-rs/releases/download/0.0.5/ocdscardinal-0.0.5-linux-64-bit.zip\n",
137+
"! unzip -oj ocdscardinal-0.0.5-linux-64-bit.zip ocdscardinal-0.0.5-linux-64-bit/ocdscardinal\n",
138+
"\n",
139+
"def cardinal_calculate_coverage(file_name):\n",
140+
" coverage = !./ocdscardinal coverage $file_name\n",
141+
" fields = (\n",
142+
" pd.DataFrame.from_dict(json.loads(coverage[0]), orient=\"index\", columns=[\"count\"])\n",
143+
" .reset_index()\n",
144+
" .rename(columns={\"index\": \"path\"})\n",
145+
" )\n",
146+
" # Leaves only object members\n",
147+
" fields_table = fields[fields.path.str.contains(\"[a-z]$\")].copy()\n",
148+
" fields_table[\"path\"] = fields_table[\"path\"].str.replace(r\"[][]|^/\", \"\", regex=True)\n",
149+
" return fields_table"
150+
]
151+
},
99152
{
100153
"cell_type": "markdown",
101154
"metadata": {
@@ -810,7 +863,7 @@
810863
"id": "vpPltvi1tjEs"
811864
},
812865
"source": [
813-
"## Select a publication to download from the [Data Registry](https://data.open-contracting.org/)"
866+
"## Select a publication to download from the [Data Registry](https://data.open-contracting.org/) and generates its field list"
814867
]
815868
},
816869
{
@@ -855,59 +908,6 @@
855908
"file_name = download_file(selected_publication, selected_year)"
856909
]
857910
},
858-
{
859-
"cell_type": "markdown",
860-
"metadata": {
861-
"id": "25lVf3PwsmmV"
862-
},
863-
"source": [
864-
"## Setup Cardinal"
865-
]
866-
},
867-
{
868-
"cell_type": "markdown",
869-
"metadata": {
870-
"id": "6fzwlrSLOF3w"
871-
},
872-
"source": [
873-
"### Install Cardinal"
874-
]
875-
},
876-
{
877-
"cell_type": "markdown",
878-
"metadata": {
879-
"id": "1H6SWBCTDq5G"
880-
},
881-
"source": [
882-
"This notebook uses [Cardinal](https://cardinal.readthedocs.io/en/latest/), a Rust package to calculate red flags and the coverage of OCDS data."
883-
]
884-
},
885-
{
886-
"cell_type": "code",
887-
"execution_count": null,
888-
"metadata": {
889-
"id": "6syz0fkkEdgj"
890-
},
891-
"outputs": [],
892-
"source": [
893-
"# @title Install { display-mode: \"form\" }\n",
894-
"\n",
895-
"! curl -sSOL https://github.com/open-contracting/cardinal-rs/releases/download/0.0.5/ocdscardinal-0.0.5-linux-64-bit.zip\n",
896-
"! unzip -oj ocdscardinal-0.0.5-linux-64-bit.zip ocdscardinal-0.0.5-linux-64-bit/ocdscardinal\n",
897-
"\n",
898-
"def cardinal_calculate_coverage(file_name):\n",
899-
" coverage = !./ocdscardinal coverage $file_name\n",
900-
" fields = (\n",
901-
" pd.DataFrame.from_dict(json.loads(coverage[0]), orient=\"index\", columns=[\"count\"])\n",
902-
" .reset_index()\n",
903-
" .rename(columns={\"index\": \"path\"})\n",
904-
" )\n",
905-
" # Leaves only object members\n",
906-
" fields_table = fields[fields.path.str.contains(\"[a-z]$\")].copy()\n",
907-
" fields_table[\"path\"] = fields_table[\"path\"].str.replace(r\"[][]|^/\", \"\", regex=True)\n",
908-
" return fields_table"
909-
]
910-
},
911911
{
912912
"cell_type": "code",
913913
"execution_count": null,

0 commit comments

Comments
 (0)