Skip to content

Commit

Permalink
Add csvw task.
Browse files Browse the repository at this point in the history
Issue #31 - Update each pipeline to output CSVW to the specified
directory instead of generating the output RDF directly. Update
the exec task to write the CSVW to a temporary directory and
write the RDF using csv2rdf. The cube pipeline now only creates
a single metadata file instead of one for each element (e.g. dataset
metadata, DSD etc.). Update each metadata function in the cube
namespace to return a table definition instead of a full metadata
definition by removing the @context key.

Add new csvw task which writes the CSVW output for each pipeline
to the specified directory.

Update the output of the describe task to include an example
invocation of the csvw command.
  • Loading branch information
lkitching committed Apr 16, 2020
1 parent 804aa86 commit db763e2
Show file tree
Hide file tree
Showing 29 changed files with 272 additions and 226 deletions.
2 changes: 1 addition & 1 deletion examples/bop-quarterly/csvw/component-specifications.json
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"@context":["http:\/\/www.w3.org\/ns\/csvw",{"@language":"en"}],"url":"component-specifications.csv","dc:title":"BoP Quarterly Example","tableSchema":{"columns":[{"name":"component_slug","titles":"component_slug","datatype":"string","suppressOutput":true},{"name":"component_attachment","titles":"component_attachment","datatype":"string","suppressOutput":true},{"name":"component_property","titles":"component_property","datatype":"string","propertyUrl":"{+component_attachment}","valueUrl":"{+component_property}"},{"name":"type","virtual":true,"propertyUrl":"rdf:type","valueUrl":"qb:ComponentSpecification"},{"name":"codes_used","virtual":true,"propertyUrl":"http:\/\/publishmydata.com\/def\/qb\/codesUsed","valueUrl":"http:\/\/gss-data.org.uk\/data\/bop-quarterly-example\/codes-used\/{component_slug}"}],"aboutUrl":"http:\/\/gss-data.org.uk\/data\/bop-quarterly-example\/component\/{component_slug}"}}
{"url":"component-specifications.csv","dc:title":"BoP Quarterly Example","tableSchema":{"columns":[{"name":"component_slug","titles":"component_slug","datatype":"string","suppressOutput":true},{"name":"component_attachment","titles":"component_attachment","datatype":"string","suppressOutput":true},{"name":"component_property","titles":"component_property","datatype":"string","propertyUrl":"{+component_attachment}","valueUrl":"{+component_property}"},{"name":"type","virtual":true,"propertyUrl":"rdf:type","valueUrl":"qb:ComponentSpecification"},{"name":"codes_used","virtual":true,"propertyUrl":"http:\/\/publishmydata.com\/def\/qb\/codesUsed","valueUrl":"http:\/\/gss-data.org.uk\/data\/bop-quarterly-example\/codes-used\/{component_slug}"}],"aboutUrl":"http:\/\/gss-data.org.uk\/data\/bop-quarterly-example\/component\/{component_slug}"}}
2 changes: 1 addition & 1 deletion examples/bop-quarterly/csvw/data-structure-definition.json
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"@context":["http:\/\/www.w3.org\/ns\/csvw",{"@language":"en"}],"@id":"http:\/\/gss-data.org.uk\/data\/bop-quarterly-example\/structure","url":"component-specifications.csv","dc:title":"BoP Quarterly Example (Data Structure Definition)","rdf:type":{"@id":"qb:DataStructureDefinition"},"rdfs:label":"BoP Quarterly Example (Data Structure Definition)","tableSchema":{"columns":[{"name":"component_slug","titles":"component_slug","datatype":"string","propertyUrl":"qb:component","valueUrl":"http:\/\/gss-data.org.uk\/data\/bop-quarterly-example\/component\/{component_slug}"},{"name":"component_attachment","titles":"component_attachment","datatype":"string","suppressOutput":true},{"name":"component_property","titles":"component_property","datatype":"string","suppressOutput":true}],"aboutUrl":"http:\/\/gss-data.org.uk\/data\/bop-quarterly-example\/structure"}}
{"@id":"http:\/\/gss-data.org.uk\/data\/bop-quarterly-example\/structure","url":"component-specifications.csv","dc:title":"BoP Quarterly Example (Data Structure Definition)","rdf:type":{"@id":"qb:DataStructureDefinition"},"rdfs:label":"BoP Quarterly Example (Data Structure Definition)","tableSchema":{"columns":[{"name":"component_slug","titles":"component_slug","datatype":"string","propertyUrl":"qb:component","valueUrl":"http:\/\/gss-data.org.uk\/data\/bop-quarterly-example\/component\/{component_slug}"},{"name":"component_attachment","titles":"component_attachment","datatype":"string","suppressOutput":true},{"name":"component_property","titles":"component_property","datatype":"string","suppressOutput":true}],"aboutUrl":"http:\/\/gss-data.org.uk\/data\/bop-quarterly-example\/structure"}}
2 changes: 1 addition & 1 deletion examples/bop-quarterly/csvw/dataset.json
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"@context":["http:\/\/www.w3.org\/ns\/csvw",{"@language":"en"}],"@id":"http:\/\/gss-data.org.uk\/data\/bop-quarterly-example","url":"component-specifications.csv","dc:title":"BoP Quarterly Example","tableSchema":{"columns":[{"name":"component_slug","titles":"component_slug","suppressOutput":true},{"name":"component_attachment","titles":"component_attachment","suppressOutput":true},{"name":"component_property","titles":"component_property","suppressOutput":true},{"name":"type","virtual":true,"propertyUrl":"rdf:type","valueUrl":"qb:DataSet"},{"name":"structure","virtual":true,"propertyUrl":"qb:structure","valueUrl":"http:\/\/gss-data.org.uk\/data\/bop-quarterly-example\/structure"}],"aboutUrl":"http:\/\/gss-data.org.uk\/data\/bop-quarterly-example"}}
{"@id":"http:\/\/gss-data.org.uk\/data\/bop-quarterly-example","url":"component-specifications.csv","dc:title":"BoP Quarterly Example","tableSchema":{"columns":[{"name":"component_slug","titles":"component_slug","suppressOutput":true},{"name":"component_attachment","titles":"component_attachment","suppressOutput":true},{"name":"component_property","titles":"component_property","suppressOutput":true},{"name":"type","virtual":true,"propertyUrl":"rdf:type","valueUrl":"qb:DataSet"},{"name":"structure","virtual":true,"propertyUrl":"qb:structure","valueUrl":"http:\/\/gss-data.org.uk\/data\/bop-quarterly-example\/structure"}],"aboutUrl":"http:\/\/gss-data.org.uk\/data\/bop-quarterly-example"}}
2 changes: 1 addition & 1 deletion examples/bop-quarterly/csvw/observations.json
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"@context":["http:\/\/www.w3.org\/ns\/csvw",{"@language":"en"}],"url":"observations.csv","tableSchema":{"columns":[{"name":"geography","titles":"geography","datatype":"string","propertyUrl":"http:\/\/purl.org\/linked-data\/sdmx\/2009\/dimension#refArea","valueUrl":"http:\/\/statistics.data.gov.uk\/id\/statistical-geography\/{geography}"},{"name":"period","titles":"period","datatype":"string","propertyUrl":"http:\/\/purl.org\/linked-data\/sdmx\/2009\/dimension#refPeriod","valueUrl":"http:\/\/reference.data.gov.uk\/id\/{+period}"},{"name":"cdid","titles":"cdid","datatype":"string","propertyUrl":"http:\/\/gss-data.org.uk\/def\/attribute\/cdid"},{"name":"services","titles":"services","datatype":"string","propertyUrl":"http:\/\/gss-data.org.uk\/def\/dimension\/service","valueUrl":"http:\/\/gss-data.org.uk\/def\/concept\/services\/{service}"},{"name":"flow","titles":"flow","datatype":"string","propertyUrl":"http:\/\/gss-data.org.uk\/def\/dimension\/flow","valueUrl":"http:\/\/gss-data.org.uk\/def\/concept\/flow-directions\/{flow}"},{"name":"measure_type","titles":"measure_type","datatype":"string","propertyUrl":"http:\/\/purl.org\/linked-data\/cube#measureType","valueUrl":"http:\/\/gss-data.org.uk\/def\/measure\/{measure_type}"},{"name":"value","titles":"value","datatype":"number","propertyUrl":"http:\/\/gss-data.org.uk\/def\/measure\/{measure_type}"},{"name":"unit","titles":"unit","datatype":"string","propertyUrl":"http:\/\/purl.org\/linked-data\/sdmx\/2009\/attribute#unitMeasure","valueUrl":"http:\/\/gss-data.org.uk\/def\/concept\/measurement-units\/{unit}"},{"name":"Observation","virtual":true,"propertyUrl":"rdf:type","valueUrl":"qb:Observation"},{"name":"DataSet","virtual":true,"propertyUrl":"qb:dataSet","valueUrl":"http:\/\/gss-data.org.uk\/data\/bop-quarterly-example"}],"aboutUrl":"http:\/\/gss-data.org.uk\/data\/bop-quarterly-example\/{+geography}\/{+period}\/{+cdid}\/{+services}\/{+flow}\/{+measure_type}\/{+unit}"}}
{"url":"observations.csv","tableSchema":{"columns":[{"name":"geography","titles":"geography","datatype":"string","propertyUrl":"http:\/\/purl.org\/linked-data\/sdmx\/2009\/dimension#refArea","valueUrl":"http:\/\/statistics.data.gov.uk\/id\/statistical-geography\/{geography}"},{"name":"period","titles":"period","datatype":"string","propertyUrl":"http:\/\/purl.org\/linked-data\/sdmx\/2009\/dimension#refPeriod","valueUrl":"http:\/\/reference.data.gov.uk\/id\/{+period}"},{"name":"cdid","titles":"cdid","datatype":"string","propertyUrl":"http:\/\/gss-data.org.uk\/def\/attribute\/cdid"},{"name":"services","titles":"services","datatype":"string","propertyUrl":"http:\/\/gss-data.org.uk\/def\/dimension\/service","valueUrl":"http:\/\/gss-data.org.uk\/def\/concept\/services\/{service}"},{"name":"flow","titles":"flow","datatype":"string","propertyUrl":"http:\/\/gss-data.org.uk\/def\/dimension\/flow","valueUrl":"http:\/\/gss-data.org.uk\/def\/concept\/flow-directions\/{flow}"},{"name":"measure_type","titles":"measure_type","datatype":"string","propertyUrl":"http:\/\/purl.org\/linked-data\/cube#measureType","valueUrl":"http:\/\/gss-data.org.uk\/def\/measure\/{measure_type}"},{"name":"value","titles":"value","datatype":"number","propertyUrl":"http:\/\/gss-data.org.uk\/def\/measure\/{measure_type}"},{"name":"unit","titles":"unit","datatype":"string","propertyUrl":"http:\/\/purl.org\/linked-data\/sdmx\/2009\/attribute#unitMeasure","valueUrl":"http:\/\/gss-data.org.uk\/def\/concept\/measurement-units\/{unit}"},{"name":"Observation","virtual":true,"propertyUrl":"rdf:type","valueUrl":"qb:Observation"},{"name":"DataSet","virtual":true,"propertyUrl":"qb:dataSet","valueUrl":"http:\/\/gss-data.org.uk\/data\/bop-quarterly-example"}],"aboutUrl":"http:\/\/gss-data.org.uk\/data\/bop-quarterly-example\/{+geography}\/{+period}\/{+cdid}\/{+services}\/{+flow}\/{+measure_type}\/{+unit}"}}
2 changes: 1 addition & 1 deletion examples/bop-quarterly/csvw/used-codes-codelists.json
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"@context":["http:\/\/www.w3.org\/ns\/csvw",{"@language":"en"}],"url":"component-specifications.csv","tableSchema":{"columns":[{"name":"component_slug","titles":"component_slug","datatype":"string","suppressOutput":true},{"name":"component_attachment","titles":"component_attachment","datatype":"string","suppressOutput":true},{"name":"component_property","titles":"component_property","datatype":"string","suppressOutput":true},{"name":"type","virtual":true,"propertyUrl":"rdf:type","valueUrl":"skos:ConceptScheme"}],"aboutUrl":"http:\/\/gss-data.org.uk\/data\/bop-quarterly-example\/codes-used\/{component_slug}"}}
{"url":"component-specifications.csv","tableSchema":{"columns":[{"name":"component_slug","titles":"component_slug","datatype":"string","suppressOutput":true},{"name":"component_attachment","titles":"component_attachment","datatype":"string","suppressOutput":true},{"name":"component_property","titles":"component_property","datatype":"string","suppressOutput":true},{"name":"type","virtual":true,"propertyUrl":"rdf:type","valueUrl":"skos:ConceptScheme"}],"aboutUrl":"http:\/\/gss-data.org.uk\/data\/bop-quarterly-example\/codes-used\/{component_slug}"}}
2 changes: 1 addition & 1 deletion examples/bop-quarterly/csvw/used-codes-codes.json
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"@context":["http:\/\/www.w3.org\/ns\/csvw",{"@language":"en"}],"url":"observations.csv","tableSchema":{"columns":[{"name":"geography","titles":"geography","datatype":"string","propertyUrl":"skos:member","valueUrl":"http:\/\/statistics.data.gov.uk\/id\/statistical-geography\/{geography}"},{"name":"period","titles":"period","datatype":"string","propertyUrl":"skos:member","valueUrl":"http:\/\/reference.data.gov.uk\/id\/{+period}"},{"name":"cdid","titles":"cdid","datatype":"string","propertyUrl":"skos:member"},{"name":"services","titles":"services","datatype":"string","propertyUrl":"skos:member","valueUrl":"http:\/\/gss-data.org.uk\/def\/concept\/services\/{service}"},{"name":"flow","titles":"flow","datatype":"string","propertyUrl":"skos:member","valueUrl":"http:\/\/gss-data.org.uk\/def\/concept\/flow-directions\/{flow}"},{"name":"measure_type","titles":"measure_type","datatype":"string","propertyUrl":"skos:member","valueUrl":"http:\/\/gss-data.org.uk\/def\/measure\/{measure_type}"},{"name":"value","titles":"value","datatype":"number","propertyUrl":"skos:member","suppressOutput":true},{"name":"unit","titles":"unit","datatype":"string","propertyUrl":"skos:member","valueUrl":"http:\/\/gss-data.org.uk\/def\/concept\/measurement-units\/{unit}"}],"aboutUrl":"http:\/\/gss-data.org.uk\/data\/bop-quarterly-example\/codes-used\/{_name}"}}
{"url":"observations.csv","tableSchema":{"columns":[{"name":"geography","titles":"geography","datatype":"string","propertyUrl":"skos:member","valueUrl":"http:\/\/statistics.data.gov.uk\/id\/statistical-geography\/{geography}"},{"name":"period","titles":"period","datatype":"string","propertyUrl":"skos:member","valueUrl":"http:\/\/reference.data.gov.uk\/id\/{+period}"},{"name":"cdid","titles":"cdid","datatype":"string","propertyUrl":"skos:member"},{"name":"services","titles":"services","datatype":"string","propertyUrl":"skos:member","valueUrl":"http:\/\/gss-data.org.uk\/def\/concept\/services\/{service}"},{"name":"flow","titles":"flow","datatype":"string","propertyUrl":"skos:member","valueUrl":"http:\/\/gss-data.org.uk\/def\/concept\/flow-directions\/{flow}"},{"name":"measure_type","titles":"measure_type","datatype":"string","propertyUrl":"skos:member","valueUrl":"http:\/\/gss-data.org.uk\/def\/measure\/{measure_type}"},{"name":"value","titles":"value","datatype":"number","propertyUrl":"skos:member","suppressOutput":true},{"name":"unit","titles":"unit","datatype":"string","propertyUrl":"skos:member","valueUrl":"http:\/\/gss-data.org.uk\/def\/concept\/measurement-units\/{unit}"}],"aboutUrl":"http:\/\/gss-data.org.uk\/data\/bop-quarterly-example\/codes-used\/{_name}"}}
2 changes: 1 addition & 1 deletion examples/overseas-trade/csvw/component-specifications.json
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"@context":["http:\/\/www.w3.org\/ns\/csvw",{"@language":"en"}],"url":"component-specifications.csv","dc:title":"Overseas Trade Sample","tableSchema":{"columns":[{"name":"component_slug","titles":"component_slug","datatype":"string","suppressOutput":true},{"name":"component_attachment","titles":"component_attachment","datatype":"string","suppressOutput":true},{"name":"component_property","titles":"component_property","datatype":"string","propertyUrl":"{+component_attachment}","valueUrl":"{+component_property}"},{"name":"type","virtual":true,"propertyUrl":"rdf:type","valueUrl":"qb:ComponentSpecification"},{"name":"codes_used","virtual":true,"propertyUrl":"http:\/\/publishmydata.com\/def\/qb\/codesUsed","valueUrl":"http:\/\/gss-data.org.uk\/data\/overseas-trade-sample\/codes-used\/{component_slug}"}],"aboutUrl":"http:\/\/gss-data.org.uk\/data\/overseas-trade-sample\/component\/{component_slug}"}}
{"url":"component-specifications.csv","dc:title":"Overseas Trade Sample","tableSchema":{"columns":[{"name":"component_slug","titles":"component_slug","datatype":"string","suppressOutput":true},{"name":"component_attachment","titles":"component_attachment","datatype":"string","suppressOutput":true},{"name":"component_property","titles":"component_property","datatype":"string","propertyUrl":"{+component_attachment}","valueUrl":"{+component_property}"},{"name":"type","virtual":true,"propertyUrl":"rdf:type","valueUrl":"qb:ComponentSpecification"},{"name":"codes_used","virtual":true,"propertyUrl":"http:\/\/publishmydata.com\/def\/qb\/codesUsed","valueUrl":"http:\/\/gss-data.org.uk\/data\/overseas-trade-sample\/codes-used\/{component_slug}"}],"aboutUrl":"http:\/\/gss-data.org.uk\/data\/overseas-trade-sample\/component\/{component_slug}"}}
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"@context":["http:\/\/www.w3.org\/ns\/csvw",{"@language":"en"}],"@id":"http:\/\/gss-data.org.uk\/data\/overseas-trade-sample\/structure","url":"component-specifications.csv","dc:title":"Overseas Trade Sample (Data Structure Definition)","rdf:type":{"@id":"qb:DataStructureDefinition"},"rdfs:label":"Overseas Trade Sample (Data Structure Definition)","tableSchema":{"columns":[{"name":"component_slug","titles":"component_slug","datatype":"string","propertyUrl":"qb:component","valueUrl":"http:\/\/gss-data.org.uk\/data\/overseas-trade-sample\/component\/{component_slug}"},{"name":"component_attachment","titles":"component_attachment","datatype":"string","suppressOutput":true},{"name":"component_property","titles":"component_property","datatype":"string","suppressOutput":true}],"aboutUrl":"http:\/\/gss-data.org.uk\/data\/overseas-trade-sample\/structure"}}
{"@id":"http:\/\/gss-data.org.uk\/data\/overseas-trade-sample\/structure","url":"component-specifications.csv","dc:title":"Overseas Trade Sample (Data Structure Definition)","rdf:type":{"@id":"qb:DataStructureDefinition"},"rdfs:label":"Overseas Trade Sample (Data Structure Definition)","tableSchema":{"columns":[{"name":"component_slug","titles":"component_slug","datatype":"string","propertyUrl":"qb:component","valueUrl":"http:\/\/gss-data.org.uk\/data\/overseas-trade-sample\/component\/{component_slug}"},{"name":"component_attachment","titles":"component_attachment","datatype":"string","suppressOutput":true},{"name":"component_property","titles":"component_property","datatype":"string","suppressOutput":true}],"aboutUrl":"http:\/\/gss-data.org.uk\/data\/overseas-trade-sample\/structure"}}
2 changes: 1 addition & 1 deletion examples/overseas-trade/csvw/dataset.json
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"@context":["http:\/\/www.w3.org\/ns\/csvw",{"@language":"en"}],"@id":"http:\/\/gss-data.org.uk\/data\/overseas-trade-sample","url":"component-specifications.csv","dc:title":"Overseas Trade Sample","tableSchema":{"columns":[{"name":"component_slug","titles":"component_slug","suppressOutput":true},{"name":"component_attachment","titles":"component_attachment","suppressOutput":true},{"name":"component_property","titles":"component_property","suppressOutput":true},{"name":"type","virtual":true,"propertyUrl":"rdf:type","valueUrl":"qb:DataSet"},{"name":"structure","virtual":true,"propertyUrl":"qb:structure","valueUrl":"http:\/\/gss-data.org.uk\/data\/overseas-trade-sample\/structure"}],"aboutUrl":"http:\/\/gss-data.org.uk\/data\/overseas-trade-sample"}}
{"@id":"http:\/\/gss-data.org.uk\/data\/overseas-trade-sample","url":"component-specifications.csv","dc:title":"Overseas Trade Sample","tableSchema":{"columns":[{"name":"component_slug","titles":"component_slug","suppressOutput":true},{"name":"component_attachment","titles":"component_attachment","suppressOutput":true},{"name":"component_property","titles":"component_property","suppressOutput":true},{"name":"type","virtual":true,"propertyUrl":"rdf:type","valueUrl":"qb:DataSet"},{"name":"structure","virtual":true,"propertyUrl":"qb:structure","valueUrl":"http:\/\/gss-data.org.uk\/data\/overseas-trade-sample\/structure"}],"aboutUrl":"http:\/\/gss-data.org.uk\/data\/overseas-trade-sample"}}
Loading

0 comments on commit db763e2

Please sign in to comment.