Skip to content
This repository was archived by the owner on Mar 12, 2020. It is now read-only.

Commit daa69f1

Browse files
author
Eric Meyer
committed
Adds support for "vertical" tables.
1 parent 91d7ea3 commit daa69f1

18 files changed

+1086
-30
lines changed

‎README‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,45 @@ Then run the tool by executing the run.sh script.
2222
Release Notes
2323
-------------
2424

25+
1.5 - 2011-08-04
26+
27+
* Adds support for "vertical" tables.
28+
29+
1.4 - 2011-03-08
30+
31+
* Creates new version of Data Tool Configuration XML schema to support bulk
32+
and full changesets and subqueries.
33+
* Adds support for bulk and full changesets in addition to snapshots.
34+
* Adds support for a wide variety of subquery types in order to better
35+
support flattening complex database structures without the use of views or
36+
stored procedures.
37+
* Fixes WebServerMain to handle paths with spaces in directory names. Fixes
38+
problem where datatool cannot be run if its path contains spaces.
39+
* Updates tests to verify tool continutes to be backwards compatible in
40+
parsing all versions of the configuration xml.
41+
* Adds support for column aliases.
42+
* Makes named SQL parameter matching case insensitive by creating and using
43+
CaseInsensitveParameterSource (based on Spring's MapSqlParameterSource,
44+
which unfortunately does not allow plugging in a different Map
45+
implementation). All tests now pass.
46+
* Updates logging.properties to log com.t11e.discovery.datatool at FINE. And
47+
adds a comment explaining how to enable logging sql query execution speed.
48+
Turns down default level to WARNING (which quiets Spring).
49+
* Adds test to show that Discovery Data Tool can call stored procedures.
50+
51+
1.3 - 2010-10-01
52+
53+
* Adds forceSnapshot boolean parameter to ChangesetController. If true, then
54+
start date of null is used and a snapshot is created regardless of the
55+
profile's last run time.
56+
57+
1.2 - 2010-09-07
58+
59+
* Adds support for protecting incoming connections via SSL and basic
60+
authentication.
61+
* Adds throwing an exception in config XML does not have a root config
62+
element in a recognized namespace.
63+
2564
1.1 - 2010-06-11
2665

2766
* Added createSql support to sqlProfile, used to auto create missing

‎src/docroot/WEB-INF/classes/datatool-config-5.xsd‎

Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<schema xmlns="http://www.w3.org/2001/XMLSchema"
3+
targetNamespace="http://transparensee.com/schema/datatool-config-5"
4+
xmlns:tns="http://transparensee.com/schema/datatool-config-5"
5+
elementFormDefault="qualified"
6+
>
7+
8+
<element name="config" type="tns:ConfigType"/>
9+
10+
<complexType name="ConfigType">
11+
<sequence>
12+
<element name="accessControl" type="tns:AccessControlType" minOccurs="0" maxOccurs="1"/>
13+
<element name="dataSources" type="tns:DataSourcesType"/>
14+
<element name="profiles" type="tns:ProfilesType" minOccurs="0" maxOccurs="1"/>
15+
<element name="publishers" type="tns:PublishersType" minOccurs="0" maxOccurs="1"/>
16+
</sequence>
17+
</complexType>
18+
19+
<complexType name="AccessControlType">
20+
<annotation>
21+
<documentation xml:lang="en">
22+
Optional restrict feed access using basic HTTP authentication.
23+
Since datatool-config-3.
24+
By adding the element, you turn on basic HTTP authentication.
25+
Add child user elements.
26+
</documentation>
27+
</annotation>
28+
<sequence minOccurs="0">
29+
<element name="user">
30+
<complexType>
31+
<attribute name="name" type="string" use="required"/>
32+
<attribute name="password" type="string" use="required"/>
33+
</complexType>
34+
</element>
35+
</sequence>
36+
</complexType>
37+
38+
<complexType name="DataSourcesType">
39+
<sequence minOccurs="1" maxOccurs="unbounded">
40+
<choice>
41+
<element name="dataSource" type="tns:DataSourceType"/>
42+
<element name="driver" type="tns:DriverType"/>
43+
</choice>
44+
</sequence>
45+
</complexType>
46+
47+
<complexType name="DataSourceType">
48+
<sequence minOccurs="0" maxOccurs="unbounded">
49+
<any processContents="lax"/>
50+
</sequence>
51+
<attribute name="name" type="string" use="required"/>
52+
<attribute name="jar" type="string" use="optional"/>
53+
<attribute name="class" type="string" use="required"/>
54+
</complexType>
55+
56+
<complexType name="DriverType">
57+
<sequence>
58+
<element name="url"/>
59+
<element name="username" minOccurs="0"/>
60+
<element name="password" minOccurs="0"/>
61+
<element name="properties" minOccurs="0">
62+
<complexType>
63+
<sequence minOccurs="0" maxOccurs="unbounded">
64+
<any processContents="lax"/>
65+
</sequence>
66+
</complexType>
67+
</element>
68+
</sequence>
69+
<attribute name="name" type="string" use="required"/>
70+
<attribute name="jar" type="string" use="optional"/>
71+
<attribute name="class" type="string" use="required"/>
72+
</complexType>
73+
74+
<complexType name="ProfilesType">
75+
<sequence minOccurs="1" maxOccurs="unbounded">
76+
<element name="sqlProfile" type="tns:SqlProfileType"/>
77+
</sequence>
78+
</complexType>
79+
80+
<complexType name="PublishersType">
81+
<sequence minOccurs="0" maxOccurs="unbounded">
82+
<element name="sqlPublisher" type="tns:SqlPublisherType"/>
83+
</sequence>
84+
</complexType>
85+
86+
<complexType name="SqlProfileType">
87+
<sequence>
88+
<element name="createSql" type="string" minOccurs="0">
89+
<annotation>
90+
<documentation xml:lang="en">
91+
Optional SQL used to create a profile when one is not found.
92+
Since datatool-config-2.
93+
Bound parameters:
94+
:name -- profile name
95+
Outputs:
96+
none
97+
</documentation>
98+
</annotation>
99+
</element>
100+
<element name="retrieveSql" type="tns:retrieveSql"/>
101+
<element name="updateSql" type="string"/>
102+
</sequence>
103+
<attribute name="name" type="string" use="required"/>
104+
<attribute name="dataSource" type="string" use="required"/>
105+
</complexType>
106+
107+
<complexType name="retrieveSql" mixed="true">
108+
<attribute name="startColumn" use="required"/>
109+
<attribute name="endColumn" use="required"/>
110+
</complexType>
111+
112+
<complexType name="SqlPublisherType">
113+
<sequence>
114+
<choice minOccurs="0" maxOccurs="1">
115+
<element name="bulk" type="tns:BulkFullType"/>
116+
<element name="full" type="tns:BulkFullType"/>
117+
<element name="snapshot" type="tns:SnapshotOrDeltaType"/>
118+
</choice>
119+
<element name="delta" minOccurs="0" maxOccurs="1" type="tns:SnapshotOrDeltaType"/>
120+
</sequence>
121+
<attribute name="name" type="string" use="required"/>
122+
<attribute name="dataSource" type="string" use="required"/>
123+
<attribute name="profile" type="string" use="optional"/>
124+
</complexType>
125+
126+
<complexType name="BulkFullType">
127+
<sequence>
128+
<element name="set-item" minOccurs="0" maxOccurs="unbounded" type="tns:SetItemType"/>
129+
</sequence>
130+
</complexType>
131+
132+
<complexType name="SnapshotOrDeltaType">
133+
<sequence minOccurs="0" maxOccurs="unbounded">
134+
<choice minOccurs="0" maxOccurs="1">
135+
<element name="set-item" minOccurs="0" maxOccurs="unbounded" type="tns:SetItemType"/>
136+
<element name="remove-item" minOccurs="0" maxOccurs="unbounded" type="tns:RemoveItemType"/>
137+
</choice>
138+
</sequence>
139+
</complexType>
140+
141+
<complexType name="SetItemType">
142+
<sequence>
143+
<element name="query" type="string"/>
144+
<element name="merge-columns" type="tns:MergeColumnsType" minOccurs="0" maxOccurs="unbounded"/>
145+
<element name="subquery" type="tns:SubQueryType" minOccurs="0" maxOccurs="unbounded"/>
146+
</sequence>
147+
<attribute name="idColumn" type="string" use="required"/>
148+
<attribute name="jsonColumnNames" type="string" use="optional"/>
149+
</complexType>
150+
151+
<complexType name="RemoveItemType">
152+
<sequence>
153+
<element name="query" type="string"/>
154+
</sequence>
155+
<attribute name="idColumn" type="string" use="required"/>
156+
</complexType>
157+
158+
<complexType name="SubQueryType">
159+
<simpleContent>
160+
<extension base="string">
161+
<attribute name="delimiter" type="string" use="optional" />
162+
<attribute name="property" type="string" use="optional"/>
163+
<attribute name="propertyPrefix" type="string" use="optional"/>
164+
<attribute name="discriminator" type="string" use="optional"/>
165+
<attribute name="type" type="tns:SubQueryTypeType" default="array" />
166+
</extension>
167+
</simpleContent>
168+
</complexType>
169+
170+
<simpleType name="SubQueryTypeType">
171+
<restriction base="string">
172+
<enumeration value="array"/>
173+
<enumeration value="delimited"/>
174+
</restriction>
175+
</simpleType>
176+
177+
<complexType name="MergeColumnsType">
178+
<attribute name="keyColumn" type="string" use="required"/>
179+
<attribute name="valueColumn" type="string" use="required"/>
180+
</complexType>
181+
182+
<simpleType name="ActionTypeType">
183+
<restriction base="string">
184+
<enumeration value="create"/>
185+
<enumeration value="delete"/>
186+
</restriction>
187+
</simpleType>
188+
189+
<simpleType name="FilterActionType">
190+
<restriction base="string">
191+
<enumeration value="delta"/>
192+
<enumeration value="snapshot"/>
193+
</restriction>
194+
</simpleType>
195+
</schema>

‎src/main/java/com/t11e/discovery/datatool/CompletionAwareRowCallbackHandler.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
package com.t11e.discovery.datatool;
2+
3+
import org.springframework.jdbc.core.RowCallbackHandler;
4+
5+
public interface CompletionAwareRowCallbackHandler
6+
extends RowCallbackHandler
7+
{
8+
void flushItem();
9+
}

‎src/main/java/com/t11e/discovery/datatool/ConfigurationManager.java‎

Lines changed: 38 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@
4444
import org.springframework.stereotype.Component;
4545
import org.xml.sax.SAXException;
4646

47+
import com.t11e.discovery.datatool.column.MergeColumns;
48+
4749
@Component("ConfigurationManager")
4850
public class ConfigurationManager
4951
{
@@ -349,31 +351,46 @@ private void fillActionBeanDefinition(final BeanDefinitionBuilder builder, final
349351
builder
350352
.addPropertyValue("query",
351353
StringUtils.trimToEmpty(parentElementToQuery.valueOf("c:query/text()".replace("c:", ns))));
352-
final List<SubQuery> subqueries = new ArrayList<SubQuery>();
353-
for (final Node subquery : (List<Node>) parentElementToQuery.selectNodes("c:subquery".replace("c:", ns)))
354+
354355
{
355-
final String sql = subquery.getText();
356-
final String property = subquery.valueOf("@property");
357-
final String propertyPrefix = subquery.valueOf("@propertyPrefix");
358-
if (StringUtils.isNotBlank(property) && StringUtils.isNotBlank(propertyPrefix))
359-
{
360-
throw new RuntimeException("Subqueries cannot specify both property and propertyPrefix: " + subquery.asXML());
361-
}
362-
String type = subquery.valueOf("@type");
363-
if (StringUtils.isBlank(type))
356+
final List<MergeColumns> mergeColumns = new ArrayList<MergeColumns>();
357+
for (final Node mergeColumnDef : (List<Node>) parentElementToQuery.selectNodes("c:merge-columns"
358+
.replace("c:", ns)))
364359
{
365-
type = SubQuery.Type.ARRAY.name();
360+
final String keyColumn = mergeColumnDef.valueOf("@keyColumn");
361+
final String valueColumn = mergeColumnDef.valueOf("@valueColumn");
362+
mergeColumns.add(new MergeColumns(keyColumn, valueColumn));
366363
}
367-
String delimiter = subquery.valueOf("@delimiter");
368-
if (StringUtils.isBlank(delimiter) && SubQuery.Type.DELIMITED.name().equalsIgnoreCase(type))
364+
builder.addPropertyValue("mergeColumns", mergeColumns);
365+
}
366+
{
367+
final List<SubQuery> subqueries = new ArrayList<SubQuery>();
368+
for (final Node subquery : (List<Node>) parentElementToQuery.selectNodes("c:subquery".replace("c:", ns)))
369369
{
370-
delimiter = ",";
370+
final String sql = subquery.getText();
371+
final String property = subquery.valueOf("@property");
372+
final String propertyPrefix = subquery.valueOf("@propertyPrefix");
373+
if (StringUtils.isNotBlank(property) && StringUtils.isNotBlank(propertyPrefix))
374+
{
375+
throw new RuntimeException("Subqueries cannot specify both property and propertyPrefix: " + subquery.asXML());
376+
}
377+
String type = subquery.valueOf("@type");
378+
if (StringUtils.isBlank(type))
379+
{
380+
type = SubQuery.Type.ARRAY.name();
381+
}
382+
String delimiter = subquery.valueOf("@delimiter");
383+
if (StringUtils.isBlank(delimiter) && SubQuery.Type.DELIMITED.name().equalsIgnoreCase(type))
384+
{
385+
delimiter = ",";
386+
}
387+
final String discriminator = subquery.valueOf("@discriminator");
388+
subqueries.add(new SubQuery(SubQuery.Type.valueOf(type.toUpperCase()), sql, property, propertyPrefix,
389+
delimiter,
390+
discriminator));
371391
}
372-
final String discriminator = subquery.valueOf("@discriminator");
373-
subqueries.add(new SubQuery(SubQuery.Type.valueOf(type.toUpperCase()), sql, property, propertyPrefix, delimiter,
374-
discriminator));
392+
builder.addPropertyValue("subqueries", subqueries);
375393
}
376-
builder.addPropertyValue("subqueries", subqueries);
377394
}
378395

379396
@SuppressWarnings("unchecked")
@@ -421,7 +438,8 @@ private void parseConfiguration(
421438
"c1", "http://transparensee.com/schema/datatool-config-1",
422439
"c2", "http://transparensee.com/schema/datatool-config-2",
423440
"c3", "http://transparensee.com/schema/datatool-config-3",
424-
"c4", "http://transparensee.com/schema/datatool-config-4");
441+
"c4", "http://transparensee.com/schema/datatool-config-4",
442+
"c5", "http://transparensee.com/schema/datatool-config-5");
425443
final Map<String, String> namespacesByUri = MapUtils.invertMap(namespacesByPrefix);
426444
{
427445
final DocumentFactory factory = new DocumentFactory();

0 commit comments

Comments
 (0)