Merge branch 'develop' into feature/add_ci_test

Author: chikamura

.github/workflows/main.yml

@@ -16,26 +16,22 @@ on:
     types: [labeled]
 
 jobs:
-  main:
+  test:
+    name: test
     runs-on: ubuntu-latest
     if: >
       ${{
         github.event_name == 'pull_request' ||
         (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'safe to test')) ||
         startsWith(github.ref, 'refs/tags/')
       }}
-    permissions:
-      packages: write
-      contents: read
-
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
       - name: Set up JDK 1.8
         uses: actions/setup-java@v1
         with:
           java-version: 1.8
       - name: lint
-        # Run ./gradlew spotlessApply when failing on this step.
         run: ./gradlew spotlessCheck
       - name: Write test secret to file
         run: envsubst < ci/config_template.yml > test-config.yml
@@ -50,22 +46,23 @@ jobs:
       - run: ./gradlew test
         env:
           EMBULK_OUTPUT_DATABRICKS_TEST_CONFIG: "./test-config.yml"
-      - name: Show Current Version
-        run: ./gradlew printVersion
-      - name: Build Gem & Put it into './build/gems/' Directory
-        run: ./gradlew gem
-      - name: Set up JRuby
+  build:
+    name: Build + Publish
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+    needs: [ test ]
+    if: ${{ github.event_name == 'workflow_dispatch' || contains(github.ref, 'refs/tags/') }}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby 2.7
         uses: ruby/setup-ruby@v1
         with:
-          ruby-version: jruby
-      - name: Publish to GPR
-        if: startsWith( github.ref, 'refs/tags/' )
-        run: |
-          mkdir -p $HOME/.gem
-          touch $HOME/.gem/credentials
-          chmod 0600 $HOME/.gem/credentials
-          printf -- "---\n:github: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
-          gem push --KEY github --host https://rubygems.pkg.github.com/${OWNER} ./build/gems/*.gem
-        env:
-          GEM_HOST_API_KEY: "Bearer ${{secrets.GITHUB_TOKEN}}"
-          OWNER: ${{ github.repository_owner }}
+          ruby-version: 2.7
+      - name: push gem
+        uses: trocco-io/push-gem-to-gpr-action@v1
+        with:
+          language: java
+          gem-path: "./pkg/*.gem"
+          github-token: "${{ secrets.GITHUB_TOKEN }}"

README.md

@@ -59,6 +59,13 @@ Databricks output plugin for Embulk loads records to Databricks Delta Table.
   * Transactional: Yes.
   * Resumable: No.
 
+## Note
+
+This plugin does not support TIMESTAMP_NTZ、INTERVAL types, if target tables contain these types, embulk will raise a runtime error.
+（Because [The official Databricks JDBC driver does not support TIMESTAMP_NTZ、INTERVAL types](https://docs.databricks.com/en/sql/language-manual/data-types/timestamp-ntz-type.html#notes).）
+
+This plugin converts empty string input to null output. If you want to empty string output, you can use continuous double quote string ("").
+
 ## Build
 
 ```
@@ -67,4 +74,4 @@ $ ./gradlew gem  # -t to watch change of files and rebuild continuously
 
 ## TEST
 
-$ EMBULK_OUTPUT_DATABRICKS_TEST_CONFIG="example/test.yml" ./gradlew test # Create exxample/test.yml based on example/test.yml.example
+$ EMBULK_OUTPUT_DATABRICKS_TEST_CONFIG="example/test.yml" ./gradlew test # Create example/test.yml based on example/test.yml.example

example/test.yml.example

@@ -1,3 +1,5 @@
+# The catalog_name and schema_name must be created in advance.
+
 server_hostname: 
 http_path: 
 personal_access_token: 

src/test/java/org/embulk/output/databricks/TestDatabricksOutputPluginByMode.java

@@ -127,23 +127,35 @@ public void testMergeToExistTableWithMergeRules2() throws Exception {
     assertQueryResults("test0,0,a", "test1,1test89,bTESTB", "test2,12,C");
   }
 
+  @Test
+  public void testMergeNoKeyToExistTable() throws Exception {
+    setPluginConfigSource(AbstractJdbcOutputPlugin.Mode.MERGE);
+    String[] sqlTypes = sqlStringTypes(2);
+    ConnectionUtil.run(
+        String.format("CREATE TABLE %s (_c0 STRING PRIMARY KEY, _c1 STRING)", quotedDstTableName));
+    ConnectionUtil.run(insertSQL(quotedDstTableName, sqlTypes, "test0,0", "test1,1"));
+    embulk.runOutput(configSource, createInputFile("test1,11", "test2,12").toPath());
+    assertQueryResults("test0,0", "test1,11", "test2,12");
+  }
+
   private void setMergeRule(String... mergeRules) {
     configSource.set("merge_rule", Arrays.asList(mergeRules));
   }
 
   private void setPluginConfigSource(AbstractJdbcOutputPlugin.Mode mode, String... mergeKeys) {
-    configSource = createPluginConfigSource(mode, Arrays.asList(mergeKeys));
+    configSource =
+        createPluginConfigSource(mode, mergeKeys.length > 0 ? Arrays.asList(mergeKeys) : null);
     quotedDstTableName = quotedDstTableName(configSource);
   }
 
   private void createTable(String... rows) {
-    String[] sqlTypes = sqlTypes(rows[0]);
+    String[] sqlTypes = sqlStringTypes(rows[0]);
     ConnectionUtil.run(createTableSQL(quotedDstTableName, sqlTypes));
     ConnectionUtil.run(insertSQL(quotedDstTableName, sqlTypes, rows));
   }
 
   private File createInputFile(String... rows) throws IOException {
-    return IOUtil.createInputFile(testFolder, csvHeader(sqlTypes(rows[0])), rows);
+    return IOUtil.createInputFile(testFolder, csvHeader(sqlStringTypes(rows[0])), rows);
   }
 
   private void assertQueryResults(String... rows) {
@@ -161,8 +173,12 @@ private void assertQueryResults(String... rows) {
     assertTableResults(results, numberOfColumns, data.toArray());
   }
 
-  private String[] sqlTypes(String sampleRow) {
+  private String[] sqlStringTypes(String sampleRow) {
     int numberOfColumns = sampleRow.split(",").length;
+    return sqlStringTypes(numberOfColumns);
+  }
+
+  private String[] sqlStringTypes(int numberOfColumns) {
     return IntStream.range(0, numberOfColumns).mapToObj(x -> "string").toArray(String[]::new);
   }
 }

src/test/java/org/embulk/output/databricks/TestDatabricksOutputPluginByTimestamp.java

@@ -1,6 +1,7 @@
 package org.embulk.output.databricks;
 
 import java.io.IOException;
+import java.sql.Timestamp;
 import java.time.ZoneId;
 import org.junit.Test;
 
@@ -9,13 +10,30 @@ public class TestDatabricksOutputPluginByTimestamp extends AbstractTestDatabrick
   public void testTimestamp() throws IOException {
     testOutputValues(
         new TestTimestampSet(
-            "2000-01-02 03:04:05.00 UTC", "%Y/%m/%d %H-%M-%S", null, "2000/01/02 03-04-05"),
+            "2000-01-02 03:04:05.00 UTC",
+            "%Y-%m-%d",
+            null,
+            Timestamp.valueOf("2000-01-02 00:00:00.0")),
+        new TestTimestampSet(
+            "2000-01-02 03:04:05.00 UTC", "%Y", null, Timestamp.valueOf("2000-01-01 00:00:00.0")),
         new TestTimestampSet(
+            "2000-01-02 20:04:05.00 UTC",
+            "%Y-%m-%d",
+            ZoneId.of("Asia/Tokyo"),
+            Timestamp.valueOf("2000-01-03 00:00:00.0")));
+  }
+
+  @Test
+  public void testTimestampToString() throws IOException {
+    testOutputValues(
+        new TestTimestampToStringSet(
+            "2000-01-02 03:04:05.00 UTC", "%Y/%m/%d %H-%M-%S", null, "2000/01/02 03-04-05"),
+        new TestTimestampToStringSet(
             "2000-01-02 03:04:05.00 UTC",
             null,
             ZoneId.of("Asia/Tokyo"),
             "2000-01-02 12:04:05.000000"),
-        new TestTimestampSet(
+        new TestTimestampToStringSet(
             "2000-01-02 03:04:05.00 UTC",
             "%Y/%m/%d %H-%M-%S",
             ZoneId.of("Asia/Tokyo"),
@@ -25,6 +43,20 @@ public void testTimestamp() throws IOException {
   protected class TestTimestampSet extends TestSet {
     TestTimestampSet(
         String value, String outputTimestampFormat, ZoneId outputTimezone, Object expected) {
+      super(
+          value,
+          "timestamp",
+          "timestamp",
+          "string",
+          outputTimestampFormat,
+          outputTimezone,
+          expected);
+    }
+  }
+
+  protected class TestTimestampToStringSet extends TestSet {
+    TestTimestampToStringSet(
+        String value, String outputTimestampFormat, ZoneId outputTimezone, Object expected) {
       super(value, "timestamp", "string", null, outputTimestampFormat, outputTimezone, expected);
     }
   }