Skip to content

Commit d4257ed

Browse files
committed
Fix S3 configuration fields to use Optional<String>
- Change s3_bucket, s3_prefix, s3_region, s3_access_key_id, s3_secret_access_key from String to Optional<String> - Remove @ConfigDefault("null") annotations as they are not needed for Optional types - Update usage in transaction method to handle Optional values properly - Update README.md to clarify that S3 credentials are optional and use default AWS credentials provider chain
1 parent 9239ca7 commit d4257ed

File tree

2 files changed

+14
-19
lines changed

2 files changed

+14
-19
lines changed

README.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,8 @@ Snowflake output plugin for Embulk loads records to Snowflake.
3434
- **s3_bucket**: S3 bucket name for JDBC log upload (string, required when upload_jdbc_log_to_s3 is true)
3535
- **s3_prefix**: S3 key prefix for JDBC log upload (string, required when upload_jdbc_log_to_s3 is true)
3636
- **s3_region**: AWS region for S3 bucket (string, required when upload_jdbc_log_to_s3 is true)
37-
- **s3_access_key_id**: AWS access key ID for S3 access (string, optional - uses IAM role if not specified)
38-
- **s3_secret_access_key**: AWS secret access key for S3 access (string, optional - uses IAM role if not specified)
37+
- **s3_access_key_id**: AWS access key ID for S3 access (string, optional - uses default AWS credentials provider chain if not specified)
38+
- **s3_secret_access_key**: AWS secret access key for S3 access (string, optional - uses default AWS credentials provider chain if not specified)
3939
- **default_timezone**: If input column type (embulk type) is timestamp, this plugin needs to format the timestamp into a SQL string. This default_timezone option is used to control the timezone. You can overwrite timezone for each columns using column_options option. (string, default: `UTC`)
4040
- **column_options**: advanced: a key-value pairs where key is a column name and value is options for the column.
4141
- **type**: type of a column when this plugin creates new tables (e.g. `VARCHAR(255)`, `INTEGER NOT NULL UNIQUE`). This used when this plugin creates intermediate tables (insert, truncate_insert and merge modes), when it creates the target table (insert_direct and replace modes), and when it creates nonexistent target table automatically. (string, default: depends on input column type. `BIGINT` if input column type is long, `BOOLEAN` if boolean, `DOUBLE PRECISION` if double, `CLOB` if string, `TIMESTAMP` if timestamp)

src/main/java/org/embulk/output/SnowflakeOutputPlugin.java

Lines changed: 12 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -98,24 +98,19 @@ public interface SnowflakePluginTask extends PluginTask {
9898
public boolean getUploadJdbcLogToS3();
9999

100100
@Config("s3_bucket")
101-
@ConfigDefault("null")
102-
public String getS3Bucket();
101+
public Optional<String> getS3Bucket();
103102

104103
@Config("s3_prefix")
105-
@ConfigDefault("null")
106-
public String getS3Prefix();
104+
public Optional<String> getS3Prefix();
107105

108106
@Config("s3_region")
109-
@ConfigDefault("null")
110-
public String getS3Region();
107+
public Optional<String> getS3Region();
111108

112109
@Config("s3_access_key_id")
113-
@ConfigDefault("null")
114-
public String getS3AccessKeyId();
110+
public Optional<String> getS3AccessKeyId();
115111

116112
@Config("s3_secret_access_key")
117-
@ConfigDefault("null")
118-
public String getS3SecretAccessKey();
113+
public Optional<String> getS3SecretAccessKey();
119114

120115
public void setCopyIntoTableColumnNames(String[] columnNames);
121116

@@ -260,15 +255,15 @@ public ConfigDiff transaction(
260255
if (message != null
261256
&& message.contains(ENCOUNTERED_COMMUNICATION_ERROR_MESSAGE)
262257
&& t.getUploadJdbcLogToS3() == true) {
263-
final String s3Bucket = t.getS3Bucket();
264-
final String s3Prefix = t.getS3Prefix();
265-
final String s3Region = t.getS3Region();
266-
final String s3AccessKeyId = t.getS3AccessKeyId();
267-
final String s3SecretAccessKey = t.getS3SecretAccessKey();
268-
if (s3Bucket == null || s3Prefix == null || s3Region == null) {
258+
final Optional<String> s3Bucket = t.getS3Bucket();
259+
final Optional<String> s3Prefix = t.getS3Prefix();
260+
final Optional<String> s3Region = t.getS3Region();
261+
final Optional<String> s3AccessKeyId = t.getS3AccessKeyId();
262+
final Optional<String> s3SecretAccessKey = t.getS3SecretAccessKey();
263+
if (!s3Bucket.isPresent() || !s3Prefix.isPresent() || !s3Region.isPresent()) {
269264
logger.warn("s3_bucket, s3_prefix, and s3_region must be set when upload_jdbc_log_to_s3 is true");
270265
} else {
271-
try (JdbcLogUploader jdbcLogUploader = new JdbcLogUploader(s3Bucket, s3Prefix, s3Region, s3AccessKeyId, s3SecretAccessKey)) {
266+
try (JdbcLogUploader jdbcLogUploader = new JdbcLogUploader(s3Bucket.get(), s3Prefix.get(), s3Region.get(), s3AccessKeyId.orElse(null), s3SecretAccessKey.orElse(null))) {
272267
// snowflake_jdbc*.log で最新のファイルを探してアップロード
273268
String tmpDir = System.getProperty("java.io.tmpdir", "/tmp");
274269
File logDir = new File(tmpDir);

0 commit comments

Comments
 (0)