Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add capability to use sse encryption: to new properties added to amaz… #84

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions src/main/java/com/upplication/s3fs/AmazonS3Factory.java
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ public abstract class AmazonS3Factory {
public static final String SOCKET_RECEIVE_BUFFER_SIZE_HINT = "s3fs_socket_receive_buffer_size_hint";
public static final String SOCKET_TIMEOUT = "s3fs_socket_timeout";
public static final String USER_AGENT = "s3fs_user_agent";
public static final String REQUIRE_SSE_ENCRYPT = "s3fs_require_encrypt";
public static final String ENCRYPT_ARN = "s3fs_encrypt_arn";

/**
* Build a new Amazon S3 instance with the URI and the properties provided
Expand Down
7 changes: 3 additions & 4 deletions src/main/java/com/upplication/s3fs/S3FileChannel.java
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.util.IOUtils;
import com.upplication.s3fs.util.PutRequestUtils;

import org.apache.tika.Tika;

import java.io.*;
Expand Down Expand Up @@ -160,10 +162,7 @@ protected void sync() throws IOException {
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(Files.size(tempFile));
metadata.setContentType(new Tika().detect(stream, path.getFileName().toString()));

String bucket = path.getFileStore().name();
String key = path.getKey();
path.getFileSystem().getClient().putObject(bucket, key, stream, metadata);
PutRequestUtils.putObjectRequestForPath(path, stream, metadata);
}
}
}
27 changes: 26 additions & 1 deletion src/main/java/com/upplication/s3fs/S3FileSystem.java
Original file line number Diff line number Diff line change
Expand Up @@ -30,14 +30,30 @@ public class S3FileSystem extends FileSystem implements Comparable<S3FileSystem>
private final AmazonS3 client;
private final String endpoint;
private int cache;
private final String arnKey;
private final boolean requiresSseEncrypt;

public S3FileSystem(S3FileSystemProvider provider, String key, AmazonS3 client, String endpoint) {
public S3FileSystem(S3FileSystemProvider provider, String key, AmazonS3 client, String endpoint, boolean requiresSseEncrypt,String arnKey) {
this.provider = provider;
this.key = key;
this.client = client;
this.endpoint = endpoint;
this.cache = 60000; // 1 minute cache for the s3Path
this.arnKey = arnKey;
this.requiresSseEncrypt = requiresSseEncrypt;
}

/**
* Construct a File system with requiresSseEncrypt eq to false
* @param provider
* @param key
* @param client
* @param endpoint
*/
public S3FileSystem(S3FileSystemProvider provider, String key, AmazonS3 client, String endpoint) {
this(provider,key,client,endpoint,false,null);
}


@Override
public S3FileSystemProvider provider() {
Expand All @@ -48,6 +64,15 @@ public String getKey() {
return key;
}

public String getArnKey() {
return arnKey;
}

public boolean requiresSseEncrypt() {
return requiresSseEncrypt;
}


@Override
public void close() throws IOException {
this.provider.close(this);
Expand Down
78 changes: 48 additions & 30 deletions src/main/java/com/upplication/s3fs/S3FileSystemProvider.java
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import com.upplication.s3fs.attribute.S3PosixFileAttributes;
import com.upplication.s3fs.util.AttributesUtils;
import com.upplication.s3fs.util.Cache;
import com.upplication.s3fs.util.PutRequestUtils;
import com.upplication.s3fs.util.S3Utils;

import java.io.ByteArrayInputStream;
Expand Down Expand Up @@ -344,27 +345,29 @@ public FileChannel newFileChannel(Path path, Set<? extends OpenOption> options,
}

/**
* Deviations from spec: Does not perform atomic check-and-create. Since a
* directory is just an S3 object, all directories in the hierarchy are
* created or it already existed.
*/
@Override
public void createDirectory(Path dir, FileAttribute<?>... attrs) throws IOException {
S3Path s3Path = toS3Path(dir);
Preconditions.checkArgument(attrs.length == 0, "attrs not yet supported: %s", ImmutableList.copyOf(attrs)); // TODO
if (exists(s3Path))
throw new FileAlreadyExistsException(format("target already exists: %s", s3Path));
// create bucket if necesary
Bucket bucket = s3Path.getFileStore().getBucket();
String bucketName = s3Path.getFileStore().name();
if (bucket == null) {
s3Path.getFileSystem().getClient().createBucket(bucketName);
}
// create the object as directory
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(0);
s3Path.getFileSystem().getClient().putObject(bucketName, s3Path.getKey() + "/", new ByteArrayInputStream(new byte[0]), metadata);
}
* Deviations from spec: Does not perform atomic check-and-create. Since a
* directory is just an S3 object, all directories in the hierarchy are
* created or it already existed.
*/
@Override
public void createDirectory(Path dir, FileAttribute<?>... attrs) throws IOException {
S3Path s3Path = toS3Path(dir);
Preconditions.checkArgument(attrs.length == 0, "attrs not yet supported: %s", ImmutableList.copyOf(attrs)); // TODO
if (exists(s3Path))
throw new FileAlreadyExistsException(format("target already exists: %s", s3Path));
// create bucket if necesary
Bucket bucket = s3Path.getFileStore().getBucket();
String bucketName = s3Path.getFileStore().name();
if (bucket == null) {
s3Path.getFileSystem().getClient().createBucket(bucketName);
}
// create the object as directory
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(0);

PutRequestUtils.putObjectRequestForPath(s3Path, "/", new ByteArrayInputStream(new byte[0]), metadata);

}

@Override
public void delete(Path path) throws IOException {
Expand Down Expand Up @@ -533,15 +536,30 @@ public void setAttribute(Path path, String attribute, Object value, LinkOption..
// ~~

/**
* Create the fileSystem
*
* @param uri URI
* @param props Properties
* @return S3FileSystem never null
*/
public S3FileSystem createFileSystem(URI uri, Properties props) {
return new S3FileSystem(this, getFileSystemKey(uri, props), getAmazonS3(uri, props), uri.getHost());
}
* Create the fileSystem
*
* @param uri
* URI
* @param props
* Properties
* @return S3FileSystem never null
*/
public S3FileSystem createFileSystem(URI uri, Properties props) {
return new S3FileSystem(this, getFileSystemKey(uri, props), getAmazonS3(uri, props), uri.getHost(), requireSseEncrypt(props), getArnKey(props));
}

/**
*
* @param props
* @return the arn used to encrypt the files, null if not specified
*/
protected String getArnKey(Properties props) {
return props.getProperty(ENCRYPT_ARN);
}

protected boolean requireSseEncrypt(Properties props) {
return Boolean.parseBoolean(props.getProperty(REQUIRE_SSE_ENCRYPT, "false"));
}

protected AmazonS3 getAmazonS3(URI uri, Properties props) {
return getAmazonS3Factory(props).getAmazonS3(uri, props);
Expand Down
6 changes: 2 additions & 4 deletions src/main/java/com/upplication/s3fs/S3SeekableByteChannel.java
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.S3Object;
import com.upplication.s3fs.util.PutRequestUtils;

public class S3SeekableByteChannel implements SeekableByteChannel {

Expand Down Expand Up @@ -104,10 +105,7 @@ protected void sync() throws IOException {
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(Files.size(tempFile));
metadata.setContentType(new Tika().detect(stream, path.getFileName().toString()));

String bucket = path.getFileStore().name();
String key = path.getKey();
path.getFileSystem().getClient().putObject(bucket, key, stream, metadata);
PutRequestUtils.putObjectRequestForPath(path, stream, metadata);
}
}

Expand Down
51 changes: 51 additions & 0 deletions src/main/java/com/upplication/s3fs/util/PutRequestUtils.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
package com.upplication.s3fs.util;

import java.io.InputStream;

import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.PutObjectResult;
import com.amazonaws.services.s3.model.SSEAwsKeyManagementParams;
import com.amazonaws.util.StringUtils;
import com.upplication.s3fs.S3FileStore;
import com.upplication.s3fs.S3Path;

public class PutRequestUtils {

/**
* Create an putObject request for the bucket represented by the given path see {@link S3Path#getFileStore()} {@link S3FileStore#name()}
*
* @param path
* the path
* @param stream
* @param metadata
* @return
*/
public static PutObjectResult putObjectRequestForPath(S3Path path, InputStream stream, ObjectMetadata metadata) {
return putObjectRequestForPath(path, "", stream, metadata);
}

/**
* Create an putObject request for the bucket represented by the given path see {@link S3Path#getFileStore()} {@link S3FileStore#name()}
* The final key is {@link S3Path#getKey()} + addToKey params
*
* @param path
* @param addToKey
* the part to be added to the {@link S3Path#getKey()}, (Ex: for reference directories it is necessary add /)
* @param stream
* @param metadata
* @return
*/
public static PutObjectResult putObjectRequestForPath(S3Path path, String addToKey, InputStream stream, ObjectMetadata metadata) {
String bucket = path.getFileStore().name();
String key = path.getKey();
PutObjectRequest putObjectRequest = new PutObjectRequest(bucket, key, stream, metadata);
if (path.getFileSystem().requiresSseEncrypt()) {
String arnKey = path.getFileSystem().getArnKey();
SSEAwsKeyManagementParams sseKeyManagementParams = StringUtils.isNullOrEmpty(arnKey) ? new SSEAwsKeyManagementParams() : new SSEAwsKeyManagementParams(arnKey);
putObjectRequest = putObjectRequest.withSSEAwsKeyManagementParams(sseKeyManagementParams);
}
return path.getFileSystem().getClient().putObject(putObjectRequest);
}

}