diff --git a/DESCRIPTION b/DESCRIPTION index 3681bf1..54f6ca0 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: rsync Type: Package Title: API to use rsync -Version: 24.10.0 +Version: 25.10.0 Author: INWT Statistics GmbH Authors@R: c(person("Sebastian", "Warnholz", email = "sebastian.warnholz@inwt-statistics.de", role = c("aut", "cre")), person("Jonathan", "Bob", email = "jonathan.bob@inwt-statistics.de", role = c("aut")), @@ -24,4 +24,4 @@ Suggests: License: MIT + file LICENSE Encoding: UTF-8 LazyData: true -RoxygenNote: 7.3.1 +RoxygenNote: 7.3.2 diff --git a/NEWS.md b/NEWS.md new file mode 100644 index 0000000..d74d5b7 --- /dev/null +++ b/NEWS.md @@ -0,0 +1,4 @@ +# rsync 25.10.0 + +## Added +- Added support for custom S3 endpoints (e.g., Hetzner) \ No newline at end of file diff --git a/R/awscli.R b/R/awscli.R index 03ca840..228346b 100644 --- a/R/awscli.R +++ b/R/awscli.R @@ -1,4 +1,4 @@ -awscli <- function(src, dest, includes = NULL, excludes = NULL, args = "", profile = NULL, intern = FALSE) { +awscli <- function(src, dest, includes = NULL, excludes = NULL, args = "", profile = NULL, endpoint_url = NULL, intern = FALSE) { constructArg <- function(x, s) { if (is.null(x)) { return(x) @@ -9,6 +9,7 @@ awscli <- function(src, dest, includes = NULL, excludes = NULL, args = "", profi includes <- constructArg(includes, "--include") excludes <- constructArg(excludes, "--exclude") profile <- if (is.null(profile)) "" else paste("--profile", profile) + endpoint <- if (is.null(endpoint_url)) "" else paste("--endpoint-url", endpoint_url) dest <- paste("\"", dest, "\"", sep = "") src <- if (!is.null(src)) paste("\"", src, "\"", sep = "") else NULL @@ -18,6 +19,7 @@ awscli <- function(src, dest, includes = NULL, excludes = NULL, args = "", profi excludes, includes, profile, + endpoint, src, dest ) diff --git a/R/awss3.R b/R/awss3.R index cf8b075..d3b9355 100644 --- a/R/awss3.R +++ b/R/awss3.R @@ -10,6 +10,8 @@ #' a profile. In case of a list a new profile will be created which is #' persistent. A profile is created using \code{aws configure} and stores #' credentials for the user in plain text. +#' @param endpoint_url (NULL|character) The endpoint URL for S3-compatible providers like +#' Hetzner. E.g. 'https://fsn1.your-objectstorage.com'. #' @param force (logical) override profile if it exists. #' @param db (awss3) connection created with \code{awss3} #' @param fileName (character) a file name in dest/src @@ -32,13 +34,14 @@ #' #' @rdname awss3 #' @export -awss3 <- function(dest, src = getwd(), profile = NULL) { +awss3 <- function(dest, src = getwd(), profile = NULL, endpoint_url = NULL) { stopifnot( is.character(dest) && length(dest) == 1, is.character(src) && length(src) == 1, is.null(profile) || (is.character(profile) && length(profile) == 1 && profileExists(profile)) || - (is.list(profile) && is.character(profile$name)) + (is.list(profile) && is.character(profile$name)), + is.null(endpoint_url) || is.character(endpoint_url) && length(endpoint_url) == 1 ) src <- if (isS3Bucket(src)) { sub("/$", "", src) @@ -57,7 +60,8 @@ awss3 <- function(dest, src = getwd(), profile = NULL) { ret <- list( dest = dest, src = src, - profile = profile + profile = profile, + endpoint_url = endpoint_url ) class(ret) <- "awss3" ret diff --git a/R/getFile.R b/R/getFile.R index e9c91fa..8e4a69f 100644 --- a/R/getFile.R +++ b/R/getFile.R @@ -10,7 +10,6 @@ getFile <- function(db, ...) { #' @rdname rsync #' @export getFile.default <- function(db, fileName, validate = FALSE, verbose = FALSE, ...) { - args <- if (verbose == TRUE) "-ltrvvx" else "-ltrx" args <- paste(args, getArgs(db)) @@ -28,15 +27,15 @@ getFile.default <- function(db, fileName, validate = FALSE, verbose = FALSE, ... #' @rdname awss3 #' @export getFile.awss3 <- function(db, fileName, validate = FALSE, verbose = FALSE, ...) { - args <- if (!verbose) "--quiet --no-progress --only-show-errors" else "" args <- paste("sync ", args) dest <- getDest(db) src <- getSrc(db) profile <- getProfile(db) + endpoint_url <- db$endpoint_url - awscli(dest, src, args = args, excludes = "*", includes = fileName, profile = profile) + awscli(dest, src, args = args, excludes = "*", includes = fileName, profile = profile, endpoint_url = endpoint_url) if (validate) validateFile(db, fileName) diff --git a/R/listFiles.R b/R/listFiles.R index 4cf4268..93ba396 100644 --- a/R/listFiles.R +++ b/R/listFiles.R @@ -57,11 +57,12 @@ emptyDir <- function() { listFiles.awss3 <- function(db, recursive = FALSE, ...) { dest <- getDest(db) profile <- getProfile(db) + endpoint_url <- db$endpoint_url if (!isS3Bucket(dest)) { return(NextMethod()) } args <- if (recursive) "ls --recursive" else "ls" - dir <- awscli(NULL, dest, args = args, profile = profile, intern = TRUE) + dir <- awscli(NULL, dest, args = args, profile = profile, endpoint_url = endpoint_url, intern = TRUE) dir <- dat::extract(dir, ~ !grepl("\\.$", .)) if (length(dir) == 0) { return(emptyDir()) diff --git a/R/removeFile.R b/R/removeFile.R index 452125d..ff0238f 100644 --- a/R/removeFile.R +++ b/R/removeFile.R @@ -38,6 +38,7 @@ removeFile.awss3 <- function(db, fileName, verbose = FALSE, ...) { dest <- getDest(db) profile <- getProfile(db) + endpoint_url <- db$endpoint_url if (!isS3Bucket(dest)) { return(NextMethod()) } @@ -45,6 +46,6 @@ removeFile.awss3 <- function(db, fileName, verbose = FALSE, ...) { args <- if (!verbose) "--quiet --only-show-errors --recursive" else "--recursive" args <- paste("rm", args) - awscli(NULL, dest, includes = fileName, excludes = "*", args = args, profile = profile) + awscli(NULL, dest, includes = fileName, excludes = "*", args = args, profile = profile, endpoint_url = endpoint_url) db } diff --git a/R/sendAllFiles.R b/R/sendAllFiles.R index 9974610..f88fddb 100644 --- a/R/sendAllFiles.R +++ b/R/sendAllFiles.R @@ -22,7 +22,8 @@ sendAllFiles.awss3 <- function(db, verbose = FALSE, ...) { src <- getSrc(db) dest <- getDest(db) profile <- getProfile(db) + endpoint_url <- db$endpoint_url - awscli(src, dest, args = args, profile = profile) + awscli(src, dest, args = args, profile = profile, endpoint_url = endpoint_url) db } diff --git a/R/sendFile.R b/R/sendFile.R index 651b111..e8173f8 100644 --- a/R/sendFile.R +++ b/R/sendFile.R @@ -35,8 +35,9 @@ sendFile.awss3 <- function(db, fileName, validate = FALSE, verbose = FALSE, args src <- getSrc(db) dest <- getDest(db) profile <- getProfile(db) + endpoint_url <- db$endpoint_url - awscli(src, dest, args = args, excludes = "*", includes = fileName, profile = profile) + awscli(src, dest, args = args, excludes = "*", includes = fileName, profile = profile, endpoint_url = endpoint_url) if (validate) validateFile(db, fileName) db diff --git a/R/syncAllFiles.R b/R/syncAllFiles.R index 48b8a80..70e96c8 100644 --- a/R/syncAllFiles.R +++ b/R/syncAllFiles.R @@ -10,7 +10,6 @@ syncAllFiles <- function(db, ...) { #' @rdname rsync #' @export syncAllFiles.default <- function(db, verbose = FALSE, ...) { - args <- if (verbose) "-ltrvvx" else "-ltrx" args <- paste(args, "--delete") @@ -25,15 +24,14 @@ syncAllFiles.default <- function(db, verbose = FALSE, ...) { #' @rdname awss3 #' @export syncAllFiles.awss3 <- function(db, verbose = FALSE, ...) { - args <- if (!verbose) "--quiet --no-progress --only-show-errors" else "" args <- paste("sync", args, "--delete") src <- getSrc(db) dest <- getDest(db) profile <- getProfile(db) + endpoint_url <- db$endpoint_url - awscli(src, dest, args = args, profile = profile) + awscli(src, dest, args = args, profile = profile, endpoint_url = endpoint_url) db - } diff --git a/man/awss3.Rd b/man/awss3.Rd index be19af3..efb650f 100644 --- a/man/awss3.Rd +++ b/man/awss3.Rd @@ -12,7 +12,7 @@ \alias{syncAllFiles.awss3} \title{Connection object to a AWS S3 bucket} \usage{ -awss3(dest, src = getwd(), profile = NULL) +awss3(dest, src = getwd(), profile = NULL, endpoint_url = NULL) profileCreate(profile, force = FALSE) @@ -37,6 +37,9 @@ a profile. In case of a list a new profile will be created which is persistent. A profile is created using \code{aws configure} and stores credentials for the user in plain text.} +\item{endpoint_url}{(NULL|character) The endpoint URL for S3-compatible providers like +Hetzner. E.g. 'https://fsn1.your-objectstorage.com'.} + \item{force}{(logical) override profile if it exists.} \item{db}{(awss3) connection created with \code{awss3}}