From 2597833903767989531c0d8444cf246f8dfc5e5e Mon Sep 17 00:00:00 2001 From: Jan Abel Date: Tue, 30 Sep 2025 15:23:41 +0200 Subject: [PATCH 01/11] add enpoint param to aws command --- R/awscli.R | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/R/awscli.R b/R/awscli.R index 03ca840..228346b 100644 --- a/R/awscli.R +++ b/R/awscli.R @@ -1,4 +1,4 @@ -awscli <- function(src, dest, includes = NULL, excludes = NULL, args = "", profile = NULL, intern = FALSE) { +awscli <- function(src, dest, includes = NULL, excludes = NULL, args = "", profile = NULL, endpoint_url = NULL, intern = FALSE) { constructArg <- function(x, s) { if (is.null(x)) { return(x) @@ -9,6 +9,7 @@ awscli <- function(src, dest, includes = NULL, excludes = NULL, args = "", profi includes <- constructArg(includes, "--include") excludes <- constructArg(excludes, "--exclude") profile <- if (is.null(profile)) "" else paste("--profile", profile) + endpoint <- if (is.null(endpoint_url)) "" else paste("--endpoint-url", endpoint_url) dest <- paste("\"", dest, "\"", sep = "") src <- if (!is.null(src)) paste("\"", src, "\"", sep = "") else NULL @@ -18,6 +19,7 @@ awscli <- function(src, dest, includes = NULL, excludes = NULL, args = "", profi excludes, includes, profile, + endpoint, src, dest ) From 4e2fabe65035dc49e4f16fe2d7acad7f81f154b2 Mon Sep 17 00:00:00 2001 From: Jan Abel Date: Tue, 30 Sep 2025 15:29:04 +0200 Subject: [PATCH 02/11] add enpoint param to awss3 function --- R/awss3.R | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/R/awss3.R b/R/awss3.R index cf8b075..1969f65 100644 --- a/R/awss3.R +++ b/R/awss3.R @@ -10,6 +10,8 @@ #' a profile. In case of a list a new profile will be created which is #' persistent. A profile is created using \code{aws configure} and stores #' credentials for the user in plain text. +#' @param enpoint_url (NULL|character) The enpoint URL for S3-compatible providers like +#' Hetzner. E.g. 'https://fsn1.your-objectstorage.com'. #' @param force (logical) override profile if it exists. #' @param db (awss3) connection created with \code{awss3} #' @param fileName (character) a file name in dest/src @@ -32,13 +34,14 @@ #' #' @rdname awss3 #' @export -awss3 <- function(dest, src = getwd(), profile = NULL) { +awss3 <- function(dest, src = getwd(), profile = NULL, endpoint_url = NULL) { stopifnot( is.character(dest) && length(dest) == 1, is.character(src) && length(src) == 1, is.null(profile) || (is.character(profile) && length(profile) == 1 && profileExists(profile)) || - (is.list(profile) && is.character(profile$name)) + (is.list(profile) && is.character(profile$name)), + is.null(endpoint_url) || is.character(endpoint_url) && length(endpoint_url) == 1 ) src <- if (isS3Bucket(src)) { sub("/$", "", src) @@ -58,6 +61,7 @@ awss3 <- function(dest, src = getwd(), profile = NULL) { dest = dest, src = src, profile = profile + endpoint = endpoint_url ) class(ret) <- "awss3" ret From 84b42419532c0709b6abb60dbac3c8c7ac1278b9 Mon Sep 17 00:00:00 2001 From: Jan Abel Date: Tue, 30 Sep 2025 15:39:47 +0200 Subject: [PATCH 03/11] update methods: add support for custom S3 endpoints --- R/getFile.R | 5 ++--- R/listFiles.R | 3 ++- R/removeFile.R | 3 ++- R/sendAllFiles.R | 3 ++- R/sendFile.R | 3 ++- R/syncAllFiles.R | 6 ++---- 6 files changed, 12 insertions(+), 11 deletions(-) diff --git a/R/getFile.R b/R/getFile.R index e9c91fa..8e4a69f 100644 --- a/R/getFile.R +++ b/R/getFile.R @@ -10,7 +10,6 @@ getFile <- function(db, ...) { #' @rdname rsync #' @export getFile.default <- function(db, fileName, validate = FALSE, verbose = FALSE, ...) { - args <- if (verbose == TRUE) "-ltrvvx" else "-ltrx" args <- paste(args, getArgs(db)) @@ -28,15 +27,15 @@ getFile.default <- function(db, fileName, validate = FALSE, verbose = FALSE, ... #' @rdname awss3 #' @export getFile.awss3 <- function(db, fileName, validate = FALSE, verbose = FALSE, ...) { - args <- if (!verbose) "--quiet --no-progress --only-show-errors" else "" args <- paste("sync ", args) dest <- getDest(db) src <- getSrc(db) profile <- getProfile(db) + endpoint_url <- db$endpoint_url - awscli(dest, src, args = args, excludes = "*", includes = fileName, profile = profile) + awscli(dest, src, args = args, excludes = "*", includes = fileName, profile = profile, endpoint_url = endpoint_url) if (validate) validateFile(db, fileName) diff --git a/R/listFiles.R b/R/listFiles.R index 4cf4268..85bbdfc 100644 --- a/R/listFiles.R +++ b/R/listFiles.R @@ -57,11 +57,12 @@ emptyDir <- function() { listFiles.awss3 <- function(db, recursive = FALSE, ...) { dest <- getDest(db) profile <- getProfile(db) + endpoint_url <- db$endpoint_url if (!isS3Bucket(dest)) { return(NextMethod()) } args <- if (recursive) "ls --recursive" else "ls" - dir <- awscli(NULL, dest, args = args, profile = profile, intern = TRUE) + dir <- awscli(NULL, dest, args = args, profile = profile, enpoint_url = endpoint_url, intern = TRUE) dir <- dat::extract(dir, ~ !grepl("\\.$", .)) if (length(dir) == 0) { return(emptyDir()) diff --git a/R/removeFile.R b/R/removeFile.R index 452125d..ff0238f 100644 --- a/R/removeFile.R +++ b/R/removeFile.R @@ -38,6 +38,7 @@ removeFile.awss3 <- function(db, fileName, verbose = FALSE, ...) { dest <- getDest(db) profile <- getProfile(db) + endpoint_url <- db$endpoint_url if (!isS3Bucket(dest)) { return(NextMethod()) } @@ -45,6 +46,6 @@ removeFile.awss3 <- function(db, fileName, verbose = FALSE, ...) { args <- if (!verbose) "--quiet --only-show-errors --recursive" else "--recursive" args <- paste("rm", args) - awscli(NULL, dest, includes = fileName, excludes = "*", args = args, profile = profile) + awscli(NULL, dest, includes = fileName, excludes = "*", args = args, profile = profile, endpoint_url = endpoint_url) db } diff --git a/R/sendAllFiles.R b/R/sendAllFiles.R index 9974610..f88fddb 100644 --- a/R/sendAllFiles.R +++ b/R/sendAllFiles.R @@ -22,7 +22,8 @@ sendAllFiles.awss3 <- function(db, verbose = FALSE, ...) { src <- getSrc(db) dest <- getDest(db) profile <- getProfile(db) + endpoint_url <- db$endpoint_url - awscli(src, dest, args = args, profile = profile) + awscli(src, dest, args = args, profile = profile, endpoint_url = endpoint_url) db } diff --git a/R/sendFile.R b/R/sendFile.R index 651b111..e8173f8 100644 --- a/R/sendFile.R +++ b/R/sendFile.R @@ -35,8 +35,9 @@ sendFile.awss3 <- function(db, fileName, validate = FALSE, verbose = FALSE, args src <- getSrc(db) dest <- getDest(db) profile <- getProfile(db) + endpoint_url <- db$endpoint_url - awscli(src, dest, args = args, excludes = "*", includes = fileName, profile = profile) + awscli(src, dest, args = args, excludes = "*", includes = fileName, profile = profile, endpoint_url = endpoint_url) if (validate) validateFile(db, fileName) db diff --git a/R/syncAllFiles.R b/R/syncAllFiles.R index 48b8a80..70e96c8 100644 --- a/R/syncAllFiles.R +++ b/R/syncAllFiles.R @@ -10,7 +10,6 @@ syncAllFiles <- function(db, ...) { #' @rdname rsync #' @export syncAllFiles.default <- function(db, verbose = FALSE, ...) { - args <- if (verbose) "-ltrvvx" else "-ltrx" args <- paste(args, "--delete") @@ -25,15 +24,14 @@ syncAllFiles.default <- function(db, verbose = FALSE, ...) { #' @rdname awss3 #' @export syncAllFiles.awss3 <- function(db, verbose = FALSE, ...) { - args <- if (!verbose) "--quiet --no-progress --only-show-errors" else "" args <- paste("sync", args, "--delete") src <- getSrc(db) dest <- getDest(db) profile <- getProfile(db) + endpoint_url <- db$endpoint_url - awscli(src, dest, args = args, profile = profile) + awscli(src, dest, args = args, profile = profile, endpoint_url = endpoint_url) db - } From ce7e0774f8f28225e46798647a8854c58a1b3fa6 Mon Sep 17 00:00:00 2001 From: Jan Abel Date: Tue, 30 Sep 2025 15:43:35 +0200 Subject: [PATCH 04/11] typo: add comma --- R/awss3.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/awss3.R b/R/awss3.R index 1969f65..6888d6c 100644 --- a/R/awss3.R +++ b/R/awss3.R @@ -60,7 +60,7 @@ awss3 <- function(dest, src = getwd(), profile = NULL, endpoint_url = NULL) { ret <- list( dest = dest, src = src, - profile = profile + profile = profile, endpoint = endpoint_url ) class(ret) <- "awss3" From 07712da67d1ef76aa41d5d3a6ca738f7b74cea48 Mon Sep 17 00:00:00 2001 From: Jan Abel Date: Tue, 30 Sep 2025 15:44:09 +0200 Subject: [PATCH 05/11] Update roxygen docs --- DESCRIPTION | 2 +- man/awss3.Rd | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 3681bf1..ee81982 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -24,4 +24,4 @@ Suggests: License: MIT + file LICENSE Encoding: UTF-8 LazyData: true -RoxygenNote: 7.3.1 +RoxygenNote: 7.3.2 diff --git a/man/awss3.Rd b/man/awss3.Rd index be19af3..3566f70 100644 --- a/man/awss3.Rd +++ b/man/awss3.Rd @@ -12,7 +12,7 @@ \alias{syncAllFiles.awss3} \title{Connection object to a AWS S3 bucket} \usage{ -awss3(dest, src = getwd(), profile = NULL) +awss3(dest, src = getwd(), profile = NULL, endpoint_url = NULL) profileCreate(profile, force = FALSE) @@ -52,6 +52,9 @@ credentials for the user in plain text.} \item{recursive}{(logical) if TRUE print full names for files in sub folders} \item{args}{(character) pass additional args to aws cli. Currently only implemented for sendFile} + +\item{enpoint_url}{(NULL|character) The enpoint URL for S3-compatible providers like +Hetzner. E.g. 'https://fsn1.your-objectstorage.com'.} } \description{ Only methods specific to this class are documented here. For others the From 35779f7924c28d7ae81e01d411a5533cf31f79ba Mon Sep 17 00:00:00 2001 From: Jan Abel Date: Tue, 30 Sep 2025 15:47:34 +0200 Subject: [PATCH 06/11] Update Version to 25.10.0 --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index ee81982..54f6ca0 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: rsync Type: Package Title: API to use rsync -Version: 24.10.0 +Version: 25.10.0 Author: INWT Statistics GmbH Authors@R: c(person("Sebastian", "Warnholz", email = "sebastian.warnholz@inwt-statistics.de", role = c("aut", "cre")), person("Jonathan", "Bob", email = "jonathan.bob@inwt-statistics.de", role = c("aut")), From 5df1e8fefd56acb38569215d41fab002c1d9c1f7 Mon Sep 17 00:00:00 2001 From: Jan Abel Date: Tue, 30 Sep 2025 15:48:13 +0200 Subject: [PATCH 07/11] fix typo in docs --- R/awss3.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/awss3.R b/R/awss3.R index 6888d6c..d9918b0 100644 --- a/R/awss3.R +++ b/R/awss3.R @@ -10,7 +10,7 @@ #' a profile. In case of a list a new profile will be created which is #' persistent. A profile is created using \code{aws configure} and stores #' credentials for the user in plain text. -#' @param enpoint_url (NULL|character) The enpoint URL for S3-compatible providers like +#' @param endpoint_url (NULL|character) The endpoint URL for S3-compatible providers like #' Hetzner. E.g. 'https://fsn1.your-objectstorage.com'. #' @param force (logical) override profile if it exists. #' @param db (awss3) connection created with \code{awss3} From 60c41ff4e8594e2a192dfdb5c8f15900f5d7137e Mon Sep 17 00:00:00 2001 From: Jan Abel Date: Tue, 30 Sep 2025 15:49:23 +0200 Subject: [PATCH 08/11] fix typo in argument --- R/listFiles.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/listFiles.R b/R/listFiles.R index 85bbdfc..93ba396 100644 --- a/R/listFiles.R +++ b/R/listFiles.R @@ -62,7 +62,7 @@ listFiles.awss3 <- function(db, recursive = FALSE, ...) { return(NextMethod()) } args <- if (recursive) "ls --recursive" else "ls" - dir <- awscli(NULL, dest, args = args, profile = profile, enpoint_url = endpoint_url, intern = TRUE) + dir <- awscli(NULL, dest, args = args, profile = profile, endpoint_url = endpoint_url, intern = TRUE) dir <- dat::extract(dir, ~ !grepl("\\.$", .)) if (length(dir) == 0) { return(emptyDir()) From 8dd18b780d631fdc267b78b7cdf95171d2bc32ef Mon Sep 17 00:00:00 2001 From: Jan Abel Date: Tue, 30 Sep 2025 15:49:46 +0200 Subject: [PATCH 09/11] Update roxygen docs --- man/awss3.Rd | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/man/awss3.Rd b/man/awss3.Rd index 3566f70..efb650f 100644 --- a/man/awss3.Rd +++ b/man/awss3.Rd @@ -37,6 +37,9 @@ a profile. In case of a list a new profile will be created which is persistent. A profile is created using \code{aws configure} and stores credentials for the user in plain text.} +\item{endpoint_url}{(NULL|character) The endpoint URL for S3-compatible providers like +Hetzner. E.g. 'https://fsn1.your-objectstorage.com'.} + \item{force}{(logical) override profile if it exists.} \item{db}{(awss3) connection created with \code{awss3}} @@ -52,9 +55,6 @@ credentials for the user in plain text.} \item{recursive}{(logical) if TRUE print full names for files in sub folders} \item{args}{(character) pass additional args to aws cli. Currently only implemented for sendFile} - -\item{enpoint_url}{(NULL|character) The enpoint URL for S3-compatible providers like -Hetzner. E.g. 'https://fsn1.your-objectstorage.com'.} } \description{ Only methods specific to this class are documented here. For others the From 009ac298e9aa4d7a08d80d59d8a2200d60c3ab3c Mon Sep 17 00:00:00 2001 From: Jan Abel Date: Tue, 30 Sep 2025 16:56:46 +0200 Subject: [PATCH 10/11] fix name in list --- R/awss3.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/awss3.R b/R/awss3.R index d9918b0..d3b9355 100644 --- a/R/awss3.R +++ b/R/awss3.R @@ -61,7 +61,7 @@ awss3 <- function(dest, src = getwd(), profile = NULL, endpoint_url = NULL) { dest = dest, src = src, profile = profile, - endpoint = endpoint_url + endpoint_url = endpoint_url ) class(ret) <- "awss3" ret From ddc2b5609c855549aee0e1647737aa4de6d081ec Mon Sep 17 00:00:00 2001 From: Antonia Runge Date: Thu, 2 Oct 2025 16:40:26 +0200 Subject: [PATCH 11/11] add news.md --- NEWS.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 NEWS.md diff --git a/NEWS.md b/NEWS.md new file mode 100644 index 0000000..d74d5b7 --- /dev/null +++ b/NEWS.md @@ -0,0 +1,4 @@ +# rsync 25.10.0 + +## Added +- Added support for custom S3 endpoints (e.g., Hetzner) \ No newline at end of file