diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 3db34618..d6276a4b 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -26,10 +26,11 @@ jobs: with: go-version: 1.20 id: go - - name: Force NDK version - run: echo "y" | sudo ${ANDROID_HOME}/tools/bin/sdkmanager --install "ndk;25.0.8775105" - - name: Build rclone - run: ./gradlew rclone:buildAll + - name: Install NDK + run: | + NDK_VERSION="$(grep -E "^de\.felixnuesse\.extract\.ndkVersion=" gradle.properties | cut -d'=' -f2)" + yes | sudo "${ANDROID_HOME}/tools/bin/sdkmanager" --licenses + sudo "${ANDROID_HOME}/tools/bin/sdkmanager" "ndk;${NDK_VERSION}" - name: Build app run: ./gradlew assembleOssDebug - name: Upload APK (arm) diff --git a/.gitignore b/.gitignore index 12538b7c..dea6586d 100644 --- a/.gitignore +++ b/.gitignore @@ -59,7 +59,6 @@ freeline_project_description.json # Don't version native libraries or IDE artifacts. Module config should be done in gradle. # Generated artifacts (apks, libraries) just slow down cloning and can be recreated from source # anyways. -app/lib/ release/ .idea/encodings.xml diff --git a/README.md b/README.md index d4806ccf..0927e4a8 100644 --- a/README.md +++ b/README.md @@ -72,21 +72,23 @@ Usage Developing ------------ -[See the developer-documentation](https://x0b.github.io/dev/). -Build rclone manually run: +You should first make sure you have: -``` -./gradlew rclone:buildAll -``` +- Go 1.20+ installed and in your PATH +- Java installed and in your PATH +- Android SDK command-line tools installed OR the NDK version specified in `gradle.properties` + installed -When building rclone with a go version that is too old (eg 1.15.5), this error may show up: +You can then build the app normally from Android Studio or from CLI by running: -``` -ld.lld: error: duplicate symbol: x_cgo_inittls -``` +```sh +# Debug build +./gradlew assembleOssDebug -It can be fixed by using a more recent version of go. +# or release build +./gradlew assembleOssRelease +``` Known Issues ------------ @@ -98,20 +100,6 @@ Contributing See [CONTRIBUTING](./CONTRIBUTING.md) -Building ------------- -``` -// choose the appropriate version for your device -cd rclone -../gradlew rclone:buildNative // For all devices -../gradlew rclone:buildArm64 -../gradlew rclone:buildArm -../gradlew rclone:buildx86 -../gradlew rclone:buildx64 -``` - - - License ----------------- ### About this app diff --git a/app/.gitignore b/app/.gitignore index 796b96d1..ddff9a24 100644 --- a/app/.gitignore +++ b/app/.gitignore @@ -1 +1,2 @@ /build +/lib diff --git a/app/build.gradle b/app/build.gradle index 14ca22ef..053d6d24 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -3,6 +3,13 @@ import java.text.SimpleDateFormat apply plugin: 'com.android.application' apply plugin: 'kotlin-android' +tasks.whenTaskAdded { task -> + // Build rclone beforehand. + if (task.name.startsWith('compile')) { + task.dependsOn(':rclone:buildAll') + } +} + android { signingConfigs { github_x0b { @@ -10,7 +17,7 @@ android { } } compileSdkVersion 33 - ndkVersion '25.0.8775105' + ndkVersion project.properties['de.felixnuesse.extract.ndkVersion'] defaultConfig { applicationId 'de.felixnuesse.extract' minSdkVersion 23 @@ -67,10 +74,11 @@ android { } project.ext.versionCodes = [ - 'armeabi-v7a': 6, - 'arm64-v8a' : 7, - 'x86' : 8, - 'x86_64' : 9] + 'armeabi-v7a': 6, + 'arm64-v8a' : 7, + 'x86' : 8, + 'x86_64' : 9 + ] android.applicationVariants.all { variant -> variant.outputs.each { output -> diff --git a/build.gradle b/build.gradle index 70fdb266..1857734e 100644 --- a/build.gradle +++ b/build.gradle @@ -1,7 +1,6 @@ // Top-level build file where you can add configuration options common to all sub-projects/modules. buildscript { - ext.kotlinVersion = '1.7.20' repositories { google() diff --git a/gradle.properties b/gradle.properties index 9f6c4497..5899df13 100644 --- a/gradle.properties +++ b/gradle.properties @@ -14,3 +14,6 @@ org.gradle.jvmargs=-Xmx2048M -Dkotlin.daemon.jvm.options\="-Xmx2048M" android.enableJetifier=false android.useAndroidX=true +de.felixnuesse.extract.rCloneVersion=1.61.1 +de.felixnuesse.extract.ndkVersion=25.0.8775105 +de.felixnuesse.extract.ndkToolchainVersion=33 diff --git a/rclone/.gitignore b/rclone/.gitignore index 71a8716d..14d86ad6 100644 --- a/rclone/.gitignore +++ b/rclone/.gitignore @@ -1 +1 @@ -/gopath +/cache diff --git a/rclone/build.gradle b/rclone/build.gradle index ebdfbd9f..4d3ec47f 100644 --- a/rclone/build.gradle +++ b/rclone/build.gradle @@ -9,193 +9,209 @@ // // Supported host systems: // - linux x64 -// - mac os x64 +// - macOS x64 +// - macOS arm64 (Silicon) // - windows x64 -// - windows x86 (with NDK 21b installed) // -// Prerequisits: -// - go 1.18 -// - ndk +// Prerequisites: +// - go 1.20+ +// - Either Android SDK command-line tools, or the expected NDK version (see gradle.properties). -// Rclone version - any git reference (tag, branch, hash) should work -def buildTag = 'v1.61.1' -ext.ndkVersion = '25.0.8775105' - -// -// DO NOT EDIT ANYTHING BELOW -// - -import java.nio.file.Files import java.nio.file.Paths -def configureNdk() { - def osName = System.properties['os.name'].toLowerCase() - def osArch = System.properties['os.arch'] +ext { + NDK_VERSION = project.properties['de.felixnuesse.extract.ndkVersion'] + NDK_TOOLCHAIN_VERSION = project.properties['de.felixnuesse.extract.ndkToolchainVersion'] + RCLONE_VERSION = project.properties['de.felixnuesse.extract.rCloneVersion'] + RCLONE_MODULE = 'github.com/rclone/rclone' + RCLONE_CUSTOM_VERSION_SUFFIX = '-extract' + + PROJECT_DIR = projectDir.absolutePath + CACHE_PATH = Paths.get(PROJECT_DIR, 'cache').toString() + GOPATH = Paths.get(CACHE_PATH, 'gopath').toString() + OUTPUT_BASE_PATH = Paths.get(PROJECT_DIR, '..', 'app', 'lib').toAbsolutePath().toString() +} + +def findSdkDir() { + def androidHome = System.getenv('ANDROID_HOME') + if (androidHome != null) { + return androidHome + } - def os = '' - if (osName.contains('windows')) { - if(osArch.equals('amd64')) { - os = "windows-x86_64" - } else if (osArch.equals('x86')) { - // NDK has dropped x86 windows support in NDK 21 and greater. This - // may break at for any reason when the golang tolchain is - // upgraded. - os = "windows" - ext.ndkVersion = '20.1.5948944' + def localPropertiesFile = project.rootProject.file('local.properties') + if (localPropertiesFile.exists()) { + Properties properties = new Properties() + properties.load(localPropertiesFile.newDataInputStream()) + def sdkDir = properties.get('sdk.dir') + if (sdkDir != null) { + return sdkDir } - } else if (osName.contains("linux")) { - os = "linux-x86_64" - } else if (osName.contains('mac')) { - os = "darwin-x86_64" - } else { - throw new GradleException("OS=${osName}/ARCH=${osArch} not supported") } + throw new GradleException( + "Couldn't locate your android SDK. Make sure you set sdk.dir property" + + " in your local.properties at the root of the project or set" + + " ANDROID_HOME environment variable." + ) +} - // locate NDK - def androidHome = System.getenv('HOME')+"/Android/Sdk/" - def ndkBasePath - if (androidHome != null) { - def canonicalPath = Paths.get(androidHome, 'ndk', ext.ndkVersion) - def bundlePath = Paths.get(androidHome, 'ndk-bundle') - if (Files.exists(canonicalPath) && checkNdk(canonicalPath)) { - ndkBasePath = canonicalPath; - } else if (Files.exists(bundlePath) && checkNdk(bundlePath)) { - ndkBasePath = bundlePath; +def findNdkDir() { + def sdkDir = findSdkDir() + def ndkPath = Paths.get(sdkDir, 'ndk', NDK_VERSION).toAbsolutePath() + if (!ndkPath.toFile().exists()) { + // NDK not found. Let's try to install it. + def sdkManagerPath = Paths.get( + sdkDir, + 'cmdline-tools', + 'latest', + 'bin', + 'sdkmanager' + ).toString() + if (System.properties['os.name'].startsWith('Windows')) { + sdkManagerPath += '.bat' + } + try { + exec { + commandLine sdkManagerPath, "--install", "ndk;${NDK_VERSION}" + } + } catch (exc) { + logger.error(exc.toString()) + // NDK installation failed. Just raise an error. + throw new GradleException( + "Couldn't find a ndk bundle in ${ndkPath.toString()}. Make sure that you have the" + + " proper version installed in Android Studio's SDK Manager or run" + + " \"${sdkManagerPath} --install 'ndk;${NDK_VERSION}'\"." + ) } - } else if (androidNdkHome != null && checkNdk(androidNdkHome)) { - ndkBasePath = Paths.get(androidNdkHome) } - // check NDK + return ndkPath.toString() +} - if (ndkBasePath == null) { - throw GradleException("NDK ${ext.ndkVersion} not found") +def getCrossCompiler(abi) { + def osName = System.properties['os.name'] + def osArch = System.properties['os.arch'] + def os = null + if (osName.startsWith('Windows') && osArch == 'amd64') { + os = 'windows-x86_64' + } else if (osName.startsWith('Linux') && osArch == 'amd64') { + os = 'linux-x86_64' + } else if (osName.startsWith('Mac') && ['aarch64', 'amd64'].contains(osArch)) { + // Note that despite what the name suggests, the clang binary shipped + // with the NDK is a universal object file which should work on both + // x86_64 and arm64 (Silicon-based) architectures. + os = 'darwin-x86_64' + } + if (os == null) { + throw new GradleException('Unsupported host OS or architecture.') } - return ndkBasePath.resolve(Paths.get('toolchains', 'llvm', 'prebuilt', os, 'bin')) + def abiToCompiler = [ + 'armeabi-v7a': "armv7a-linux-androideabi${NDK_TOOLCHAIN_VERSION}-clang", + 'arm64-v8a': "aarch64-linux-android${NDK_TOOLCHAIN_VERSION}-clang", + 'x86': "i686-linux-android${NDK_TOOLCHAIN_VERSION}-clang", + 'x86_64': "x86_64-linux-android${NDK_TOOLCHAIN_VERSION}-clang", + ] + + return Paths.get( + findNdkDir(), + 'toolchains', + 'llvm', + 'prebuilt', + os, + 'bin', + abiToCompiler[abi], + ) } -def checkNdk(ndkBasePath) { - def propertiesPath = ndkBasePath.resolve('source.properties') - def ndkProperties = new Properties() - ndkProperties.load(file(propertiesPath).newReader()) - return ndkProperties['Pkg.Revision'] == ext.ndkVersion +def getOutputPath(abi) { + return Paths.get(OUTPUT_BASE_PATH, abi, 'librclone.so').toString() } -def configureGo() { - def localGo = Paths.get('golang/go/bin/go') - return Files.exists(localGo) ? localGo : 'go' +def buildRclone(abi) { + def abiToEnv = [ + 'armeabi-v7a': ['GOARCH': 'arm', 'GOARM': '7'], + 'arm64-v8a': ['GOARCH': 'arm64'], + 'x86': ['GOARCH': '386'], + 'x86_64': ['GOARCH': 'amd64'], + ] + + return { + doLast { + exec { + environment 'GOPATH', GOPATH + def crossCompiler = getCrossCompiler(abi) + environment 'CC', crossCompiler + environment 'CC_FOR_TARGET', crossCompiler + environment 'GOOS', 'android' + environment 'CGO_ENABLED', '1' + environment 'CGO_LDFLAGS', '-fuse-ld=lld -s' + abiToEnv[abi].each {entry -> environment entry.key, entry.value} + workingDir CACHE_PATH + def ldflags = "-X github.com/rclone/rclone/fs.Version=${RCLONE_VERSION}${RCLONE_CUSTOM_VERSION_SUFFIX}" + commandLine ( + 'go', + 'build', + '-tags', + 'android noselfupdate', + '-trimpath', + '-ldflags', + ldflags, + '-o', + getOutputPath(abi), + RCLONE_MODULE + ) + } + } + } } -def repository = 'github.com/rclone/rclone' -def repositoryRef = repository + '@' + buildTag -def ldflags = "-X github.com/rclone/rclone/fs.Version=${buildTag}-rcx" -def goPath = Paths.get(projectDir.absolutePath, 'gopath').toAbsolutePath().toString() -def appLibPath = Paths.get(projectDir.absolutePath, '../app/lib').toAbsolutePath().toString() -def ndkPrefix = configureNdk() -def goBinary = configureGo() - - -task fetchRclone(type: Exec) { - mkdir "gopath" - environment 'GOPATH', goPath - environment "GO111MODULE", "on" - commandLine 'go', 'install', repositoryRef +task createRcloneModule(type: Exec) { + // We create a dummy go module to be able to checkout our specific rclone + // version later on. + onlyIf { !Paths.get(CACHE_PATH, 'go.mod').toFile().exists() } + Paths.get(CACHE_PATH).toFile().mkdirs() + workingDir CACHE_PATH + environment 'GOPATH', GOPATH + commandLine 'go', 'mod', 'init', 'rclone' +} - ignoreExitValue true - errorOutput = new ByteArrayOutputStream() - doLast { - if (execResult.getExitValue() != 0) { - throw new GradleException("Error running go get: \n${errorOutput.toString()}") - } - } +task checkoutRclone(type: Exec, dependsOn: createRcloneModule) { + workingDir CACHE_PATH + environment 'GOPATH', GOPATH + commandLine 'go', 'get', '-v', '-d', "${RCLONE_MODULE}@v${RCLONE_VERSION}" } -task cleanNative { - enabled = false - doLast { - delete "${appLibPath}/armeabi-v7a/librclone.so" - delete "${appLibPath}/arm64-v8a/librclone.so" - delete "${appLibPath}/x86/librclone.so" - delete "${appLibPath}/x86_64/librclone.so" - } +task buildArm(dependsOn: checkoutRclone) { + configure buildRclone('armeabi-v7a') } -task buildArm(type: Exec) { - dependsOn fetchRclone - environment 'GOPATH', goPath - def crossCompiler = ndkPrefix.resolve('armv7a-linux-androideabi21-clang') - environment 'CC', crossCompiler - environment 'CC_FOR_TARGET', crossCompiler - environment 'GOOS', 'android' - environment 'GOARCH', 'arm' - environment 'GOARM', '7' - environment 'CGO_ENABLED', '1' - environment 'CGO_LDFLAGS', "-fuse-ld=lld -s" - workingDir Paths.get(goPath, "pkg/mod/${repositoryRef}".split('/')) - def artifactTarget = "${appLibPath}/armeabi-v7a/librclone.so" - commandLine 'go', 'build', '-v', '-tags', 'android noselfupdate', '-trimpath', '-ldflags', ldflags, '-o', artifactTarget, '.' - - ignoreExitValue true - errorOutput = new ByteArrayOutputStream() - doLast { - if (execResult.getExitValue() != 0) { - throw new GradleException("Error running go build: \n${errorOutput.toString()}") - } - } +task buildArm64(dependsOn: checkoutRclone) { + configure buildRclone('arm64-v8a') } -task buildArm64(type: Exec) { - dependsOn fetchRclone - environment 'GOPATH', goPath - def crossCompiler = ndkPrefix.resolve('aarch64-linux-android21-clang') - environment 'CC', crossCompiler - environment 'CC_FOR_TARGET', crossCompiler - environment 'GOOS', 'android' - environment 'GOARCH', 'arm64' - environment 'CGO_ENABLED', '1' - environment 'CGO_LDFLAGS', "-fuse-ld=lld -s" - workingDir Paths.get(goPath, "pkg/mod/${repositoryRef}".split('/')) - def artifactTarget = "${appLibPath}/arm64-v8a/librclone.so" - commandLine 'go', 'build', '-v', '-tags', 'android noselfupdate', '-trimpath', '-ldflags', ldflags, '-o', artifactTarget, '.'} - -task buildx86(type: Exec) { - dependsOn fetchRclone - environment 'GOPATH', goPath - def crossCompiler = ndkPrefix.resolve('i686-linux-android21-clang') - environment 'CC', crossCompiler - environment 'CC_FOR_TARGET', crossCompiler - environment 'GOOS', 'android' - environment 'GOARCH', '386' - environment 'CGO_ENABLED', '1' - environment 'CGO_LDFLAGS', "-fuse-ld=lld -s" - workingDir Paths.get(goPath, "pkg/mod/${repositoryRef}".split('/')) - def artifactTarget = "${appLibPath}/x86/librclone.so" - commandLine 'go', 'build', '-v', '-tags', 'android noselfupdate', '-trimpath', '-ldflags', ldflags, '-o', artifactTarget, '.' +task buildx86(dependsOn: checkoutRclone) { + configure buildRclone('x86') } -task buildx64(type: Exec) { - dependsOn fetchRclone - environment 'GOPATH', goPath - def crossCompiler = ndkPrefix.resolve('x86_64-linux-android21-clang') - environment 'CC', crossCompiler - environment 'CC_FOR_TARGET', crossCompiler - environment 'GOOS', 'android' - environment 'GOARCH', 'amd64' - environment 'CGO_ENABLED', '1' - environment 'CGO_LDFLAGS', "-fuse-ld=lld -s" - workingDir Paths.get(goPath, "pkg/mod/${repositoryRef}".split('/')) - def artifactTarget = "${appLibPath}/x86_64/librclone.so" - commandLine 'go', 'build', '-v', '-tags', 'android noselfupdate', '-trimpath', '-ldflags', ldflags, '-o', artifactTarget, '.' +task buildx64(dependsOn: checkoutRclone) { + configure buildRclone('x86_64') } task buildAll { - dependsOn fetchRclone - dependsOn buildArm - dependsOn buildArm64 - dependsOn buildx86 - dependsOn buildx64 + dependsOn buildArm, buildArm64, buildx86, buildx64 +} + +task clean { + doLast { + exec { + environment 'GOPATH', GOPATH + commandLine 'go', 'clean', '-cache', '-testcache', '-modcache', '-fuzzcache' + } + delete CACHE_PATH + delete fileTree(OUTPUT_BASE_PATH).matching { + include '**/librclone.so' + } + } } -buildAll.mustRunAfter(buildArm, buildArm64, buildx86, buildx64) defaultTasks 'buildAll'