diff --git a/Android/README.md b/Android/README.md index 789bcbe5edff44..6cabd6ba5d6844 100644 --- a/Android/README.md +++ b/Android/README.md @@ -25,11 +25,13 @@ it: `android-sdk/cmdline-tools/latest`. * `export ANDROID_HOME=/path/to/android-sdk` -The `android.py` script also requires the following commands to be on the `PATH`: +The `android.py` script will automatically use the SDK's `sdkmanager` to install +any packages it needs. + +The script also requires the following commands to be on the `PATH`: * `curl` * `java` (or set the `JAVA_HOME` environment variable) -* `tar` ## Building @@ -97,7 +99,7 @@ similar to the `Android` directory of the CPython source tree. The Python test suite can be run on Linux, macOS, or Windows: * On Linux, the emulator needs access to the KVM virtualization interface, and - a DISPLAY environment variable pointing at an X server. + a DISPLAY environment variable pointing at an X server. Xvfb is acceptable. The test suite can usually be run on a device with 2 GB of RAM, but this is borderline, so you may need to increase it to 4 GB. As of Android diff --git a/Android/android-env.sh b/Android/android-env.sh index bab4130c9e92d0..ae1385034a37f2 100644 --- a/Android/android-env.sh +++ b/Android/android-env.sh @@ -3,7 +3,7 @@ : "${HOST:?}" # GNU target triplet # You may also override the following: -: "${api_level:=24}" # Minimum Android API level the build will run on +: "${ANDROID_API_LEVEL:=24}" # Minimum Android API level the build will run on : "${PREFIX:-}" # Path in which to find required libraries @@ -43,7 +43,7 @@ fi toolchain=$(echo "$ndk"/toolchains/llvm/prebuilt/*) export AR="$toolchain/bin/llvm-ar" export AS="$toolchain/bin/llvm-as" -export CC="$toolchain/bin/${clang_triplet}${api_level}-clang" +export CC="$toolchain/bin/${clang_triplet}${ANDROID_API_LEVEL}-clang" export CXX="${CC}++" export LD="$toolchain/bin/ld" export NM="$toolchain/bin/llvm-nm" diff --git a/Android/android.py b/Android/android.py index 1b20820b784371..b410f3e2eec22f 100755 --- a/Android/android.py +++ b/Android/android.py @@ -22,9 +22,13 @@ SCRIPT_NAME = Path(__file__).name ANDROID_DIR = Path(__file__).resolve().parent -CHECKOUT = ANDROID_DIR.parent +PYTHON_DIR = ANDROID_DIR.parent +in_source_tree = ( + ANDROID_DIR.name == "Android" and (PYTHON_DIR / "pyconfig.h.in").exists() +) + TESTBED_DIR = ANDROID_DIR / "testbed" -CROSS_BUILD_DIR = CHECKOUT / "cross-build" +CROSS_BUILD_DIR = PYTHON_DIR / "cross-build" HOSTS = ["aarch64-linux-android", "x86_64-linux-android"] APP_ID = "org.python.testbed" @@ -74,41 +78,61 @@ def subdir(*parts, create=False): def run(command, *, host=None, env=None, log=True, **kwargs): kwargs.setdefault("check", True) + if env is None: env = os.environ.copy() - original_env = env.copy() - if host: - env_script = ANDROID_DIR / "android-env.sh" - env_output = subprocess.run( - f"set -eu; " - f"HOST={host}; " - f"PREFIX={subdir(host)}/prefix; " - f". {env_script}; " - f"export", - check=True, shell=True, text=True, stdout=subprocess.PIPE - ).stdout - - for line in env_output.splitlines(): - # We don't require every line to match, as there may be some other - # output from installing the NDK. - if match := re.search( - "^(declare -x |export )?(\\w+)=['\"]?(.*?)['\"]?$", line - ): - key, value = match[2], match[3] - if env.get(key) != value: - print(line) - env[key] = value - - if env == original_env: - raise ValueError(f"Found no variables in {env_script.name} output:\n" - + env_output) + env.update(android_env(host)) if log: print(">", " ".join(map(str, command))) return subprocess.run(command, env=env, **kwargs) +def print_env(context): + android_env(getattr(context, "host", None)) + + +def android_env(host): + if host: + prefix = subdir(host) / "prefix" + else: + prefix = ANDROID_DIR / "prefix" + sysconfig_files = prefix.glob("lib/python*/_sysconfigdata__android_*.py") + sysconfig_filename = next(sysconfig_files).name + host = re.fullmatch(r"_sysconfigdata__android_(.+).py", sysconfig_filename)[1] + + env_script = ANDROID_DIR / "android-env.sh" + env_output = subprocess.run( + f"set -eu; " + f"export HOST={host}; " + f"PREFIX={prefix}; " + f". {env_script}; " + f"export", + check=True, shell=True, capture_output=True, encoding='utf-8', + ).stdout + + env = {} + for line in env_output.splitlines(): + # We don't require every line to match, as there may be some other + # output from installing the NDK. + if match := re.search( + "^(declare -x |export )?(\\w+)=['\"]?(.*?)['\"]?$", line + ): + key, value = match[2], match[3] + if os.environ.get(key) != value: + env[key] = value + + if not env: + raise ValueError(f"Found no variables in {env_script.name} output:\n" + + env_output) + + # Format the environment so it can be pasted into a shell. + for key, value in sorted(env.items()): + print(f"export {key}={shlex.quote(value)}") + return env + + def build_python_path(): """The path to the build Python binary.""" build_dir = subdir("build") @@ -127,7 +151,7 @@ def configure_build_python(context): clean("build") os.chdir(subdir("build", create=True)) - command = [relpath(CHECKOUT / "configure")] + command = [relpath(PYTHON_DIR / "configure")] if context.args: command.extend(context.args) run(command) @@ -138,19 +162,19 @@ def make_build_python(context): run(["make", "-j", str(os.cpu_count())]) -def unpack_deps(host): +def unpack_deps(host, prefix_dir): deps_url = "https://github.com/beeware/cpython-android-source-deps/releases/download" for name_ver in ["bzip2-1.0.8-2", "libffi-3.4.4-3", "openssl-3.0.15-4", "sqlite-3.49.1-0", "xz-5.4.6-1"]: filename = f"{name_ver}-{host}.tar.gz" download(f"{deps_url}/{name_ver}/{filename}") - run(["tar", "-xf", filename]) + shutil.unpack_archive(filename, prefix_dir) os.remove(filename) def download(url, target_dir="."): out_path = f"{target_dir}/{basename(url)}" - run(["curl", "-Lf", "-o", out_path, url]) + run(["curl", "-Lf", "--retry", "5", "--retry-all-errors", "-o", out_path, url]) return out_path @@ -162,13 +186,12 @@ def configure_host_python(context): prefix_dir = host_dir / "prefix" if not prefix_dir.exists(): prefix_dir.mkdir() - os.chdir(prefix_dir) - unpack_deps(context.host) + unpack_deps(context.host, prefix_dir) os.chdir(host_dir) command = [ # Basic cross-compiling configuration - relpath(CHECKOUT / "configure"), + relpath(PYTHON_DIR / "configure"), f"--host={context.host}", f"--build={sysconfig.get_config_var('BUILD_GNU_TYPE')}", f"--with-build-python={build_python_path()}", @@ -241,16 +264,15 @@ def setup_sdk(): # the Gradle wrapper is not included in the CPython repository. Instead, we # extract it from the Gradle GitHub repository. def setup_testbed(): - # The Gradle version used for the build is specified in - # testbed/gradle/wrapper/gradle-wrapper.properties. This wrapper version - # doesn't need to match, as any version of the wrapper can download any - # version of Gradle. - version = "8.9.0" paths = ["gradlew", "gradlew.bat", "gradle/wrapper/gradle-wrapper.jar"] - if all((TESTBED_DIR / path).exists() for path in paths): return + # The wrapper version isn't important, as any version of the wrapper can + # download any version of Gradle. The Gradle version actually used for the + # build is specified in testbed/gradle/wrapper/gradle-wrapper.properties. + version = "8.9.0" + for path in paths: out_path = TESTBED_DIR / path out_path.parent.mkdir(exist_ok=True) @@ -624,8 +646,7 @@ def parse_args(): configure_build = subcommands.add_parser("configure-build", help="Run `configure` for the " "build Python") - make_build = subcommands.add_parser("make-build", - help="Run `make` for the build Python") + subcommands.add_parser("make-build", help="Run `make` for the build Python") configure_host = subcommands.add_parser("configure-host", help="Run `configure` for Android") make_host = subcommands.add_parser("make-host", @@ -637,16 +658,22 @@ def parse_args(): test = subcommands.add_parser( "test", help="Run the test suite") package = subcommands.add_parser("package", help="Make a release package") + env = subcommands.add_parser("env", help="Print environment variables") # Common arguments for subcommand in build, configure_build, configure_host: subcommand.add_argument( "--clean", action="store_true", default=False, dest="clean", help="Delete the relevant build and prefix directories first") - for subcommand in [build, configure_host, make_host, package]: + + host_commands = [build, configure_host, make_host, package] + if in_source_tree: + host_commands.append(env) + for subcommand in host_commands: subcommand.add_argument( "host", metavar="HOST", choices=HOSTS, help="Host triplet: choices=[%(choices)s]") + for subcommand in build, configure_build, configure_host: subcommand.add_argument("args", nargs="*", help="Extra arguments to pass to `configure`") @@ -690,6 +717,7 @@ def main(): "build-testbed": build_testbed, "test": run_testbed, "package": package, + "env": print_env, } try: diff --git a/Android/testbed/app/build.gradle.kts b/Android/testbed/app/build.gradle.kts index c627cb1b0e0b22..2a284f619db9ec 100644 --- a/Android/testbed/app/build.gradle.kts +++ b/Android/testbed/app/build.gradle.kts @@ -85,7 +85,7 @@ android { minSdk = androidEnvFile.useLines { for (line in it) { - """api_level:=(\d+)""".toRegex().find(line)?.let { + """ANDROID_API_LEVEL:=(\d+)""".toRegex().find(line)?.let { return@useLines it.groupValues[1].toInt() } }