diff --git a/.gitignore b/.gitignore index a1f0fdfba78..88771b805d4 100644 --- a/.gitignore +++ b/.gitignore @@ -11,7 +11,6 @@ Unreal/CarlaUE4/Plugins/Streetmap Unreal/CarlaUE4/Plugins/HoudiniEngine build/ - /ExportedMaps /Import/* !/Import/README.md @@ -45,6 +44,7 @@ build/ .tags* .vs .vscode +.zed .cache __pycache__ _benchmarks_results @@ -55,4 +55,6 @@ core profiler.csv .env .venv +Unreal/CarlaUE4/.ignore devcontainer.json +compile_commands.json diff --git a/Util/BuildTools/BuildOSM2ODR.sh b/Util/BuildTools/BuildOSM2ODR.sh index 782991e781f..9c0b3e084df 100755 --- a/Util/BuildTools/BuildOSM2ODR.sh +++ b/Util/BuildTools/BuildOSM2ODR.sh @@ -95,6 +95,7 @@ if ${BUILD_OSM2ODR} ; then export CC="$UE4_ROOT/Engine/Extras/ThirdPartyNotUE/SDKs/HostLinux/Linux_x64/v17_clang-10.0.1-centos7/x86_64-unknown-linux-gnu/bin/clang" export CXX="$UE4_ROOT/Engine/Extras/ThirdPartyNotUE/SDKs/HostLinux/Linux_x64/v17_clang-10.0.1-centos7/x86_64-unknown-linux-gnu/bin/clang++" export PATH="$UE4_ROOT/Engine/Extras/ThirdPartyNotUE/SDKs/HostLinux/Linux_x64/v17_clang-10.0.1-centos7/x86_64-unknown-linux-gnu/bin:$PATH" + source $(dirname "$0")/Ubuntu24Compat.sh cmake ${OSM2ODR_SOURCE_FOLDER} \ -G "Eclipse CDT4 - Ninja" \ @@ -114,13 +115,14 @@ if ${BUILD_OSM2ODR} ; then LLVM_BASENAME=llvm-8.0 LLVM_INCLUDE="$UE4_ROOT/Engine/Source/ThirdParty/Linux/LibCxx/include/c++/v1" LLVM_LIBPATH="$UE4_ROOT/Engine/Source/ThirdParty/Linux/LibCxx/lib/Linux/x86_64-unknown-linux-gnu" + UNREAL_HOSTED_CFLAGS="--sysroot=$UE4_ROOT/Engine/Extras/ThirdPartyNotUE/SDKs/HostLinux/Linux_x64/v17_clang-10.0.1-centos7/x86_64-unknown-linux-gnu/ -idirafter /usr/include -idirafter /usr/include/x86_64-linux-gnu" echo $LLVM_INCLUDE echo $LLVM_LIBPATH cmake ${OSM2ODR_SOURCE_FOLDER} \ -G "Eclipse CDT4 - Ninja" \ - -DCMAKE_CXX_FLAGS="-fPIC -std=c++14 -stdlib=libc++ -I${LLVM_INCLUDE} -L${LLVM_LIBPATH}" \ + -DCMAKE_CXX_FLAGS="-fPIC -std=c++14 -stdlib=libc++ ${UNREAL_HOSTED_CFLAGS} -I${LLVM_INCLUDE} -L${LLVM_LIBPATH}" \ -DCMAKE_INSTALL_PREFIX=${LIBCARLA_INSTALL_SERVER_FOLDER} \ -DPROJ_INCLUDE_DIR=${CARLA_BUILD_FOLDER}/proj-install-server/include \ -DPROJ_LIBRARY=${CARLA_BUILD_FOLDER}/proj-install-server/lib/libproj.a \ @@ -139,7 +141,7 @@ if ${BUILD_OSM2ODR} ; then cmake ${OSM2ODR_SOURCE_FOLDER} \ -G "Eclipse CDT4 - Ninja" \ - -DCMAKE_CXX_FLAGS="-fPIC -std=c++14 -stdlib=libc++ -I${LLVM_INCLUDE} -L${LLVM_LIBPATH}" \ + -DCMAKE_CXX_FLAGS="-fPIC -std=c++14 -stdlib=libc++ ${UNREAL_HOSTED_CFLAGS} -I${LLVM_INCLUDE} -L${LLVM_LIBPATH}" \ -DCMAKE_INSTALL_PREFIX=${LIBCARLA_INSTALL_SERVER_FOLDER} \ -DPROJ_INCLUDE_DIR=${CARLA_BUILD_FOLDER}/proj-install-server/include \ -DPROJ_LIBRARY=${CARLA_BUILD_FOLDER}/proj-install-server/lib/libproj.a \ diff --git a/Util/BuildTools/BuildPythonAPI.sh b/Util/BuildTools/BuildPythonAPI.sh index f3b88c87cf5..a488ec4e1b3 100755 --- a/Util/BuildTools/BuildPythonAPI.sh +++ b/Util/BuildTools/BuildPythonAPI.sh @@ -61,6 +61,7 @@ export CXX="$UE4_ROOT/Engine/Extras/ThirdPartyNotUE/SDKs/HostLinux/Linux_x64/v17 export PATH="$UE4_ROOT/Engine/Extras/ThirdPartyNotUE/SDKs/HostLinux/Linux_x64/v17_clang-10.0.1-centos7/x86_64-unknown-linux-gnu/bin:$PATH" source $(dirname "$0")/Environment.sh +source $(dirname "$0")/Ubuntu24Compat.sh if ! { ${REMOVE_INTERMEDIATE} || ${BUILD_PYTHONAPI} || ${BUILD_PYTHONAPI_WHEEL} ; }; then fatal_error "Nothing selected to be done." @@ -98,18 +99,26 @@ fi if ${BUILD_PYTHONAPI} ; then # Add patchelf to the path. Auditwheel relies on patchelf to repair ELF files. export PATH="${LIBCARLA_INSTALL_CLIENT_FOLDER}/bin:${PATH}" - + for PY_VERSION in ${PY_VERSION_LIST[@]} ; do log "Building Python API wheel for Python ${PY_VERSION}." - + # Building the RSS variant adds files to SOURCES.txt we do not want included in a normal build - rm -Rf source/carla.egg-info - + rm -Rf source/carla.egg-info + /usr/bin/env python${PY_VERSION} -m build --wheel --outdir dist/.tmp . - if ${INSTALL_PYTHONAPI} ; then + if ${INSTALL_PYTHONAPI} && ! ${_SKIP_PIP_INSTALL:-false} ; then log "Installing Python API for Python ${PY_VERSION}." /usr/bin/env python${PY_VERSION} -m pip install --force-reinstall dist/.tmp/$(ls dist/.tmp | grep .whl) + elif ${_SKIP_PIP_INSTALL:-false} ; then + _WHEEL_FILE=$(ls dist/.tmp | grep .whl) + log "Python API wheel built: PythonAPI/carla/dist/${_WHEEL_FILE}" + log "System Python is externally managed (PEP 668) — install the wheel manually:" + log " pip install --break-system-packages PythonAPI/carla/dist/${_WHEEL_FILE}" + log " uv pip install PythonAPI/carla/dist/${_WHEEL_FILE}" + log " conda run -n pip install PythonAPI/carla/dist/${_WHEEL_FILE}" + log " pyenv exec pip install PythonAPI/carla/dist/${_WHEEL_FILE}" fi if [[ -z ${TARGET_WHEEL_PLATFORM} ]] ; then diff --git a/Util/BuildTools/Setup.sh b/Util/BuildTools/Setup.sh index 918ca8288e3..676b2e578db 100755 --- a/Util/BuildTools/Setup.sh +++ b/Util/BuildTools/Setup.sh @@ -64,9 +64,11 @@ IFS="," read -r -a PY_VERSION_LIST <<< "${PY_VERSION_LIST}" mkdir -p ${CARLA_BUILD_FOLDER} pushd ${CARLA_BUILD_FOLDER} >/dev/null +source $(dirname "$0")/Ubuntu24Compat.sh + LLVM_INCLUDE="$UE4_ROOT/Engine/Source/ThirdParty/Linux/LibCxx/include/c++/v1" LLVM_LIBPATH="$UE4_ROOT/Engine/Source/ThirdParty/Linux/LibCxx/lib/Linux/x86_64-unknown-linux-gnu" -UNREAL_HOSTED_CFLAGS="--sysroot=$UE4_ROOT/Engine/Extras/ThirdPartyNotUE/SDKs/HostLinux/Linux_x64/v17_clang-10.0.1-centos7/x86_64-unknown-linux-gnu/" +UNREAL_HOSTED_CFLAGS="--sysroot=$UE4_ROOT/Engine/Extras/ThirdPartyNotUE/SDKs/HostLinux/Linux_x64/v17_clang-10.0.1-centos7/x86_64-unknown-linux-gnu/ -idirafter /usr/include -idirafter /usr/include/x86_64-linux-gnu" # ============================================================================== # -- Generate CMake toolchains ------------------------------------------------- @@ -171,7 +173,7 @@ for PY_VERSION in ${PY_VERSION_LIST[@]} ; do pushd ${BOOST_BASENAME}-source >/dev/null - BOOST_TOOLSET="clang-10.0" + BOOST_TOOLSET="clang" BOOST_CFLAGS="-fPIC -std=c++14 -DBOOST_ERROR_CODE_HEADER_ONLY" py3="/usr/bin/env python${PY_VERSION}" @@ -180,7 +182,7 @@ for PY_VERSION in ${PY_VERSION_LIST[@]} ; do py3_lib=$(${py3} -c "from sysconfig import get_paths as gp; print(gp()['stdlib'])") pyv=`$py3 -c "import sys;x='{v[0]}.{v[1]}'.format(v=list(sys.version_info[:2]));sys.stdout.write(x)";` ./bootstrap.sh \ - --with-toolset=clang \ + --with-toolset=gcc \ --prefix=../boost-install \ --with-libraries=python,filesystem,system,program_options \ --with-python=${py3} --with-python-root=${py3_root} @@ -192,9 +194,16 @@ for PY_VERSION in ${PY_VERSION_LIST[@]} ; do echo "using python : ${pyv} : ${py3_root}/bin/python${PY_VERSION} : \ ${py3_include} : ${py3_lib} ;" > project-config.jam fi + if ${_NEED_LLD:-false}; then + echo "using clang : : \"${_CXX_REAL}\" : -fuse-ld=lld ;" >> project-config.jam + _B2_LINK_EXTRA="linkflags=-fuse-ld=lld" + else + echo "using clang : : \"${CXX}\" ;" >> project-config.jam + _B2_LINK_EXTRA="" + fi - ./b2 toolset="${BOOST_TOOLSET}" cxxflags="${BOOST_CFLAGS}" --prefix="../${BOOST_BASENAME}-install" -j ${CARLA_BUILD_CONCURRENCY} stage release - ./b2 toolset="${BOOST_TOOLSET}" cxxflags="${BOOST_CFLAGS}" --prefix="../${BOOST_BASENAME}-install" -j ${CARLA_BUILD_CONCURRENCY} install + ./b2 toolset="${BOOST_TOOLSET}" cxxflags="${BOOST_CFLAGS}" ${_B2_LINK_EXTRA} --prefix="../${BOOST_BASENAME}-install" -j ${CARLA_BUILD_CONCURRENCY} stage release + ./b2 toolset="${BOOST_TOOLSET}" cxxflags="${BOOST_CFLAGS}" ${_B2_LINK_EXTRA} --prefix="../${BOOST_BASENAME}-install" -j ${CARLA_BUILD_CONCURRENCY} install popd >/dev/null @@ -499,6 +508,7 @@ else echo "Elapsed Time Extracting xerces-c: $(($end-$start)) seconds" mv ${XERCESC_BASENAME} ${XERCESC_SRC_DIR} + mkdir -p ${XERCESC_INSTALL_DIR} mkdir -p ${XERCESC_SRC_DIR}/build @@ -522,7 +532,7 @@ else pushd ${XERCESC_SRC_DIR}/build >/dev/null cmake -G "Ninja" \ - -DCMAKE_CXX_FLAGS="-std=c++14 -stdlib=libc++ -fPIC -w -I${LLVM_INCLUDE} -L${LLVM_LIBPATH}" \ + -DCMAKE_CXX_FLAGS="-std=c++14 -stdlib=libc++ -fPIC -w ${UNREAL_HOSTED_CFLAGS} -I${LLVM_INCLUDE} -L${LLVM_LIBPATH}" \ -DCMAKE_INSTALL_PREFIX="../../${XERCESC_INSTALL_SERVER_DIR}" \ -DCMAKE_BUILD_TYPE=Release \ -DBUILD_SHARED_LIBS=OFF \ @@ -735,7 +745,7 @@ else pushd ${PROJ_SRC_DIR}/build >/dev/null cmake -G "Ninja" .. \ - -DCMAKE_CXX_FLAGS="-std=c++14 -fPIC" \ + -DCMAKE_CXX_FLAGS="-std=c++14 -fPIC -include cstdint" \ -DSQLITE3_INCLUDE_DIR=${SQLITE_INCLUDE_DIR} -DSQLITE3_LIBRARY=${SQLITE_LIB} \ -DEXE_SQLITE3=${SQLITE_EXE} \ -DENABLE_TIFF=OFF -DENABLE_CURL=OFF -DBUILD_SHARED_LIBS=OFF -DBUILD_PROJSYNC=OFF \ @@ -753,7 +763,7 @@ else pushd ${PROJ_SRC_DIR}/build >/dev/null cmake -G "Ninja" .. \ - -DCMAKE_CXX_FLAGS="-std=c++14 -fPIC -stdlib=libc++ -I${LLVM_INCLUDE} -Wl,-L${LLVM_LIBPATH}" \ + -DCMAKE_CXX_FLAGS="-std=c++14 -fPIC -include cstdint -stdlib=libc++ ${UNREAL_HOSTED_CFLAGS} -I${LLVM_INCLUDE} -Wl,-L${LLVM_LIBPATH}" \ -DSQLITE3_INCLUDE_DIR=${SQLITE_INCLUDE_DIR} -DSQLITE3_LIBRARY=${SQLITE_LIB} \ -DEXE_SQLITE3=${SQLITE_EXE} \ -DENABLE_TIFF=OFF -DENABLE_CURL=OFF -DBUILD_SHARED_LIBS=OFF -DBUILD_PROJSYNC=OFF \ diff --git a/Util/BuildTools/Ubuntu24Compat.sh b/Util/BuildTools/Ubuntu24Compat.sh new file mode 100644 index 00000000000..23988905787 --- /dev/null +++ b/Util/BuildTools/Ubuntu24Compat.sh @@ -0,0 +1,77 @@ +#!/bin/bash +# Compatibility shim for building CARLA with the UE4 bundled clang on modern +# Linux distributions. Source this file AFTER exporting CC and CXX. +# +# Two independent issues are detected and patched at runtime so older distros +# that don't have these problems are completely unaffected: +# +# 1. Bundled ld (2019) cannot read .relr.dyn sections present in glibc ≥ 2.36 +# (Ubuntu 22.04+, Fedora 36+, Arch …). Fix: wrap CC/CXX to use lld. +# +# 2. CMake ≥ 4.0 errors on cmake_minimum_required < 3.5 in third-party sources. +# Fix: wrap cmake to inject -DCMAKE_POLICY_VERSION_MINIMUM=3.5. +# +# Usage (after CC/CXX are exported): +# source "$(dirname "$0")/Ubuntu24Compat.sh" + +_WRAP_DIR="${CARLA_BUILD_FOLDER}" +mkdir -p "${_WRAP_DIR}" + +# --------------------------------------------------------------------------- +# 1. Bundled ld compatibility check +# --------------------------------------------------------------------------- +_NEED_LLD=false +_LD_TEST_SRC="$(mktemp /tmp/ue4_ld_test_XXXXXX.c)" +_LD_TEST_BIN="$(mktemp /tmp/ue4_ld_test_XXXXXX)" +echo 'int main(void){return 0;}' > "${_LD_TEST_SRC}" +if ! "${CC}" "${_LD_TEST_SRC}" -o "${_LD_TEST_BIN}" 2>/dev/null; then + _NEED_LLD=true +fi +rm -f "${_LD_TEST_SRC}" "${_LD_TEST_BIN}" + +if ${_NEED_LLD}; then + if ! which ld.lld >/dev/null 2>&1; then + echo "ERROR: The UE4 bundled linker cannot link on this system." + echo " Install lld and try again: sudo apt install lld" + exit 1 + fi + _CC_REAL="${CC}" + _CXX_REAL="${CXX}" + CC="${_WRAP_DIR}/clang.sh" + CXX="${_WRAP_DIR}/clang++.sh" + # -Wno-unused-command-line-argument silences the warning that -fuse-ld=lld + # produces when clang is invoked for compilation only (not linking). + printf '#!/bin/bash\nexec "%s" -fuse-ld=lld -Wno-unused-command-line-argument "$@"\n' \ + "${_CC_REAL}" > "${CC}" + printf '#!/bin/bash\nexec "%s" -fuse-ld=lld -Wno-unused-command-line-argument "$@"\n' \ + "${_CXX_REAL}" > "${CXX}" + chmod +x "${CC}" "${CXX}" + export CC CXX + export PATH="${_WRAP_DIR}:${PATH}" +fi + +# --------------------------------------------------------------------------- +# 2. System Python PEP 668 check (Ubuntu ≥ 24) +# --------------------------------------------------------------------------- +# Ubuntu 24+ ships Python as "externally managed" (PEP 668), so a bare +# `pip install` into the system environment is blocked. The wheel is still +# built and placed in PythonAPI/carla/dist/ for manual installation. +_OS_MAJOR=$(. /etc/os-release 2>/dev/null && printf '%s' "${VERSION_ID}" | cut -d. -f1) +if [ "${_OS_MAJOR:-0}" -ge 24 ] 2>/dev/null; then + export _SKIP_PIP_INSTALL=true +fi + +# --------------------------------------------------------------------------- +# 3. CMake 4.x compatibility check +# --------------------------------------------------------------------------- +_CMAKE_MAJOR=$(cmake --version 2>/dev/null | grep -oP '(?<=cmake version )\d+' | head -1) +if [ "${_CMAKE_MAJOR:-0}" -ge 4 ]; then + printf '#!/bin/bash\nexec /usr/bin/cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 "$@"\n' \ + > "${_WRAP_DIR}/cmake" + chmod +x "${_WRAP_DIR}/cmake" + # Only prepend to PATH if not already there (avoid duplicates on re-source) + case ":${PATH}:" in + *":${_WRAP_DIR}:"*) ;; + *) export PATH="${_WRAP_DIR}:${PATH}" ;; + esac +fi