diff --git a/.github/workflows/quic-organization-repolinter.yml b/.github/workflows/quic-organization-repolinter.yml new file mode 100644 index 000000000..54c1da1da --- /dev/null +++ b/.github/workflows/quic-organization-repolinter.yml @@ -0,0 +1,31 @@ +name: QuIC Organization Repolinter + +on: + push: + branches: [ "hexagon" ] + pull_request: + branches: [ "hexagon" ] + workflow_dispatch: + +jobs: + repolinter: + runs-on: ubuntu-latest + steps: + - name: Checkout Repo + uses: actions/checkout@v4 + - name: Verify repolinter config file is present + id: check_files + uses: andstor/file-existence-action@v3 + with: + files: "repolint.json" + - name: Run Repolinter with local repolint.json + if: steps.check_files.outputs.files_exists == 'true' + uses: todogroup/repolinter-action@v1 + with: + config_file: "repolint.json" + - name: Run Repolinter with default ruleset + if: steps.check_files.outputs.files_exists == 'false' + uses: todogroup/repolinter-action@v1 + with: + config_url: "https://raw.githubusercontent.com/quic/.github/main/repolint.json" + diff --git a/COPYRIGHT b/COPYRIGHT index e64723714..c1628e9ac 100644 --- a/COPYRIGHT +++ b/COPYRIGHT @@ -127,10 +127,13 @@ Copyright © 2017-2018 Arm Limited and labelled as such in comments in the individual source files. All have been licensed under extremely permissive terms. -The ARM memcpy code (src/string/arm/memcpy_el.S) is Copyright © 2008 +The ARM memcpy code (src/string/arm/memcpy.S) is Copyright © 2008 The Android Open Source Project and is licensed under a two-clause BSD license. It was taken from Bionic libc, used on Android. +The AArch64 memcpy and memset code (src/string/aarch64/*) are +Copyright © 1999-2019, Arm Limited. + The implementation of DES for crypt (src/crypt/crypt_des.c) is Copyright © 1994 David Burren. It is licensed under a BSD license. diff --git a/INSTALL b/INSTALL index 22477b6ba..c583691d7 100644 --- a/INSTALL +++ b/INSTALL @@ -86,7 +86,7 @@ and ABI combinations: * SuperH (SH) * Standard ELF ABI or FDPIC ABI (shared-text without MMU) - * Little-endian by default; big-engian variant also supported + * Little-endian by default; big-endian variant also supported * Full FPU ABI or soft-float ABI is supported, but the single-precision-only FPU ABI is not diff --git a/Makefile b/Makefile index bd8f5c389..e8cc44367 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,8 @@ includedir = $(prefix)/include libdir = $(prefix)/lib syslibdir = /lib -SRC_DIRS = $(addprefix $(srcdir)/,src/* crt ldso $(COMPAT_SRC_DIRS)) +MALLOC_DIR = mallocng +SRC_DIRS = $(addprefix $(srcdir)/,src/* src/malloc/$(MALLOC_DIR) crt ldso $(COMPAT_SRC_DIRS)) BASE_GLOBS = $(addsuffix /*.c,$(SRC_DIRS)) ARCH_GLOBS = $(addsuffix /$(ARCH)/*.[csS],$(SRC_DIRS)) BASE_SRCS = $(sort $(wildcard $(BASE_GLOBS))) diff --git a/VERSION b/VERSION index 26aaba0e8..23aa83906 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.2.0 +1.2.2 diff --git a/WHATSNEW b/WHATSNEW index ecf0cebab..e1d01982a 100644 --- a/WHATSNEW +++ b/WHATSNEW @@ -2200,3 +2200,90 @@ arch-specific bugs fixed: - arm dynamic linker chose wrong tls/atomic variants since 1.1.21 - some math library functions returned excess precision on i386 - unconfirmed regression in fchmodat AT_SYMLINK_NOFOLLOW on mips* + + + +1.2.1 release notes + +major changes: +- new malloc implementation (mallocng & overhauled bump allocator) + +new features: +- DNS queries via res_* now set AD flag, report zone signedness (DNSSEC) +- PTHREAD_NULL macro (POSIX-future) + +performance: +- optimized memcpy and memset for aarch64 +- optimized memcpy for arm now supports big endian +- optimized x86_64 remquol +- improved strerror without linear search + +bugs fixed: +- lock-skipping for processes that returned to single-threaded was wrong +- AF_UNSPEC dns lookups mishandled single failure in paired A+AAAA +- res_send and res_query returned wrong value on errors from nameserver +- corrupted sysvipc timestamps on 32-bit archs with old kernels +- incorrect parsing of timezone offsets after overly-long zone name +- clock_adjtime was broken on 32-bit archs (time64) +- pthread_kill as not async-signal-safe +- pthread_cancel was not async-cancel-safe +- large-ulp errors in various math functions in non-default rounding modes + +arch-specific bugs fixed: +- arm clock_gettime was broken on some hw due to bad time64 vdso +- m68k sqrtl lacked long double precision +- mips* syscall mechanism regressions on older kernels +- mips* had negated error codes for some syscalls (kernel bug) +- mips* SIGEMT was wrongly called SIGSTKFLT +- sh fesetround didn't work correctly on sh + + + +1.2.2 release notes + +major changes: +- child restrictions lifted after fork of multithreaded parent + +new features: +- _Fork function (POSIX-future) +- reallocarray function (extension from OpenBSD, now widespread) +- gettid function (kernel tid as supported concept) +- SIGEV_THREAD_ID sigevent API (Linux extension) +- tcgetwinsize and tcsetwinsize functions (POSIX-future) + +performance: +- faster software sqrt on archs without native sqrt instruction + +compatibility: +- realpath no longer depends on procfs availability & accuracy +- time zone parser now always prefers 64-bit tables if present +- crypt_blowfish now supports $2b$ prefix +- res_query now reports errors via h_errno +- set*id and setrlimit are now safe in vforked/cloned child +- setgroups now applies to all threads +- dlopen debugger notification is improved, should work with lldb +- setrlimit no longer needs __synccall broadcast on linux 2.6.36+ +- faccessat with AT_EACCESS no longer needs child process on linux 5.8+ + +bugs fixed: +- buffer overflow and infinite loop errors in wcsnrtombs (CVE-2020-28928) +- sem_close unmapped still-referenced semaphores +- fork of process with active aio could deadlock or crash paren +- pthread_cond_wait was broken with priority-inheritance mutex +- getgrouplist wrongly failed when nscd reported an empty list +- abort could leak modified SIGABRT disposition to fork or posix_spawn child +- regression with mallocng: malloc_usable_size(0) crashed +- readlink wrongly gave EINVAL on zero length dest buffer +- sqrtl was severely inaccurate (not correctly rounded) on ldquad archs +- assert failure wrongly flushed stdio (possible deadlock) +- MUSL_LOCPATH search was broken with multiple components +- missing newline in herror output +- possible deadlock in pthread_exit with pshared mutex or barrier usage +- pthread_mutexattr_getprotocol didn't read back protocol +- v4l2 ioctl translation for pre-time64 kernels didn't work + +arch-specific bugs fixed: +- x86_64 longjmp failed to handle 0 argument reliably +- i386 __set_thread_area fallback for pre-2.6 kernels didn't work +- missing O_LARGEFILE macro value on x86_64, x32, mips64 +- unpredictable s390x breakage from failure to preserve call-saved registers diff --git a/arch/aarch64/bits/hwcap.h b/arch/aarch64/bits/hwcap.h index a7484028e..7ab73f99b 100644 --- a/arch/aarch64/bits/hwcap.h +++ b/arch/aarch64/bits/hwcap.h @@ -38,3 +38,13 @@ #define HWCAP2_SVEBITPERM (1 << 4) #define HWCAP2_SVESHA3 (1 << 5) #define HWCAP2_SVESM4 (1 << 6) +#define HWCAP2_FLAGM2 (1 << 7) +#define HWCAP2_FRINT (1 << 8) +#define HWCAP2_SVEI8MM (1 << 9) +#define HWCAP2_SVEF32MM (1 << 10) +#define HWCAP2_SVEF64MM (1 << 11) +#define HWCAP2_SVEBF16 (1 << 12) +#define HWCAP2_I8MM (1 << 13) +#define HWCAP2_BF16 (1 << 14) +#define HWCAP2_DGH (1 << 15) +#define HWCAP2_RNG (1 << 16) diff --git a/arch/aarch64/bits/signal.h b/arch/aarch64/bits/signal.h index b71261f56..5098c7341 100644 --- a/arch/aarch64/bits/signal.h +++ b/arch/aarch64/bits/signal.h @@ -11,7 +11,7 @@ typedef unsigned long greg_t; typedef unsigned long gregset_t[34]; typedef struct { - long double vregs[32]; + __uint128_t vregs[32]; unsigned int fpsr; unsigned int fpcr; } fpregset_t; @@ -34,7 +34,7 @@ struct fpsimd_context { struct _aarch64_ctx head; unsigned int fpsr; unsigned int fpcr; - long double vregs[32]; + __uint128_t vregs[32]; }; struct esr_context { struct _aarch64_ctx head; diff --git a/arch/aarch64/bits/syscall.h.in b/arch/aarch64/bits/syscall.h.in index 93648afdf..f9457c184 100644 --- a/arch/aarch64/bits/syscall.h.in +++ b/arch/aarch64/bits/syscall.h.in @@ -289,4 +289,8 @@ #define __NR_fspick 433 #define __NR_pidfd_open 434 #define __NR_clone3 435 +#define __NR_close_range 436 +#define __NR_openat2 437 +#define __NR_pidfd_getfd 438 +#define __NR_faccessat2 439 diff --git a/arch/aarch64/bits/user.h b/arch/aarch64/bits/user.h index d12cdf7fe..8a1002aa6 100644 --- a/arch/aarch64/bits/user.h +++ b/arch/aarch64/bits/user.h @@ -6,7 +6,7 @@ struct user_regs_struct { }; struct user_fpsimd_struct { - long double vregs[32]; + __uint128_t vregs[32]; unsigned int fpsr; unsigned int fpcr; }; diff --git a/arch/aarch64/pthread_arch.h b/arch/aarch64/pthread_arch.h index e64b126d2..3909616c3 100644 --- a/arch/aarch64/pthread_arch.h +++ b/arch/aarch64/pthread_arch.h @@ -1,12 +1,11 @@ -static inline struct pthread *__pthread_self() +static inline uintptr_t __get_tp() { - char *self; - __asm__ ("mrs %0,tpidr_el0" : "=r"(self)); - return (void*)(self - sizeof(struct pthread)); + uintptr_t tp; + __asm__ ("mrs %0,tpidr_el0" : "=r"(tp)); + return tp; } #define TLS_ABOVE_TP #define GAP_ABOVE_TP 16 -#define TP_ADJ(p) ((char *)(p) + sizeof(struct pthread)) #define MC_PC pc diff --git a/arch/arm/bits/syscall.h.in b/arch/arm/bits/syscall.h.in index 11d677635..7e2fc2669 100644 --- a/arch/arm/bits/syscall.h.in +++ b/arch/arm/bits/syscall.h.in @@ -389,6 +389,10 @@ #define __NR_fspick 433 #define __NR_pidfd_open 434 #define __NR_clone3 435 +#define __NR_close_range 436 +#define __NR_openat2 437 +#define __NR_pidfd_getfd 438 +#define __NR_faccessat2 439 #define __ARM_NR_breakpoint 0x0f0001 #define __ARM_NR_cacheflush 0x0f0002 diff --git a/arch/arm/pthread_arch.h b/arch/arm/pthread_arch.h index e689ea212..157e2eae6 100644 --- a/arch/arm/pthread_arch.h +++ b/arch/arm/pthread_arch.h @@ -1,11 +1,11 @@ #if ((__ARM_ARCH_6K__ || __ARM_ARCH_6KZ__ || __ARM_ARCH_6ZK__) && !__thumb__) \ || __ARM_ARCH_7A__ || __ARM_ARCH_7R__ || __ARM_ARCH >= 7 -static inline pthread_t __pthread_self() +static inline uintptr_t __get_tp() { - char *p; - __asm__ ( "mrc p15,0,%0,c13,c0,3" : "=r"(p) ); - return (void *)(p-sizeof(struct pthread)); + uintptr_t tp; + __asm__ ( "mrc p15,0,%0,c13,c0,3" : "=r"(tp) ); + return tp; } #else @@ -16,18 +16,17 @@ static inline pthread_t __pthread_self() #define BLX "blx" #endif -static inline pthread_t __pthread_self() +static inline uintptr_t __get_tp() { extern hidden uintptr_t __a_gettp_ptr; - register uintptr_t p __asm__("r0"); - __asm__ ( BLX " %1" : "=r"(p) : "r"(__a_gettp_ptr) : "cc", "lr" ); - return (void *)(p-sizeof(struct pthread)); + register uintptr_t tp __asm__("r0"); + __asm__ ( BLX " %1" : "=r"(tp) : "r"(__a_gettp_ptr) : "cc", "lr" ); + return tp; } #endif #define TLS_ABOVE_TP #define GAP_ABOVE_TP 8 -#define TP_ADJ(p) ((char *)(p) + sizeof(struct pthread)) #define MC_PC arm_pc diff --git a/arch/arm/syscall_arch.h b/arch/arm/syscall_arch.h index 4b08762d7..a877b2cff 100644 --- a/arch/arm/syscall_arch.h +++ b/arch/arm/syscall_arch.h @@ -98,12 +98,6 @@ static inline long __syscall6(long n, long a, long b, long c, long d, long e, lo __asm_syscall(R7_OPERAND, "0"(r0), "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5)); } -#define VDSO_USEFUL -#define VDSO_CGT32_SYM "__vdso_clock_gettime" -#define VDSO_CGT32_VER "LINUX_2.6" -#define VDSO_CGT_SYM "__vdso_clock_gettime64" -#define VDSO_CGT_VER "LINUX_2.6" - #define SYSCALL_FADVISE_6_ARG #define SYSCALL_IPC_BROKEN_MODE diff --git a/arch/generic/bits/fcntl.h b/arch/generic/bits/fcntl.h index ae233cc00..730a98cfe 100644 --- a/arch/generic/bits/fcntl.h +++ b/arch/generic/bits/fcntl.h @@ -30,9 +30,15 @@ #define F_SETSIG 10 #define F_GETSIG 11 +#if __LONG_MAX == 0x7fffffffL #define F_GETLK 12 #define F_SETLK 13 #define F_SETLKW 14 +#else +#define F_GETLK 5 +#define F_SETLK 6 +#define F_SETLKW 7 +#endif #define F_SETOWN_EX 15 #define F_GETOWN_EX 16 diff --git a/arch/i386/bits/syscall.h.in b/arch/i386/bits/syscall.h.in index 1ae4e48a8..abdb210d3 100644 --- a/arch/i386/bits/syscall.h.in +++ b/arch/i386/bits/syscall.h.in @@ -426,4 +426,8 @@ #define __NR_fspick 433 #define __NR_pidfd_open 434 #define __NR_clone3 435 +#define __NR_close_range 436 +#define __NR_openat2 437 +#define __NR_pidfd_getfd 438 +#define __NR_faccessat2 439 diff --git a/arch/i386/pthread_arch.h b/arch/i386/pthread_arch.h index 6f600b9e0..a639c382a 100644 --- a/arch/i386/pthread_arch.h +++ b/arch/i386/pthread_arch.h @@ -1,10 +1,8 @@ -static inline struct pthread *__pthread_self() +static inline uintptr_t __get_tp() { - struct pthread *self; - __asm__ ("movl %%gs:0,%0" : "=r" (self) ); - return self; + uintptr_t tp; + __asm__ ("movl %%gs:0,%0" : "=r" (tp) ); + return tp; } -#define TP_ADJ(p) (p) - #define MC_PC gregs[REG_EIP] diff --git a/arch/i386/syscall_arch.h b/arch/i386/syscall_arch.h index 69642e578..f92b7aa9f 100644 --- a/arch/i386/syscall_arch.h +++ b/arch/i386/syscall_arch.h @@ -87,5 +87,3 @@ static inline long __syscall6(long n, long a1, long a2, long a3, long a4, long a #define VDSO_CGT32_VER "LINUX_2.6" #define VDSO_CGT_SYM "__vdso_clock_gettime64" #define VDSO_CGT_VER "LINUX_2.6" - -#define SYSCALL_USE_SOCKETCALL diff --git a/arch/m68k/bits/syscall.h.in b/arch/m68k/bits/syscall.h.in index ddfa72e4d..e10969a26 100644 --- a/arch/m68k/bits/syscall.h.in +++ b/arch/m68k/bits/syscall.h.in @@ -405,3 +405,8 @@ #define __NR_fsmount 432 #define __NR_fspick 433 #define __NR_pidfd_open 434 +#define __NR_clone3 435 +#define __NR_close_range 436 +#define __NR_openat2 437 +#define __NR_pidfd_getfd 438 +#define __NR_faccessat2 439 diff --git a/arch/m68k/pthread_arch.h b/arch/m68k/pthread_arch.h index 02d5b8a08..5bea4e1ad 100644 --- a/arch/m68k/pthread_arch.h +++ b/arch/m68k/pthread_arch.h @@ -1,13 +1,12 @@ -static inline struct pthread *__pthread_self() +static inline uintptr_t __get_tp() { - uintptr_t tp = __syscall(SYS_get_thread_area); - return (pthread_t)(tp - 0x7000 - sizeof(struct pthread)); + return __syscall(SYS_get_thread_area); } #define TLS_ABOVE_TP #define GAP_ABOVE_TP 0 -#define TP_ADJ(p) ((char *)(p) + sizeof(struct pthread) + 0x7000) +#define TP_OFFSET 0x7000 #define DTP_OFFSET 0x8000 #define MC_PC gregs[R_PC] diff --git a/arch/m68k/syscall_arch.h b/arch/m68k/syscall_arch.h index af79c3069..6a9d0ae8e 100644 --- a/arch/m68k/syscall_arch.h +++ b/arch/m68k/syscall_arch.h @@ -87,5 +87,4 @@ static inline long __syscall6(long n, long a, long b, long c, long d, long e, lo return d0; } -#define SYSCALL_USE_SOCKETCALL #define SYSCALL_IPC_BROKEN_MODE diff --git a/arch/microblaze/bits/syscall.h.in b/arch/microblaze/bits/syscall.h.in index 963386a83..9d4690473 100644 --- a/arch/microblaze/bits/syscall.h.in +++ b/arch/microblaze/bits/syscall.h.in @@ -427,4 +427,8 @@ #define __NR_fspick 433 #define __NR_pidfd_open 434 #define __NR_clone3 435 +#define __NR_close_range 436 +#define __NR_openat2 437 +#define __NR_pidfd_getfd 438 +#define __NR_faccessat2 439 diff --git a/arch/microblaze/pthread_arch.h b/arch/microblaze/pthread_arch.h index f6ba8de98..ff26624ef 100644 --- a/arch/microblaze/pthread_arch.h +++ b/arch/microblaze/pthread_arch.h @@ -1,10 +1,8 @@ -static inline struct pthread *__pthread_self() +static inline uintptr_t __get_tp() { - struct pthread *self; - __asm__ ("ori %0, r21, 0" : "=r" (self) ); - return self; + uintptr_t tp; + __asm__ ("ori %0, r21, 0" : "=r" (tp) ); + return tp; } -#define TP_ADJ(p) (p) - #define MC_PC regs.pc diff --git a/arch/microblaze/syscall_arch.h b/arch/microblaze/syscall_arch.h index 169013f80..61d8248e8 100644 --- a/arch/microblaze/syscall_arch.h +++ b/arch/microblaze/syscall_arch.h @@ -95,3 +95,5 @@ static inline long __syscall6(long n, long a, long b, long c, long d, long e, lo } #define SYSCALL_IPC_BROKEN_MODE + +#undef SYS_socketcall diff --git a/arch/mips/bits/signal.h b/arch/mips/bits/signal.h index e1d97ac78..1b69e7625 100644 --- a/arch/mips/bits/signal.h +++ b/arch/mips/bits/signal.h @@ -93,7 +93,7 @@ typedef struct __ucontext { #define SIGTRAP 5 #define SIGABRT 6 #define SIGIOT SIGABRT -#define SIGSTKFLT 7 +#define SIGEMT 7 #define SIGFPE 8 #define SIGKILL 9 #define SIGBUS 10 diff --git a/arch/mips/bits/syscall.h.in b/arch/mips/bits/syscall.h.in index 86251bf31..2bb03f067 100644 --- a/arch/mips/bits/syscall.h.in +++ b/arch/mips/bits/syscall.h.in @@ -408,4 +408,8 @@ #define __NR_fspick 4433 #define __NR_pidfd_open 4434 #define __NR_clone3 4435 +#define __NR_close_range 4436 +#define __NR_openat2 4437 +#define __NR_pidfd_getfd 4438 +#define __NR_faccessat2 4439 diff --git a/arch/mips/pthread_arch.h b/arch/mips/pthread_arch.h index 1e7839ea6..c45347ab9 100644 --- a/arch/mips/pthread_arch.h +++ b/arch/mips/pthread_arch.h @@ -1,19 +1,19 @@ -static inline struct pthread *__pthread_self() +static inline uintptr_t __get_tp() { #if __mips_isa_rev < 2 - register char *tp __asm__("$3"); + register uintptr_t tp __asm__("$3"); __asm__ (".word 0x7c03e83b" : "=r" (tp) ); #else - char *tp; + uintptr_t tp; __asm__ ("rdhwr %0, $29" : "=r" (tp) ); #endif - return (pthread_t)(tp - 0x7000 - sizeof(struct pthread)); + return tp; } #define TLS_ABOVE_TP #define GAP_ABOVE_TP 0 -#define TP_ADJ(p) ((char *)(p) + sizeof(struct pthread) + 0x7000) +#define TP_OFFSET 0x7000 #define DTP_OFFSET 0x8000 #define MC_PC pc diff --git a/arch/mips/syscall_arch.h b/arch/mips/syscall_arch.h index 380a94b36..5b7c38de2 100644 --- a/arch/mips/syscall_arch.h +++ b/arch/mips/syscall_arch.h @@ -149,3 +149,5 @@ static inline long __syscall7(long n, long a, long b, long c, long d, long e, lo #define SO_SNDTIMEO_OLD 0x1005 #define SO_RCVTIMEO_OLD 0x1006 + +#undef SYS_socketcall diff --git a/arch/mips64/bits/fcntl.h b/arch/mips64/bits/fcntl.h index 3bcec15e0..5da1eef80 100644 --- a/arch/mips64/bits/fcntl.h +++ b/arch/mips64/bits/fcntl.h @@ -13,7 +13,7 @@ #define O_ASYNC 010000 #define O_DIRECT 0100000 -#define O_LARGEFILE 0 +#define O_LARGEFILE 020000 #define O_NOATIME 01000000 #define O_PATH 010000000 #define O_TMPFILE 020200000 diff --git a/arch/mips64/bits/signal.h b/arch/mips64/bits/signal.h index c31ad07ec..4f91c9fc8 100644 --- a/arch/mips64/bits/signal.h +++ b/arch/mips64/bits/signal.h @@ -112,7 +112,7 @@ typedef struct __ucontext { #define SIGTRAP 5 #define SIGABRT 6 #define SIGIOT SIGABRT -#define SIGSTKFLT 7 +#define SIGEMT 7 #define SIGFPE 8 #define SIGKILL 9 #define SIGBUS 10 diff --git a/arch/mips64/bits/syscall.h.in b/arch/mips64/bits/syscall.h.in index 9b406e9a4..045e8238a 100644 --- a/arch/mips64/bits/syscall.h.in +++ b/arch/mips64/bits/syscall.h.in @@ -338,4 +338,8 @@ #define __NR_fspick 5433 #define __NR_pidfd_open 5434 #define __NR_clone3 5435 +#define __NR_close_range 5436 +#define __NR_openat2 5437 +#define __NR_pidfd_getfd 5438 +#define __NR_faccessat2 5439 diff --git a/arch/mips64/pthread_arch.h b/arch/mips64/pthread_arch.h index 1e7839ea6..c45347ab9 100644 --- a/arch/mips64/pthread_arch.h +++ b/arch/mips64/pthread_arch.h @@ -1,19 +1,19 @@ -static inline struct pthread *__pthread_self() +static inline uintptr_t __get_tp() { #if __mips_isa_rev < 2 - register char *tp __asm__("$3"); + register uintptr_t tp __asm__("$3"); __asm__ (".word 0x7c03e83b" : "=r" (tp) ); #else - char *tp; + uintptr_t tp; __asm__ ("rdhwr %0, $29" : "=r" (tp) ); #endif - return (pthread_t)(tp - 0x7000 - sizeof(struct pthread)); + return tp; } #define TLS_ABOVE_TP #define GAP_ABOVE_TP 0 -#define TP_ADJ(p) ((char *)(p) + sizeof(struct pthread) + 0x7000) +#define TP_OFFSET 0x7000 #define DTP_OFFSET 0x8000 #define MC_PC pc diff --git a/arch/mipsn32/bits/signal.h b/arch/mipsn32/bits/signal.h index c31ad07ec..4f91c9fc8 100644 --- a/arch/mipsn32/bits/signal.h +++ b/arch/mipsn32/bits/signal.h @@ -112,7 +112,7 @@ typedef struct __ucontext { #define SIGTRAP 5 #define SIGABRT 6 #define SIGIOT SIGABRT -#define SIGSTKFLT 7 +#define SIGEMT 7 #define SIGFPE 8 #define SIGKILL 9 #define SIGBUS 10 diff --git a/arch/mipsn32/bits/syscall.h.in b/arch/mipsn32/bits/syscall.h.in index 2ad48d10d..5b3225585 100644 --- a/arch/mipsn32/bits/syscall.h.in +++ b/arch/mipsn32/bits/syscall.h.in @@ -362,4 +362,8 @@ #define __NR_fspick 6433 #define __NR_pidfd_open 6434 #define __NR_clone3 6435 +#define __NR_close_range 6436 +#define __NR_openat2 6437 +#define __NR_pidfd_getfd 6438 +#define __NR_faccessat2 6439 diff --git a/arch/mipsn32/pthread_arch.h b/arch/mipsn32/pthread_arch.h index 1e7839ea6..c45347ab9 100644 --- a/arch/mipsn32/pthread_arch.h +++ b/arch/mipsn32/pthread_arch.h @@ -1,19 +1,19 @@ -static inline struct pthread *__pthread_self() +static inline uintptr_t __get_tp() { #if __mips_isa_rev < 2 - register char *tp __asm__("$3"); + register uintptr_t tp __asm__("$3"); __asm__ (".word 0x7c03e83b" : "=r" (tp) ); #else - char *tp; + uintptr_t tp; __asm__ ("rdhwr %0, $29" : "=r" (tp) ); #endif - return (pthread_t)(tp - 0x7000 - sizeof(struct pthread)); + return tp; } #define TLS_ABOVE_TP #define GAP_ABOVE_TP 0 -#define TP_ADJ(p) ((char *)(p) + sizeof(struct pthread) + 0x7000) +#define TP_OFFSET 0x7000 #define DTP_OFFSET 0x8000 #define MC_PC pc diff --git a/arch/or1k/bits/syscall.h.in b/arch/or1k/bits/syscall.h.in index e9c925e41..b36038915 100644 --- a/arch/or1k/bits/syscall.h.in +++ b/arch/or1k/bits/syscall.h.in @@ -311,4 +311,8 @@ #define __NR_fspick 433 #define __NR_pidfd_open 434 #define __NR_clone3 435 +#define __NR_close_range 436 +#define __NR_openat2 437 +#define __NR_pidfd_getfd 438 +#define __NR_faccessat2 439 diff --git a/arch/or1k/pthread_arch.h b/arch/or1k/pthread_arch.h index 1b806f891..f75ea7e4e 100644 --- a/arch/or1k/pthread_arch.h +++ b/arch/or1k/pthread_arch.h @@ -1,18 +1,16 @@ -/* or1k use variant I, but with the twist that tp points to the end of TCB */ -static inline struct pthread *__pthread_self() +static inline uintptr_t __get_tp() { #ifdef __clang__ - char *tp; + uintptr_t tp; __asm__ ("l.ori %0, r10, 0" : "=r" (tp) ); #else - register char *tp __asm__("r10"); + register uintptr_t tp __asm__("r10"); __asm__ ("" : "=r" (tp) ); #endif - return (struct pthread *) (tp - sizeof(struct pthread)); + return tp; } #define TLS_ABOVE_TP #define GAP_ABOVE_TP 0 -#define TP_ADJ(p) ((char *)(p) + sizeof(struct pthread)) #define MC_PC regs.pc diff --git a/arch/powerpc/bits/syscall.h.in b/arch/powerpc/bits/syscall.h.in index 8d4f79b52..5c6fae3e5 100644 --- a/arch/powerpc/bits/syscall.h.in +++ b/arch/powerpc/bits/syscall.h.in @@ -415,4 +415,8 @@ #define __NR_fspick 433 #define __NR_pidfd_open 434 #define __NR_clone3 435 +#define __NR_close_range 436 +#define __NR_openat2 437 +#define __NR_pidfd_getfd 438 +#define __NR_faccessat2 439 diff --git a/arch/powerpc/pthread_arch.h b/arch/powerpc/pthread_arch.h index ae0f28d6d..42e88b07e 100644 --- a/arch/powerpc/pthread_arch.h +++ b/arch/powerpc/pthread_arch.h @@ -1,18 +1,16 @@ -static inline struct pthread *__pthread_self() +static inline uintptr_t __get_tp() { - register char *tp __asm__("r2"); + register uintptr_t tp __asm__("r2"); __asm__ ("" : "=r" (tp) ); - return (pthread_t)(tp - 0x7000 - sizeof(struct pthread)); + return tp; } #define TLS_ABOVE_TP #define GAP_ABOVE_TP 0 -#define TP_ADJ(p) ((char *)(p) + sizeof(struct pthread) + 0x7000) +#define TP_OFFSET 0x7000 #define DTP_OFFSET 0x8000 // the kernel calls the ip "nip", it's the first saved value after the 32 // GPRs. #define MC_PC gregs[32] - -#define CANARY canary_at_end diff --git a/arch/powerpc64/bits/syscall.h.in b/arch/powerpc64/bits/syscall.h.in index b935864c4..edf73d3d6 100644 --- a/arch/powerpc64/bits/syscall.h.in +++ b/arch/powerpc64/bits/syscall.h.in @@ -387,4 +387,8 @@ #define __NR_fspick 433 #define __NR_pidfd_open 434 #define __NR_clone3 435 +#define __NR_close_range 436 +#define __NR_openat2 437 +#define __NR_pidfd_getfd 438 +#define __NR_faccessat2 439 diff --git a/arch/powerpc64/pthread_arch.h b/arch/powerpc64/pthread_arch.h index 79c3ecd8a..1b7b90797 100644 --- a/arch/powerpc64/pthread_arch.h +++ b/arch/powerpc64/pthread_arch.h @@ -1,18 +1,16 @@ -static inline struct pthread *__pthread_self() +static inline uintptr_t __get_tp() { - register char *tp __asm__("r13"); + register uintptr_t tp __asm__("r13"); __asm__ ("" : "=r" (tp) ); - return (pthread_t)(tp - 0x7000 - sizeof(struct pthread)); + return tp; } #define TLS_ABOVE_TP #define GAP_ABOVE_TP 0 -#define TP_ADJ(p) ((char *)(p) + sizeof(struct pthread) + 0x7000) +#define TP_OFFSET 0x7000 #define DTP_OFFSET 0x8000 // the kernel calls the ip "nip", it's the first saved value after the 32 // GPRs. #define MC_PC gp_regs[32] - -#define CANARY canary_at_end diff --git a/arch/riscv64/bits/fcntl.h b/arch/riscv64/bits/fcntl.h deleted file mode 100644 index ecb4d18fd..000000000 --- a/arch/riscv64/bits/fcntl.h +++ /dev/null @@ -1,38 +0,0 @@ -#define O_CREAT 0100 -#define O_EXCL 0200 -#define O_NOCTTY 0400 -#define O_TRUNC 01000 -#define O_APPEND 02000 -#define O_NONBLOCK 04000 -#define O_DSYNC 010000 -#define O_SYNC 04010000 -#define O_RSYNC 04010000 -#define O_DIRECTORY 0200000 -#define O_NOFOLLOW 0400000 -#define O_CLOEXEC 02000000 - -#define O_ASYNC 020000 -#define O_DIRECT 040000 -#define O_LARGEFILE 0100000 -#define O_NOATIME 01000000 -#define O_PATH 010000000 -#define O_TMPFILE 020200000 -#define O_NDELAY O_NONBLOCK - -#define F_DUPFD 0 -#define F_GETFD 1 -#define F_SETFD 2 -#define F_GETFL 3 -#define F_SETFL 4 -#define F_GETLK 5 -#define F_SETLK 6 -#define F_SETLKW 7 -#define F_SETOWN 8 -#define F_GETOWN 9 -#define F_SETSIG 10 -#define F_GETSIG 11 - -#define F_SETOWN_EX 15 -#define F_GETOWN_EX 16 - -#define F_GETOWNER_UIDS 17 diff --git a/arch/riscv64/bits/signal.h b/arch/riscv64/bits/signal.h index b006334f7..287367db7 100644 --- a/arch/riscv64/bits/signal.h +++ b/arch/riscv64/bits/signal.h @@ -60,10 +60,10 @@ struct sigaltstack { size_t ss_size; }; -typedef struct ucontext_t +typedef struct __ucontext { unsigned long uc_flags; - struct ucontext_t *uc_link; + struct __ucontext *uc_link; stack_t uc_stack; sigset_t uc_sigmask; mcontext_t uc_mcontext; diff --git a/arch/riscv64/bits/syscall.h.in b/arch/riscv64/bits/syscall.h.in index 0043eeba3..5def016b1 100644 --- a/arch/riscv64/bits/syscall.h.in +++ b/arch/riscv64/bits/syscall.h.in @@ -289,6 +289,10 @@ #define __NR_fspick 433 #define __NR_pidfd_open 434 #define __NR_clone3 435 +#define __NR_close_range 436 +#define __NR_openat2 437 +#define __NR_pidfd_getfd 438 +#define __NR_faccessat2 439 #define __NR_sysriscv __NR_arch_specific_syscall #define __NR_riscv_flush_icache (__NR_sysriscv + 15) diff --git a/arch/riscv64/pthread_arch.h b/arch/riscv64/pthread_arch.h index db414b170..a20d7fba0 100644 --- a/arch/riscv64/pthread_arch.h +++ b/arch/riscv64/pthread_arch.h @@ -1,13 +1,12 @@ -static inline struct pthread *__pthread_self() +static inline uintptr_t __get_tp() { - char *tp; + uintptr_t tp; __asm__ __volatile__("mv %0, tp" : "=r"(tp)); - return (void *)(tp - sizeof(struct pthread)); + return tp; } #define TLS_ABOVE_TP #define GAP_ABOVE_TP 0 -#define TP_ADJ(p) ((char *)p + sizeof(struct pthread)) #define DTP_OFFSET 0x800 diff --git a/arch/s390x/bits/alltypes.h.in b/arch/s390x/bits/alltypes.h.in index 15d18c8f4..6c0eb7f4b 100644 --- a/arch/s390x/bits/alltypes.h.in +++ b/arch/s390x/bits/alltypes.h.in @@ -9,7 +9,11 @@ TYPEDEF int wchar_t; #endif +#if defined(__FLT_EVAL_METHOD__) && __FLT_EVAL_METHOD__ == 1 TYPEDEF double float_t; +#else +TYPEDEF float float_t; +#endif TYPEDEF double double_t; TYPEDEF struct { long long __ll; long double __ld; } max_align_t; diff --git a/arch/s390x/bits/float.h b/arch/s390x/bits/float.h index 90b73beed..e188cb619 100644 --- a/arch/s390x/bits/float.h +++ b/arch/s390x/bits/float.h @@ -1,4 +1,8 @@ -#define FLT_EVAL_METHOD 1 +#ifdef __FLT_EVAL_METHOD__ +#define FLT_EVAL_METHOD __FLT_EVAL_METHOD__ +#else +#define FLT_EVAL_METHOD 0 +#endif #define LDBL_TRUE_MIN 6.47517511943802511092443895822764655e-4966L #define LDBL_MIN 3.36210314311209350626267781732175260e-4932L diff --git a/arch/s390x/bits/syscall.h.in b/arch/s390x/bits/syscall.h.in index e89f37829..fb2e60e30 100644 --- a/arch/s390x/bits/syscall.h.in +++ b/arch/s390x/bits/syscall.h.in @@ -352,4 +352,8 @@ #define __NR_fspick 433 #define __NR_pidfd_open 434 #define __NR_clone3 435 +#define __NR_close_range 436 +#define __NR_openat2 437 +#define __NR_pidfd_getfd 438 +#define __NR_faccessat2 439 diff --git a/arch/s390x/pthread_arch.h b/arch/s390x/pthread_arch.h index e2251f1fa..e54fec3fe 100644 --- a/arch/s390x/pthread_arch.h +++ b/arch/s390x/pthread_arch.h @@ -1,14 +1,12 @@ -static inline struct pthread *__pthread_self() +static inline uintptr_t __get_tp() { - struct pthread *self; + uintptr_t tp; __asm__ ( "ear %0, %%a0\n" "sllg %0, %0, 32\n" "ear %0, %%a1\n" - : "=r"(self)); - return self; + : "=r"(tp)); + return tp; } -#define TP_ADJ(p) (p) - #define MC_PC psw.addr diff --git a/arch/s390x/syscall_arch.h b/arch/s390x/syscall_arch.h index afb99852e..83cc9a27c 100644 --- a/arch/s390x/syscall_arch.h +++ b/arch/s390x/syscall_arch.h @@ -72,5 +72,3 @@ static inline long __syscall6(long n, long a, long b, long c, long d, long e, lo register long r7 __asm__("r7") = f; __asm_syscall("+r"(r2), "r"(r1), "r"(r3), "r"(r4), "r"(r5), "r"(r6), "r"(r7)); } - -#define SYSCALL_USE_SOCKETCALL diff --git a/arch/sh/bits/signal.h b/arch/sh/bits/signal.h index 160311fae..d0b148282 100644 --- a/arch/sh/bits/signal.h +++ b/arch/sh/bits/signal.h @@ -9,7 +9,16 @@ #if defined(_GNU_SOURCE) || defined(_BSD_SOURCE) typedef int greg_t, gregset_t[16]; typedef int freg_t, fpregset_t[16]; -typedef struct sigcontext { +typedef struct { + unsigned long oldmask; + unsigned long gregs[16]; + unsigned long pc, pr, sr; + unsigned long gbr, mach, macl; + unsigned long fpregs[16]; + unsigned long xfpregs[16]; + unsigned int fpscr, fpul, ownedfp; +} mcontext_t; +struct sigcontext { unsigned long oldmask; unsigned long sc_regs[16]; unsigned long sc_pc, sc_pr, sc_sr; @@ -17,7 +26,7 @@ typedef struct sigcontext { unsigned long sc_fpregs[16]; unsigned long sc_xfpregs[16]; unsigned int sc_fpscr, sc_fpul, sc_ownedfp; -} mcontext_t; +}; #else typedef struct { unsigned long __regs[58]; diff --git a/arch/sh/bits/syscall.h.in b/arch/sh/bits/syscall.h.in index 0102ddaf8..158afc091 100644 --- a/arch/sh/bits/syscall.h.in +++ b/arch/sh/bits/syscall.h.in @@ -398,4 +398,9 @@ #define __NR_fsmount 432 #define __NR_fspick 433 #define __NR_pidfd_open 434 +#define __NR_clone3 435 +#define __NR_close_range 436 +#define __NR_openat2 437 +#define __NR_pidfd_getfd 438 +#define __NR_faccessat2 439 diff --git a/arch/sh/pthread_arch.h b/arch/sh/pthread_arch.h index 3ee9c1a93..199c2d55f 100644 --- a/arch/sh/pthread_arch.h +++ b/arch/sh/pthread_arch.h @@ -1,17 +1,16 @@ -static inline struct pthread *__pthread_self() +static inline uintptr_t __get_tp() { - char *self; - __asm__ ("stc gbr,%0" : "=r" (self) ); - return (struct pthread *) (self - sizeof(struct pthread)); + uintptr_t tp; + __asm__ ("stc gbr,%0" : "=r" (tp) ); + return tp; } #define TLS_ABOVE_TP #define GAP_ABOVE_TP 8 -#define TP_ADJ(p) ((char *)(p) + sizeof(struct pthread)) -#define MC_PC sc_pc +#define MC_PC pc #ifdef __FDPIC__ -#define MC_GOT sc_regs[12] +#define MC_GOT gregs[12] #define CANCEL_GOT (*(uintptr_t *)((char *)__syscall_cp_asm+sizeof(uintptr_t))) #endif diff --git a/arch/x32/bits/fcntl.h b/arch/x32/bits/fcntl.h index 1b88ad391..08627f812 100644 --- a/arch/x32/bits/fcntl.h +++ b/arch/x32/bits/fcntl.h @@ -13,7 +13,7 @@ #define O_ASYNC 020000 #define O_DIRECT 040000 -#define O_LARGEFILE 0 +#define O_LARGEFILE 0100000 #define O_NOATIME 01000000 #define O_PATH 010000000 #define O_TMPFILE 020200000 diff --git a/arch/x32/bits/syscall.h.in b/arch/x32/bits/syscall.h.in index f47bdee5b..cfd9856f1 100644 --- a/arch/x32/bits/syscall.h.in +++ b/arch/x32/bits/syscall.h.in @@ -298,6 +298,10 @@ #define __NR_fspick (0x40000000 + 433) #define __NR_pidfd_open (0x40000000 + 434) #define __NR_clone3 (0x40000000 + 435) +#define __NR_close_range (0x40000000 + 436) +#define __NR_openat2 (0x40000000 + 437) +#define __NR_pidfd_getfd (0x40000000 + 438) +#define __NR_faccessat2 (0x40000000 + 439) #define __NR_rt_sigaction (0x40000000 + 512) diff --git a/arch/x32/pthread_arch.h b/arch/x32/pthread_arch.h index f640a1a15..c1e7716d5 100644 --- a/arch/x32/pthread_arch.h +++ b/arch/x32/pthread_arch.h @@ -1,14 +1,12 @@ -static inline struct pthread *__pthread_self() +static inline uintptr_t __get_tp() { - struct pthread *self; - __asm__ ("mov %%fs:0,%0" : "=r" (self) ); - return self; + uintptr_t tp; + __asm__ ("mov %%fs:0,%0" : "=r" (tp) ); + return tp; } -#define TP_ADJ(p) (p) - #define MC_PC gregs[REG_RIP] -#define CANARY canary2 +#define CANARY_PAD #define tls_mod_off_t unsigned long long diff --git a/arch/x86_64/bits/fcntl.h b/arch/x86_64/bits/fcntl.h deleted file mode 100644 index 1b88ad391..000000000 --- a/arch/x86_64/bits/fcntl.h +++ /dev/null @@ -1,40 +0,0 @@ -#define O_CREAT 0100 -#define O_EXCL 0200 -#define O_NOCTTY 0400 -#define O_TRUNC 01000 -#define O_APPEND 02000 -#define O_NONBLOCK 04000 -#define O_DSYNC 010000 -#define O_SYNC 04010000 -#define O_RSYNC 04010000 -#define O_DIRECTORY 0200000 -#define O_NOFOLLOW 0400000 -#define O_CLOEXEC 02000000 - -#define O_ASYNC 020000 -#define O_DIRECT 040000 -#define O_LARGEFILE 0 -#define O_NOATIME 01000000 -#define O_PATH 010000000 -#define O_TMPFILE 020200000 -#define O_NDELAY O_NONBLOCK - -#define F_DUPFD 0 -#define F_GETFD 1 -#define F_SETFD 2 -#define F_GETFL 3 -#define F_SETFL 4 - -#define F_SETOWN 8 -#define F_GETOWN 9 -#define F_SETSIG 10 -#define F_GETSIG 11 - -#define F_GETLK 5 -#define F_SETLK 6 -#define F_SETLKW 7 - -#define F_SETOWN_EX 15 -#define F_GETOWN_EX 16 - -#define F_GETOWNER_UIDS 17 diff --git a/arch/x86_64/bits/syscall.h.in b/arch/x86_64/bits/syscall.h.in index 6a646ad34..a61179510 100644 --- a/arch/x86_64/bits/syscall.h.in +++ b/arch/x86_64/bits/syscall.h.in @@ -345,4 +345,8 @@ #define __NR_fspick 433 #define __NR_pidfd_open 434 #define __NR_clone3 435 +#define __NR_close_range 436 +#define __NR_openat2 437 +#define __NR_pidfd_getfd 438 +#define __NR_faccessat2 439 diff --git a/arch/x86_64/pthread_arch.h b/arch/x86_64/pthread_arch.h index 65e880c62..c8c63f2e7 100644 --- a/arch/x86_64/pthread_arch.h +++ b/arch/x86_64/pthread_arch.h @@ -1,10 +1,8 @@ -static inline struct pthread *__pthread_self() +static inline uintptr_t __get_tp() { - struct pthread *self; - __asm__ ("mov %%fs:0,%0" : "=r" (self) ); - return self; + uintptr_t tp; + __asm__ ("mov %%fs:0,%0" : "=r" (tp) ); + return tp; } -#define TP_ADJ(p) (p) - #define MC_PC gregs[REG_RIP] diff --git a/configure b/configure index a2728969f..a5231a0eb 100755 --- a/configure +++ b/configure @@ -30,11 +30,14 @@ System types: Optional features: --enable-optimize=... optimize listed components for speed over size [auto] --enable-debug build with debugging information [disabled] - --enable-warnings build with recommended warnings flags [disabled] + --disable-warnings build with recommended warnings flags [enabled] --enable-wrapper=... build given musl toolchain wrapper [auto] --disable-shared inhibit building shared library [enabled] --disable-static inhibit building static library [enabled] +Optional packages: + --with-malloc=... choose malloc implementation [mallocng] + Some influential environment variables: CC C compiler command [detected] CFLAGS C compiler flags [-Os -pipe ...] @@ -133,12 +136,13 @@ build= target= optimize=auto debug=no -warnings=no +warnings=yes shared=auto static=yes wrapper=auto gcc_wrapper=no clang_wrapper=no +malloc_dir=mallocng for arg ; do case "$arg" in @@ -168,6 +172,7 @@ case "$arg" in --disable-wrapper|--enable-wrapper=no) wrapper=no ;; --enable-gcc-wrapper|--enable-gcc-wrapper=yes) wrapper=yes ; gcc_wrapper=yes ;; --disable-gcc-wrapper|--enable-gcc-wrapper=no) wrapper=no ;; +--with-malloc=*) malloc_dir=${arg#*=} ;; --enable-*|--disable-*|--with-*|--without-*|--*dir=*) ;; --host=*|--target=*) target=${arg#*=} ;; --build=*) build=${arg#*=} ;; @@ -199,7 +204,7 @@ fi abs_builddir="$(pwd)" || fail "$0: cannot determine working directory" abs_srcdir="$(cd $srcdir && pwd)" || fail "$0: invalid source directory $srcdir" test "$abs_srcdir" = "$abs_builddir" && srcdir=. -test "$srcdir" != "." -a -f Makefile -a ! -h Makefile && fail "$0: Makefile already exists in the working directory" +test "$srcdir" != "." && test -f Makefile && test ! -h Makefile && fail "$0: Makefile already exists in the working directory" # # Get a temp filename we can use @@ -214,6 +219,12 @@ done set +C trap 'rm "$tmpc"' EXIT INT QUIT TERM HUP +# +# Check that the requested malloc implementation exists +# +test -d "$srcdir/src/malloc/$malloc_dir" \ +|| fail "$0: error: chosen malloc implementation '$malloc_dir' does not exist" + # # Check whether we are cross-compiling, and set a default # CROSS_COMPILE prefix if none was provided. @@ -268,7 +279,7 @@ echo "$cc_family" # # Figure out toolchain wrapper to build # -if test "$wrapper" = auto -o "$wrapper" = detect ; then +if test "$wrapper" = auto || test "$wrapper" = detect ; then echo "#include " > "$tmpc" echo "#if ! __GLIBC__" >> "$tmpc" echo "#error no" >> "$tmpc" @@ -457,7 +468,7 @@ tryflag CFLAGS_AUTO -pipe # pointer is no longer needed for debugging. # if fnmatch '-g*|*\ -g*' "$CFLAGS_AUTO $CFLAGS" ; then : -else +else tryflag CFLAGS_AUTO -fomit-frame-pointer fi @@ -494,6 +505,16 @@ fnmatch '-march=*|*\ -march=*' "$CC $CFLAGS" || tryldflag CFLAGS_AUTO -march=i48 fnmatch '-mtune=*|*\ -mtune=*' "$CC $CFLAGS" || tryldflag CFLAGS_AUTO -mtune=generic fi +# +# GCC defines -w as overriding any -W options, regardless of order, but +# clang has a bunch of annoying warnings enabled by default and needs -w +# to start from a clean slate. So use -w if building with clang. Also +# turn off a common on-by-default cast warning regardless of compiler. +# +test "$cc_family" = clang && tryflag CFLAGS_AUTO -w + +tryflag CFLAGS_AUTO -Wno-pointer-to-int-cast + # # Even with -std=c99, gcc accepts some constructs which are constraint # violations. We want to treat these as errors regardless of whether @@ -504,6 +525,10 @@ tryflag CFLAGS_AUTO -Werror=implicit-function-declaration tryflag CFLAGS_AUTO -Werror=implicit-int tryflag CFLAGS_AUTO -Werror=pointer-sign tryflag CFLAGS_AUTO -Werror=pointer-arith +tryflag CFLAGS_AUTO -Werror=int-conversion +tryflag CFLAGS_AUTO -Werror=incompatible-pointer-types +tryflag CFLAGS_AUTO -Werror=discarded-qualifiers +tryflag CFLAGS_AUTO -Werror=discarded-array-qualifiers # # GCC ignores unused arguements by default, but Clang needs this extra @@ -513,14 +538,17 @@ tryflag CFLAGS_AUTO -Werror=pointer-arith test "$cc_family" = clang && tryflag CFLAGS_AUTO -Qunused-arguments if test "x$warnings" = xyes ; then -tryflag CFLAGS_AUTO -Wall -tryflag CFLAGS_AUTO -Wno-parentheses -tryflag CFLAGS_AUTO -Wno-uninitialized -tryflag CFLAGS_AUTO -Wno-missing-braces -tryflag CFLAGS_AUTO -Wno-unused-value -tryflag CFLAGS_AUTO -Wno-unused-but-set-variable -tryflag CFLAGS_AUTO -Wno-unknown-pragmas -tryflag CFLAGS_AUTO -Wno-pointer-to-int-cast +tryflag CFLAGS_AUTO -Waddress +tryflag CFLAGS_AUTO -Warray-bounds +tryflag CFLAGS_AUTO -Wchar-subscripts +tryflag CFLAGS_AUTO -Wduplicate-decl-specifier +tryflag CFLAGS_AUTO -Winit-self +tryflag CFLAGS_AUTO -Wreturn-type +tryflag CFLAGS_AUTO -Wsequence-point +tryflag CFLAGS_AUTO -Wstrict-aliasing +tryflag CFLAGS_AUTO -Wunused-function +tryflag CFLAGS_AUTO -Wunused-label +tryflag CFLAGS_AUTO -Wunused-variable fi # Determine if the compiler produces position-independent code (PIC) @@ -772,6 +800,7 @@ OPTIMIZE_GLOBS = $OPTIMIZE_GLOBS ALL_TOOLS = $tools TOOL_LIBS = $tool_libs ADD_CFI = $ADD_CFI +MALLOC_DIR = $malloc_dir EOF test "x$static" = xno && echo "STATIC_LIBS =" test "x$shared" = xno && echo "SHARED_LIBS =" diff --git a/include/alltypes.h.in b/include/alltypes.h.in index d9ff462e1..d47aeea9a 100644 --- a/include/alltypes.h.in +++ b/include/alltypes.h.in @@ -77,6 +77,8 @@ TYPEDEF struct __sigset_t { unsigned long __bits[128/sizeof(long)]; } sigset_t; STRUCT iovec { void *iov_base; size_t iov_len; }; +STRUCT winsize { unsigned short ws_row, ws_col, ws_xpixel, ws_ypixel; }; + TYPEDEF unsigned socklen_t; TYPEDEF unsigned short sa_family_t; diff --git a/include/elf.h b/include/elf.h index 549f92c1a..b5e7befb0 100644 --- a/include/elf.h +++ b/include/elf.h @@ -603,6 +603,7 @@ typedef struct { #define PT_GNU_EH_FRAME 0x6474e550 #define PT_GNU_STACK 0x6474e551 #define PT_GNU_RELRO 0x6474e552 +#define PT_GNU_PROPERTY 0x6474e553 #define PT_LOSUNW 0x6ffffffa #define PT_SUNWBSS 0x6ffffffa #define PT_SUNWSTACK 0x6ffffffb @@ -1085,6 +1086,7 @@ typedef struct { #define NT_GNU_BUILD_ID 3 #define NT_GNU_GOLD_VERSION 4 +#define NT_GNU_PROPERTY_TYPE_0 5 diff --git a/include/netinet/if_ether.h b/include/netinet/if_ether.h index a08485e7f..55a2ff1b1 100644 --- a/include/netinet/if_ether.h +++ b/include/netinet/if_ether.h @@ -59,6 +59,7 @@ #define ETH_P_PREAUTH 0x88C7 #define ETH_P_TIPC 0x88CA #define ETH_P_LLDP 0x88CC +#define ETH_P_MRP 0x88E3 #define ETH_P_MACSEC 0x88E5 #define ETH_P_8021AH 0x88E7 #define ETH_P_MVRP 0x88F5 diff --git a/include/netinet/in.h b/include/netinet/in.h index 103d2e044..f9594339f 100644 --- a/include/netinet/in.h +++ b/include/netinet/in.h @@ -101,8 +101,10 @@ uint16_t ntohs(uint16_t); #define IPPROTO_MH 135 #define IPPROTO_UDPLITE 136 #define IPPROTO_MPLS 137 +#define IPPROTO_ETHERNET 143 #define IPPROTO_RAW 255 -#define IPPROTO_MAX 256 +#define IPPROTO_MPTCP 262 +#define IPPROTO_MAX 263 #define IN6_IS_ADDR_UNSPECIFIED(a) \ (((uint32_t *) (a))[0] == 0 && ((uint32_t *) (a))[1] == 0 && \ @@ -200,6 +202,7 @@ uint16_t ntohs(uint16_t); #define IP_CHECKSUM 23 #define IP_BIND_ADDRESS_NO_PORT 24 #define IP_RECVFRAGSIZE 25 +#define IP_RECVERR_RFC4884 26 #define IP_MULTICAST_IF 32 #define IP_MULTICAST_TTL 33 #define IP_MULTICAST_LOOP 34 diff --git a/include/netinet/tcp.h b/include/netinet/tcp.h index 44a007aaf..b7b997f5f 100644 --- a/include/netinet/tcp.h +++ b/include/netinet/tcp.h @@ -78,6 +78,8 @@ enum { TCP_NLA_DSACK_DUPS, TCP_NLA_REORD_SEEN, TCP_NLA_SRTT, + TCP_NLA_TIMEOUT_REHASH, + TCP_NLA_BYTES_NOTSENT, }; #if defined(_GNU_SOURCE) || defined(_BSD_SOURCE) @@ -181,6 +183,13 @@ struct tcphdr { #define TCP_CA_Recovery 3 #define TCP_CA_Loss 4 +enum tcp_fastopen_client_fail { + TFO_STATUS_UNSPEC, + TFO_COOKIE_UNAVAILABLE, + TFO_DATA_NOT_ACKED, + TFO_SYN_RETRANSMITTED, +}; + struct tcp_info { uint8_t tcpi_state; uint8_t tcpi_ca_state; @@ -189,7 +198,7 @@ struct tcp_info { uint8_t tcpi_backoff; uint8_t tcpi_options; uint8_t tcpi_snd_wscale : 4, tcpi_rcv_wscale : 4; - uint8_t tcpi_delivery_rate_app_limited : 1; + uint8_t tcpi_delivery_rate_app_limited : 1, tcpi_fastopen_client_fail : 2; uint32_t tcpi_rto; uint32_t tcpi_ato; uint32_t tcpi_snd_mss; @@ -240,14 +249,15 @@ struct tcp_info { #define TCP_MD5SIG_MAXKEYLEN 80 -#define TCP_MD5SIG_FLAG_PREFIX 1 +#define TCP_MD5SIG_FLAG_PREFIX 0x1 +#define TCP_MD5SIG_FLAG_IFINDEX 0x2 struct tcp_md5sig { struct sockaddr_storage tcpm_addr; uint8_t tcpm_flags; uint8_t tcpm_prefixlen; uint16_t tcpm_keylen; - uint32_t __tcpm_pad; + int tcpm_ifindex; uint8_t tcpm_key[TCP_MD5SIG_MAXKEYLEN]; }; @@ -275,6 +285,8 @@ struct tcp_zerocopy_receive { uint64_t address; uint32_t length; uint32_t recv_skip_hint; + uint32_t inq; + int32_t err; }; #endif diff --git a/include/netinet/udp.h b/include/netinet/udp.h index ffd890796..40c3f2034 100644 --- a/include/netinet/udp.h +++ b/include/netinet/udp.h @@ -35,6 +35,7 @@ struct udphdr { #define UDP_ENCAP_GTP0 4 #define UDP_ENCAP_GTP1U 5 #define UDP_ENCAP_RXRPC 6 +#define TCP_ENCAP_ESPINTCP 7 #define SOL_UDP 17 diff --git a/include/sched.h b/include/sched.h index 822f464ef..fda4b4846 100644 --- a/include/sched.h +++ b/include/sched.h @@ -49,6 +49,7 @@ int sched_yield(void); #ifdef _GNU_SOURCE #define CSIGNAL 0x000000ff +#define CLONE_NEWTIME 0x00000080 #define CLONE_VM 0x00000100 #define CLONE_FS 0x00000200 #define CLONE_FILES 0x00000400 diff --git a/include/signal.h b/include/signal.h index fbdf667b2..9ed929e4f 100644 --- a/include/signal.h +++ b/include/signal.h @@ -180,14 +180,24 @@ struct sigevent { union sigval sigev_value; int sigev_signo; int sigev_notify; - void (*sigev_notify_function)(union sigval); - pthread_attr_t *sigev_notify_attributes; - char __pad[56-3*sizeof(long)]; + union { + char __pad[64 - 2*sizeof(int) - sizeof(union sigval)]; + pid_t sigev_notify_thread_id; + struct { + void (*sigev_notify_function)(union sigval); + pthread_attr_t *sigev_notify_attributes; + } __sev_thread; + } __sev_fields; }; +#define sigev_notify_thread_id __sev_fields.sigev_notify_thread_id +#define sigev_notify_function __sev_fields.__sev_thread.sigev_notify_function +#define sigev_notify_attributes __sev_fields.__sev_thread.sigev_notify_attributes + #define SIGEV_SIGNAL 0 #define SIGEV_NONE 1 #define SIGEV_THREAD 2 +#define SIGEV_THREAD_ID 4 int __libc_current_sigrtmin(void); int __libc_current_sigrtmax(void); diff --git a/include/stdlib.h b/include/stdlib.h index 194c20339..b54a051fe 100644 --- a/include/stdlib.h +++ b/include/stdlib.h @@ -145,6 +145,7 @@ int getloadavg(double *, int); int clearenv(void); #define WCOREDUMP(s) ((s) & 0x80) #define WIFCONTINUED(s) ((s) == 0xffff) +void *reallocarray (void *, size_t, size_t); #endif #ifdef _GNU_SOURCE diff --git a/include/sys/fanotify.h b/include/sys/fanotify.h index b637c8f58..10e5f15e2 100644 --- a/include/sys/fanotify.h +++ b/include/sys/fanotify.h @@ -55,8 +55,9 @@ struct fanotify_response { #define FAN_OPEN_PERM 0x10000 #define FAN_ACCESS_PERM 0x20000 #define FAN_OPEN_EXEC_PERM 0x40000 -#define FAN_ONDIR 0x40000000 +#define FAN_DIR_MODIFY 0x00080000 #define FAN_EVENT_ON_CHILD 0x08000000 +#define FAN_ONDIR 0x40000000 #define FAN_CLOSE (FAN_CLOSE_WRITE | FAN_CLOSE_NOWRITE) #define FAN_MOVE (FAN_MOVED_FROM | FAN_MOVED_TO) #define FAN_CLOEXEC 0x01 @@ -70,6 +71,9 @@ struct fanotify_response { #define FAN_ENABLE_AUDIT 0x40 #define FAN_REPORT_TID 0x100 #define FAN_REPORT_FID 0x200 +#define FAN_REPORT_DIR_FID 0x00000400 +#define FAN_REPORT_NAME 0x00000800 +#define FAN_REPORT_DFID_NAME (FAN_REPORT_DIR_FID | FAN_REPORT_NAME) #define FAN_ALL_INIT_FLAGS (FAN_CLOEXEC | FAN_NONBLOCK | FAN_ALL_CLASS_BITS | FAN_UNLIMITED_QUEUE | FAN_UNLIMITED_MARKS) #define FAN_MARK_ADD 0x01 #define FAN_MARK_REMOVE 0x02 @@ -88,6 +92,8 @@ struct fanotify_response { #define FAN_ALL_OUTGOING_EVENTS (FAN_ALL_EVENTS | FAN_ALL_PERM_EVENTS | FAN_Q_OVERFLOW) #define FANOTIFY_METADATA_VERSION 3 #define FAN_EVENT_INFO_TYPE_FID 1 +#define FAN_EVENT_INFO_TYPE_DFID_NAME 2 +#define FAN_EVENT_INFO_TYPE_DFID 3 #define FAN_ALLOW 0x01 #define FAN_DENY 0x02 #define FAN_AUDIT 0x10 diff --git a/include/sys/ioctl.h b/include/sys/ioctl.h index c2ce3b484..a9a2346ee 100644 --- a/include/sys/ioctl.h +++ b/include/sys/ioctl.h @@ -4,6 +4,8 @@ extern "C" { #endif +#define __NEED_struct_winsize + #include #include @@ -47,13 +49,6 @@ extern "C" { #define TIOCSER_TEMT 1 -struct winsize { - unsigned short ws_row; - unsigned short ws_col; - unsigned short ws_xpixel; - unsigned short ws_ypixel; -}; - #define SIOCADDRT 0x890B #define SIOCDELRT 0x890C #define SIOCRTMSG 0x890D diff --git a/include/sys/mman.h b/include/sys/mman.h index 3bade7272..4d603e910 100644 --- a/include/sys/mman.h +++ b/include/sys/mman.h @@ -101,6 +101,7 @@ extern "C" { #ifdef _GNU_SOURCE #define MREMAP_MAYMOVE 1 #define MREMAP_FIXED 2 +#define MREMAP_DONTUNMAP 4 #define MLOCK_ONFAULT 0x01 diff --git a/include/sys/personality.h b/include/sys/personality.h index 31d43dfe1..411dc4756 100644 --- a/include/sys/personality.h +++ b/include/sys/personality.h @@ -5,7 +5,9 @@ extern "C" { #endif +#define UNAME26 0x0020000 #define ADDR_NO_RANDOMIZE 0x0040000 +#define FDPIC_FUNCPTRS 0x0080000 #define MMAP_PAGE_ZERO 0x0100000 #define ADDR_COMPAT_LAYOUT 0x0200000 #define READ_IMPLIES_EXEC 0x0400000 @@ -17,6 +19,7 @@ extern "C" { #define PER_LINUX 0 #define PER_LINUX_32BIT ADDR_LIMIT_32BIT +#define PER_LINUX_FDPIC FDPIC_FUNCPTRS #define PER_SVR4 (1 | STICKY_TIMEOUTS | MMAP_PAGE_ZERO) #define PER_SVR3 (2 | STICKY_TIMEOUTS | SHORT_INODE) #define PER_SCOSVR3 (3 | STICKY_TIMEOUTS | WHOLE_SECONDS | SHORT_INODE) diff --git a/include/sys/prctl.h b/include/sys/prctl.h index d9c846e9c..4b9fcc050 100644 --- a/include/sys/prctl.h +++ b/include/sys/prctl.h @@ -158,6 +158,9 @@ struct prctl_mm_map { #define PR_GET_TAGGED_ADDR_CTRL 56 #define PR_TAGGED_ADDR_ENABLE (1UL << 0) +#define PR_SET_IO_FLUSHER 57 +#define PR_GET_IO_FLUSHER 58 + int prctl (int, ...); #ifdef __cplusplus diff --git a/include/sys/random.h b/include/sys/random.h index 4ee7bf2cc..59e40ab89 100644 --- a/include/sys/random.h +++ b/include/sys/random.h @@ -10,6 +10,7 @@ extern "C" { #define GRND_NONBLOCK 0x0001 #define GRND_RANDOM 0x0002 +#define GRND_INSECURE 0x0004 ssize_t getrandom(void *, size_t, unsigned); diff --git a/include/termios.h b/include/termios.h index d73c780d4..cbb533010 100644 --- a/include/termios.h +++ b/include/termios.h @@ -8,6 +8,7 @@ extern "C" { #include #define __NEED_pid_t +#define __NEED_struct_winsize #include @@ -27,6 +28,9 @@ int cfsetispeed (struct termios *, speed_t); int tcgetattr (int, struct termios *); int tcsetattr (int, int, const struct termios *); +int tcgetwinsize (int, struct winsize *); +int tcsetwinsize (int, const struct winsize *); + int tcsendbreak (int, int); int tcdrain (int); int tcflush (int, int); diff --git a/include/unistd.h b/include/unistd.h index 7bcbff943..130640260 100644 --- a/include/unistd.h +++ b/include/unistd.h @@ -82,6 +82,7 @@ unsigned sleep(unsigned); int pause(void); pid_t fork(void); +pid_t _Fork(void); int execve(const char *, char *const [], char *const []); int execv(const char *, char *const []); int execle(const char *, const char *, ...); @@ -190,6 +191,7 @@ int syncfs(int); int euidaccess(const char *, int); int eaccess(const char *, int); ssize_t copy_file_range(int, off_t *, int, off_t *, size_t, unsigned); +pid_t gettid(void); #endif #if defined(_LARGEFILE64_SOURCE) || defined(_GNU_SOURCE) diff --git a/ldso/dynlink.c b/ldso/dynlink.c index 6468f203c..6b868c841 100644 --- a/ldso/dynlink.c +++ b/ldso/dynlink.c @@ -1,6 +1,5 @@ #define _GNU_SOURCE #define SYSCALL_NO_TLS 1 -#include #include #include #include @@ -21,9 +20,14 @@ #include #include #include "pthread_impl.h" +#include "fork_impl.h" #include "libc.h" #include "dynlink.h" -#include "malloc_impl.h" + +#define malloc __libc_malloc +#define calloc __libc_calloc +#define realloc __libc_realloc +#define free __libc_free static void error(const char *, ...); @@ -79,7 +83,7 @@ struct dso { struct dso **deps, *needed_by; size_t ndeps_direct; size_t next_dep; - int ctor_visitor; + pthread_t ctor_visitor; char *rpath_orig, *rpath; struct tls_module tls; size_t tls_id; @@ -557,6 +561,20 @@ static void reclaim_gaps(struct dso *dso) } } +static ssize_t read_loop(int fd, void *p, size_t n) +{ + for (size_t i=0; i=0) { + size_t n = 0; + if (!fstat(fd, &st)) n = st.st_size; + if ((sys_path = malloc(n+1))) + sys_path[n] = 0; + if (!sys_path || read_loop(fd, sys_path, n)<0) { free(sys_path); sys_path = ""; } - fclose(f); + close(fd); } else if (errno != ENOENT) { sys_path = ""; } @@ -1379,7 +1401,7 @@ void __libc_exit_fini() { struct dso *p; size_t dyn[DYN_CNT]; - int self = __pthread_self()->tid; + pthread_t self = __pthread_self(); /* Take both locks before setting shutting_down, so that * either lock is sufficient to read its value. The lock @@ -1405,6 +1427,17 @@ void __libc_exit_fini() } } +void __ldso_atfork(int who) +{ + if (who<0) { + pthread_rwlock_wrlock(&lock); + pthread_mutex_lock(&init_fini_lock); + } else { + pthread_mutex_unlock(&init_fini_lock); + pthread_rwlock_unlock(&lock); + } +} + static struct dso **queue_ctors(struct dso *dso) { size_t cnt, qpos, spos, i; @@ -1463,6 +1496,13 @@ static struct dso **queue_ctors(struct dso *dso) } queue[qpos] = 0; for (i=0; imark = 0; + for (i=0; ictor_visitor && queue[i]->ctor_visitor->tid < 0) { + error("State of %s is inconsistent due to multithreaded fork\n", + queue[i]->name); + free(queue); + if (runtime) longjmp(*rtld_fail, 1); + } return queue; } @@ -1471,7 +1511,7 @@ static void do_init_fini(struct dso **queue) { struct dso *p; size_t dyn[DYN_CNT], i; - int self = __pthread_self()->tid; + pthread_t self = __pthread_self(); pthread_mutex_lock(&init_fini_lock); for (i=0; (p=queue[i]); i++) { @@ -1580,7 +1620,7 @@ static void install_new_tls(void) /* Install new dtv for each thread. */ for (j=0, td=self; !j || td!=self; j++, td=td->next) { - td->dtv = td->dtv_copy = newdtv[j]; + td->dtv = newdtv[j]; } __tl_unlock(); @@ -1936,6 +1976,8 @@ void __dls3(size_t *sp, size_t *auxv) * possibility of incomplete replacement. */ if (find_sym(head, "malloc", 1).dso != &ldso) __malloc_replaced = 1; + if (find_sym(head, "aligned_alloc", 1).dso != &ldso) + __aligned_alloc_replaced = 1; /* Switch to runtime mode: any further failures in the dynamic * linker are a reportable failure rather than a fatal startup @@ -1946,7 +1988,7 @@ void __dls3(size_t *sp, size_t *auxv) debug.bp = dl_debug_state; debug.head = head; debug.base = ldso.base; - debug.state = 0; + debug.state = RT_CONSISTENT; _dl_debug_state(); if (replace_argv0) argv[0] = replace_argv0; @@ -1995,6 +2037,9 @@ void *dlopen(const char *file, int mode) pthread_rwlock_wrlock(&lock); __inhibit_ptc(); + debug.state = RT_ADD; + _dl_debug_state(); + p = 0; if (shutting_down) { error("Cannot dlopen while program is exiting."); @@ -2054,8 +2099,9 @@ void *dlopen(const char *file, int mode) load_deps(p); extend_bfs_deps(p); pthread_mutex_lock(&init_fini_lock); - if (!p->constructed) ctor_queue = queue_ctors(p); + int constructed = p->constructed; pthread_mutex_unlock(&init_fini_lock); + if (!constructed) ctor_queue = queue_ctors(p); if (!p->relocated && (mode & RTLD_LAZY)) { prepare_lazy(p); for (i=0; p->deps[i]; i++) @@ -2087,9 +2133,10 @@ void *dlopen(const char *file, int mode) update_tls_size(); if (tls_cnt != orig_tls_cnt) install_new_tls(); - _dl_debug_state(); orig_tail = tail; end: + debug.state = RT_CONSISTENT; + _dl_debug_state(); __release_ptc(); if (p) gencnt++; pthread_rwlock_unlock(&lock); diff --git a/src/aio/aio.c b/src/aio/aio.c index 6d34fa869..a1a3e7914 100644 --- a/src/aio/aio.c +++ b/src/aio/aio.c @@ -9,6 +9,12 @@ #include "syscall.h" #include "atomic.h" #include "pthread_impl.h" +#include "aio_impl.h" + +#define malloc __libc_malloc +#define calloc __libc_calloc +#define realloc __libc_realloc +#define free __libc_free /* The following is a threads-based implementation of AIO with minimal * dependence on implementation details. Most synchronization is @@ -70,6 +76,10 @@ static struct aio_queue *****map; static volatile int aio_fd_cnt; volatile int __aio_fut; +static size_t io_thread_stack_size; + +#define MAX(a,b) ((a)>(b) ? (a) : (b)) + static struct aio_queue *__aio_get_queue(int fd, int need) { if (fd < 0) { @@ -84,6 +94,10 @@ static struct aio_queue *__aio_get_queue(int fd, int need) pthread_rwlock_unlock(&maplock); if (fcntl(fd, F_GETFD) < 0) return 0; pthread_rwlock_wrlock(&maplock); + if (!io_thread_stack_size) { + unsigned long val = __getauxval(AT_MINSIGSTKSZ); + io_thread_stack_size = MAX(MINSIGSTKSZ+2048, val+512); + } if (!map) map = calloc(sizeof *map, (-1U/2+1)>>24); if (!map) goto out; if (!map[a]) map[a] = calloc(sizeof **map, 256); @@ -259,15 +273,6 @@ static void *io_thread_func(void *ctx) return 0; } -static size_t io_thread_stack_size = MINSIGSTKSZ+2048; -static pthread_once_t init_stack_size_once; - -static void init_stack_size() -{ - unsigned long val = __getauxval(AT_MINSIGSTKSZ); - if (val > MINSIGSTKSZ) io_thread_stack_size = val + 512; -} - static int submit(struct aiocb *cb, int op) { int ret = 0; @@ -293,7 +298,6 @@ static int submit(struct aiocb *cb, int op) else pthread_attr_init(&a); } else { - pthread_once(&init_stack_size_once, init_stack_size); pthread_attr_init(&a); pthread_attr_setstacksize(&a, io_thread_stack_size); pthread_attr_setguardsize(&a, 0); @@ -392,6 +396,20 @@ int __aio_close(int fd) return fd; } +void __aio_atfork(int who) +{ + if (who<0) { + pthread_rwlock_rdlock(&maplock); + return; + } + if (who>0 && map) for (int a=0; a<(-1U/2+1)>>24; a++) + if (map[a]) for (int b=0; b<256; b++) + if (map[a][b]) for (int c=0; c<256; c++) + if (map[a][b][c]) for (int d=0; d<256; d++) + map[a][b][c][d] = 0; + pthread_rwlock_unlock(&maplock); +} + weak_alias(aio_cancel, aio_cancel64); weak_alias(aio_error, aio_error64); weak_alias(aio_fsync, aio_fsync64); diff --git a/src/aio/aio_suspend.c b/src/aio/aio_suspend.c index 34b66f878..1c1060e34 100644 --- a/src/aio/aio_suspend.c +++ b/src/aio/aio_suspend.c @@ -3,6 +3,7 @@ #include #include "atomic.h" #include "pthread_impl.h" +#include "aio_impl.h" int aio_suspend(const struct aiocb *const cbs[], int cnt, const struct timespec *ts) { diff --git a/src/crypt/crypt_blowfish.c b/src/crypt/crypt_blowfish.c index d3f798517..d722607b0 100644 --- a/src/crypt/crypt_blowfish.c +++ b/src/crypt/crypt_blowfish.c @@ -15,7 +15,7 @@ * No copyright is claimed, and the software is hereby placed in the public * domain. In case this attempt to disclaim copyright and place the software * in the public domain is deemed null and void, then the software is - * Copyright (c) 1998-2012 Solar Designer and it is hereby released to the + * Copyright (c) 1998-2014 Solar Designer and it is hereby released to the * general public under the following terms: * * Redistribution and use in source and binary forms, with or without @@ -31,12 +31,12 @@ * you place this code and any modifications you make under a license * of your choice. * - * This implementation is mostly compatible with OpenBSD's bcrypt.c (prefix - * "$2a$") by Niels Provos , and uses some of his - * ideas. The password hashing algorithm was designed by David Mazieres - * . For more information on the level of compatibility, - * please refer to the comments in BF_set_key() below and to the included - * crypt(3) man page. + * This implementation is fully compatible with OpenBSD's bcrypt.c for prefix + * "$2b$", originally by Niels Provos , and it uses + * some of his ideas. The password hashing algorithm was designed by David + * Mazieres . For information on the level of + * compatibility for bcrypt hash prefixes other than "$2b$", please refer to + * the comments in BF_set_key() below and to the included crypt(3) man page. * * There's a paper on the algorithm that explains its design decisions: * @@ -533,6 +533,7 @@ static void BF_set_key(const char *key, BF_key expanded, BF_key initial, * Valid combinations of settings are: * * Prefix "$2a$": bug = 0, safety = 0x10000 + * Prefix "$2b$": bug = 0, safety = 0 * Prefix "$2x$": bug = 1, safety = 0 * Prefix "$2y$": bug = 0, safety = 0 */ @@ -596,12 +597,14 @@ static void BF_set_key(const char *key, BF_key expanded, BF_key initial, initial[0] ^= sign; } +static const unsigned char flags_by_subtype[26] = { + 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0 +}; + static char *BF_crypt(const char *key, const char *setting, char *output, BF_word min) { - static const unsigned char flags_by_subtype[26] = - {2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0}; struct { BF_ctx ctx; BF_key expanded_key; @@ -746,9 +749,11 @@ char *__crypt_blowfish(const char *key, const char *setting, char *output) { const char *test_key = "8b \xd0\xc1\xd2\xcf\xcc\xd8"; const char *test_setting = "$2a$00$abcdefghijklmnopqrstuu"; - static const char test_hash[2][34] = - {"VUrPmXD6q/nVSSp7pNDhCR9071IfIRe\0\x55", /* $2x$ */ - "i1D709vfamulimlGcq0qq3UvuUasvEa\0\x55"}; /* $2a$, $2y$ */ + static const char test_hashes[2][34] = { + "i1D709vfamulimlGcq0qq3UvuUasvEa\0\x55", /* 'a', 'b', 'y' */ + "VUrPmXD6q/nVSSp7pNDhCR9071IfIRe\0\x55", /* 'x' */ + }; + const char *test_hash = test_hashes[0]; char *retval; const char *p; int ok; @@ -768,8 +773,11 @@ char *__crypt_blowfish(const char *key, const char *setting, char *output) * detected by the self-test. */ memcpy(buf.s, test_setting, sizeof(buf.s)); - if (retval) + if (retval) { + unsigned int flags = flags_by_subtype[setting[2] - 'a']; + test_hash = test_hashes[flags & 1]; buf.s[2] = setting[2]; + } memset(buf.o, 0x55, sizeof(buf.o)); buf.o[sizeof(buf.o) - 1] = 0; p = BF_crypt(test_key, buf.s, buf.o, 1); @@ -777,7 +785,7 @@ char *__crypt_blowfish(const char *key, const char *setting, char *output) ok = (p == buf.o && !memcmp(p, buf.s, 7 + 22) && !memcmp(p + (7 + 22), - test_hash[buf.s[2] & 1], + test_hash, 31 + 1 + 1 + 1)); { diff --git a/src/env/__init_tls.c b/src/env/__init_tls.c index 772baba32..a93141ed3 100644 --- a/src/env/__init_tls.c +++ b/src/env/__init_tls.c @@ -67,7 +67,7 @@ void *__copy_tls(unsigned char *mem) } #endif dtv[0] = libc.tls_cnt; - td->dtv = td->dtv_copy = dtv; + td->dtv = dtv; return td; } diff --git a/src/env/__libc_start_main.c b/src/env/__libc_start_main.c index 8fbe52627..c5b277bdc 100644 --- a/src/env/__libc_start_main.c +++ b/src/env/__libc_start_main.c @@ -69,7 +69,8 @@ weak_alias(libc_start_init, __libc_start_init); typedef int lsm2_fn(int (*)(int,char **,char **), int, char **); static lsm2_fn libc_start_main_stage2; -int __libc_start_main(int (*main)(int,char **,char **), int argc, char **argv) +int __libc_start_main(int (*main)(int,char **,char **), int argc, char **argv, + void (*init_dummy)(), void(*fini_dummy)(), void(*ldso_dummy)()) { char **envp = argv+argc+1; diff --git a/src/env/__stack_chk_fail.c b/src/env/__stack_chk_fail.c index e32596d10..bf5a280ad 100644 --- a/src/env/__stack_chk_fail.c +++ b/src/env/__stack_chk_fail.c @@ -9,7 +9,7 @@ void __init_ssp(void *entropy) if (entropy) memcpy(&__stack_chk_guard, entropy, sizeof(uintptr_t)); else __stack_chk_guard = (uintptr_t)&__stack_chk_guard * 1103515245; - __pthread_self()->CANARY = __stack_chk_guard; + __pthread_self()->canary = __stack_chk_guard; } void __stack_chk_fail(void) diff --git a/src/exit/abort.c b/src/exit/abort.c index e1980f10a..f21f458ec 100644 --- a/src/exit/abort.c +++ b/src/exit/abort.c @@ -6,8 +6,6 @@ #include "lock.h" #include "ksigaction.h" -hidden volatile int __abort_lock[1]; - _Noreturn void abort(void) { raise(SIGABRT); diff --git a/src/exit/abort_lock.c b/src/exit/abort_lock.c new file mode 100644 index 000000000..3af72c7b6 --- /dev/null +++ b/src/exit/abort_lock.c @@ -0,0 +1,3 @@ +#include "pthread_impl.h" + +volatile int __abort_lock[1]; diff --git a/src/exit/assert.c b/src/exit/assert.c index 49b0dc3ec..94edd8272 100644 --- a/src/exit/assert.c +++ b/src/exit/assert.c @@ -4,6 +4,5 @@ _Noreturn void __assert_fail(const char *expr, const char *file, int line, const char *func) { fprintf(stderr, "Assertion failed: %s (%s: %s: %d)\n", expr, file, func, line); - fflush(NULL); abort(); } diff --git a/src/exit/at_quick_exit.c b/src/exit/at_quick_exit.c index d3ce6522d..e4b5d78db 100644 --- a/src/exit/at_quick_exit.c +++ b/src/exit/at_quick_exit.c @@ -1,12 +1,14 @@ #include #include "libc.h" #include "lock.h" +#include "fork_impl.h" #define COUNT 32 static void (*funcs[COUNT])(void); static int count; static volatile int lock[1]; +volatile int *const __at_quick_exit_lockptr = lock; void __funcs_on_quick_exit() { diff --git a/src/exit/atexit.c b/src/exit/atexit.c index 160d277ae..854e9fddb 100644 --- a/src/exit/atexit.c +++ b/src/exit/atexit.c @@ -2,6 +2,12 @@ #include #include "libc.h" #include "lock.h" +#include "fork_impl.h" + +#define malloc __libc_malloc +#define calloc __libc_calloc +#define realloc undef +#define free undef /* Ensure that at least 32 atexit handlers can be registered without malloc */ #define COUNT 32 @@ -15,6 +21,7 @@ static struct fl static int slot; static volatile int lock[1]; +volatile int *const __atexit_lockptr = lock; void __funcs_on_exit() { diff --git a/src/fenv/sh/fenv.S b/src/fenv/sh/fenv.S index 907aefc0d..b3b7d66ad 100644 --- a/src/fenv/sh/fenv.S +++ b/src/fenv/sh/fenv.S @@ -12,6 +12,8 @@ fegetround: .type __fesetround, @function __fesetround: sts fpscr, r0 + mov #-4, r1 + and r1, r0 or r4, r0 lds r0, fpscr rts diff --git a/src/include/stdlib.h b/src/include/stdlib.h index d38a5417f..e9da20158 100644 --- a/src/include/stdlib.h +++ b/src/include/stdlib.h @@ -9,4 +9,10 @@ hidden int __mkostemps(char *, int, int); hidden int __ptsname_r(int, char *, size_t); hidden char *__randname(char *); +hidden void *__libc_malloc(size_t); +hidden void *__libc_malloc_impl(size_t); +hidden void *__libc_calloc(size_t, size_t); +hidden void *__libc_realloc(void *, size_t); +hidden void __libc_free(void *); + #endif diff --git a/src/include/unistd.h b/src/include/unistd.h index 1b4605c7c..7b52a9249 100644 --- a/src/include/unistd.h +++ b/src/include/unistd.h @@ -8,7 +8,6 @@ extern char **__environ; hidden int __dup3(int, int, int); hidden int __mkostemps(char *, int, int); hidden int __execvpe(const char *, char *const *, char *const *); -hidden int __aio_close(int); hidden off_t __lseek(int, off_t, int); #endif diff --git a/src/internal/aio_impl.h b/src/internal/aio_impl.h new file mode 100644 index 000000000..a86576654 --- /dev/null +++ b/src/internal/aio_impl.h @@ -0,0 +1,9 @@ +#ifndef AIO_IMPL_H +#define AIO_IMPL_H + +extern hidden volatile int __aio_fut; + +extern hidden int __aio_close(int); +extern hidden void __aio_atfork(int); + +#endif diff --git a/src/internal/atomic.h b/src/internal/atomic.h index f938879b0..96c1552d6 100644 --- a/src/internal/atomic.h +++ b/src/internal/atomic.h @@ -315,4 +315,19 @@ static inline int a_clz_64(uint64_t x) } #endif +#ifndef a_clz_32 +#define a_clz_32 a_clz_32 +static inline int a_clz_32(uint32_t x) +{ + x >>= 1; + x |= x >> 1; + x |= x >> 2; + x |= x >> 4; + x |= x >> 8; + x |= x >> 16; + x++; + return 31-a_ctz_32(x); +} +#endif + #endif diff --git a/src/internal/dynlink.h b/src/internal/dynlink.h index 764e3a1a8..51c0639ff 100644 --- a/src/internal/dynlink.h +++ b/src/internal/dynlink.h @@ -105,4 +105,9 @@ hidden void __dl_vseterr(const char *, va_list); hidden ptrdiff_t __tlsdesc_static(), __tlsdesc_dynamic(); +hidden extern int __malloc_replaced; +hidden extern int __aligned_alloc_replaced; +hidden void __malloc_donate(char *, char *); +hidden int __malloc_allzerop(void *); + #endif diff --git a/src/internal/fork_impl.h b/src/internal/fork_impl.h new file mode 100644 index 000000000..5892c13bf --- /dev/null +++ b/src/internal/fork_impl.h @@ -0,0 +1,19 @@ +#include + +extern hidden volatile int *const __at_quick_exit_lockptr; +extern hidden volatile int *const __atexit_lockptr; +extern hidden volatile int *const __dlerror_lockptr; +extern hidden volatile int *const __gettext_lockptr; +extern hidden volatile int *const __locale_lockptr; +extern hidden volatile int *const __random_lockptr; +extern hidden volatile int *const __sem_open_lockptr; +extern hidden volatile int *const __stdio_ofl_lockptr; +extern hidden volatile int *const __syslog_lockptr; +extern hidden volatile int *const __timezone_lockptr; + +extern hidden volatile int *const __bump_lockptr; + +extern hidden volatile int *const __vmlock_lockptr; + +hidden void __malloc_atfork(int); +hidden void __ldso_atfork(int); diff --git a/src/internal/libc.h b/src/internal/libc.h index ac97dc7eb..619bba861 100644 --- a/src/internal/libc.h +++ b/src/internal/libc.h @@ -18,10 +18,11 @@ struct tls_module { }; struct __libc { - int can_do_threads; - int threaded; - int secure; - volatile int threads_minus_1; + char can_do_threads; + char threaded; + char secure; + volatile signed char need_locks; + int threads_minus_1; size_t *auxv; struct tls_module *tls_head; size_t tls_size, tls_align, tls_cnt; diff --git a/src/internal/libm.h b/src/internal/libm.h index 7533f6bae..72ad17d8e 100644 --- a/src/internal/libm.h +++ b/src/internal/libm.h @@ -267,5 +267,8 @@ hidden double __math_uflow(uint32_t); hidden double __math_oflow(uint32_t); hidden double __math_divzero(uint32_t); hidden double __math_invalid(double); +#if LDBL_MANT_DIG != DBL_MANT_DIG +hidden long double __math_invalidl(long double); +#endif #endif diff --git a/src/internal/locale_impl.h b/src/internal/locale_impl.h index 741a71c4d..4431a92eb 100644 --- a/src/internal/locale_impl.h +++ b/src/internal/locale_impl.h @@ -15,6 +15,8 @@ struct __locale_map { const struct __locale_map *next; }; +extern hidden volatile int __locale_lock[1]; + extern hidden const struct __locale_map __c_dot_utf8; extern hidden const struct __locale_struct __c_locale; extern hidden const struct __locale_struct __c_dot_utf8_locale; diff --git a/src/internal/pthread_impl.h b/src/internal/pthread_impl.h index 5742dfc55..de2b9d8b4 100644 --- a/src/internal/pthread_impl.h +++ b/src/internal/pthread_impl.h @@ -11,16 +11,25 @@ #include "atomic.h" #include "futex.h" +#include "pthread_arch.h" + #define pthread __pthread struct pthread { /* Part 1 -- these fields may be external or * internal (accessed via asm) ABI. Do not change. */ struct pthread *self; +#ifndef TLS_ABOVE_TP uintptr_t *dtv; +#endif struct pthread *prev, *next; /* non-ABI */ uintptr_t sysinfo; - uintptr_t canary, canary2; +#ifndef TLS_ABOVE_TP +#ifdef CANARY_PAD + uintptr_t canary_pad; +#endif + uintptr_t canary; +#endif /* Part 2 -- implementation details, non-ABI. */ int tid; @@ -43,6 +52,7 @@ struct pthread { long off; volatile void *volatile pending; } robust_list; + int h_errno_val; volatile int timer_id; locale_t locale; volatile int killlock[1]; @@ -51,21 +61,19 @@ struct pthread { /* Part 3 -- the positions of these fields relative to * the end of the structure is external and internal ABI. */ - uintptr_t canary_at_end; - uintptr_t *dtv_copy; +#ifdef TLS_ABOVE_TP + uintptr_t canary; + uintptr_t *dtv; +#endif }; enum { - DT_EXITING = 0, + DT_EXITED = 0, + DT_EXITING, DT_JOINABLE, DT_DETACHED, }; -struct __timer { - int timerid; - pthread_t thread; -}; - #define __SU (sizeof(size_t)/sizeof(int)) #define _a_stacksize __u.__s[0] @@ -98,16 +106,22 @@ struct __timer { #define _b_waiters2 __u.__vi[4] #define _b_inst __u.__p[3] -#include "pthread_arch.h" - -#ifndef CANARY -#define CANARY canary +#ifndef TP_OFFSET +#define TP_OFFSET 0 #endif #ifndef DTP_OFFSET #define DTP_OFFSET 0 #endif +#ifdef TLS_ABOVE_TP +#define TP_ADJ(p) ((char *)(p) + sizeof(struct pthread) + TP_OFFSET) +#define __pthread_self() ((pthread_t)(__get_tp() - sizeof(struct __pthread) - TP_OFFSET)) +#else +#define TP_ADJ(p) (p) +#define __pthread_self() ((pthread_t)__get_tp()) +#endif + #ifndef tls_mod_off_t #define tls_mod_off_t size_t #endif @@ -141,7 +155,6 @@ hidden int __pthread_key_delete_impl(pthread_key_t); extern hidden volatile size_t __pthread_tsd_size; extern hidden void *__pthread_tsd_main[]; -extern hidden volatile int __aio_fut; extern hidden volatile int __eintr_valid_flag; hidden int __clone(int (*)(void *), void *, int, void *, ...); @@ -176,6 +189,8 @@ hidden void __tl_sync(pthread_t); extern hidden volatile int __thread_list_lock; +extern hidden volatile int __abort_lock[1]; + extern hidden unsigned __default_stacksize; extern hidden unsigned __default_guardsize; diff --git a/src/internal/shgetc.c b/src/internal/shgetc.c index a4a9c633d..7455d2f00 100644 --- a/src/internal/shgetc.c +++ b/src/internal/shgetc.c @@ -32,6 +32,6 @@ int __shgetc(FILE *f) else f->shend = f->rend; f->shcnt = f->buf - f->rpos + cnt; - if (f->rpos[-1] != c) f->rpos[-1] = c; + if (f->rpos <= f->buf) f->rpos[-1] = c; return c; } diff --git a/src/internal/stdio_impl.h b/src/internal/stdio_impl.h index d7398f591..0b2438d61 100644 --- a/src/internal/stdio_impl.h +++ b/src/internal/stdio_impl.h @@ -60,8 +60,6 @@ hidden size_t __stdout_write(FILE *, const unsigned char *, size_t); hidden off_t __stdio_seek(FILE *, off_t, int); hidden int __stdio_close(FILE *); -hidden size_t __string_read(FILE *, unsigned char *, size_t); - hidden int __toread(FILE *); hidden int __towrite(FILE *); diff --git a/src/internal/syscall.h b/src/internal/syscall.h index 975a0031d..d5f294d43 100644 --- a/src/internal/syscall.h +++ b/src/internal/syscall.h @@ -2,6 +2,7 @@ #define _INTERNAL_SYSCALL_H #include +#include #include #include "syscall_arch.h" @@ -57,15 +58,22 @@ hidden long __syscall_ret(unsigned long), #define __syscall_cp(...) __SYSCALL_DISP(__syscall_cp,__VA_ARGS__) #define syscall_cp(...) __syscall_ret(__syscall_cp(__VA_ARGS__)) -#ifndef SYSCALL_USE_SOCKETCALL -#define __socketcall(nm,a,b,c,d,e,f) __syscall(SYS_##nm, a, b, c, d, e, f) -#define __socketcall_cp(nm,a,b,c,d,e,f) __syscall_cp(SYS_##nm, a, b, c, d, e, f) -#else -#define __socketcall(nm,a,b,c,d,e,f) __syscall(SYS_socketcall, __SC_##nm, \ - ((long [6]){ (long)a, (long)b, (long)c, (long)d, (long)e, (long)f })) -#define __socketcall_cp(nm,a,b,c,d,e,f) __syscall_cp(SYS_socketcall, __SC_##nm, \ - ((long [6]){ (long)a, (long)b, (long)c, (long)d, (long)e, (long)f })) -#endif +static inline long __alt_socketcall(int sys, int sock, int cp, long a, long b, long c, long d, long e, long f) +{ + long r; + if (cp) r = __syscall_cp(sys, a, b, c, d, e, f); + else r = __syscall(sys, a, b, c, d, e, f); + if (r != -ENOSYS) return r; +#ifdef SYS_socketcall + if (cp) r = __syscall_cp(SYS_socketcall, sock, ((long[6]){a, b, c, d, e, f})); + else r = __syscall(SYS_socketcall, sock, ((long[6]){a, b, c, d, e, f})); +#endif + return r; +} +#define __socketcall(nm, a, b, c, d, e, f) __alt_socketcall(SYS_##nm, __SC_##nm, 0, \ + (long)(a), (long)(b), (long)(c), (long)(d), (long)(e), (long)(f)) +#define __socketcall_cp(nm, a, b, c, d, e, f) __alt_socketcall(SYS_##nm, __SC_##nm, 1, \ + (long)(a), (long)(b), (long)(c), (long)(d), (long)(e), (long)(f)) /* fixup legacy 16-bit junk */ @@ -338,6 +346,12 @@ hidden long __syscall_ret(unsigned long), #define __SC_recvmmsg 19 #define __SC_sendmmsg 20 +/* This is valid only because all socket syscalls are made via + * socketcall, which always fills unused argument slots with zeros. */ +#ifndef SYS_accept +#define SYS_accept SYS_accept4 +#endif + #ifndef SO_RCVTIMEO_OLD #define SO_RCVTIMEO_OLD 20 #endif diff --git a/src/ldso/dlerror.c b/src/ldso/dlerror.c index 3fcc77795..afe59253e 100644 --- a/src/ldso/dlerror.c +++ b/src/ldso/dlerror.c @@ -4,6 +4,12 @@ #include "pthread_impl.h" #include "dynlink.h" #include "lock.h" +#include "fork_impl.h" + +#define malloc __libc_malloc +#define calloc __libc_calloc +#define realloc __libc_realloc +#define free __libc_free char *dlerror() { @@ -19,6 +25,7 @@ char *dlerror() static volatile int freebuf_queue_lock[1]; static void **freebuf_queue; +volatile int *const __dlerror_lockptr = freebuf_queue_lock; void __dl_thread_cleanup(void) { @@ -35,13 +42,16 @@ void __dl_thread_cleanup(void) hidden void __dl_vseterr(const char *fmt, va_list ap) { LOCK(freebuf_queue_lock); - while (freebuf_queue) { - void **p = freebuf_queue; - freebuf_queue = *p; - free(p); - } + void **q = freebuf_queue; + freebuf_queue = 0; UNLOCK(freebuf_queue_lock); + while (q) { + void **p = *q; + free(q); + q = p; + } + va_list ap2; va_copy(ap2, ap); pthread_t self = __pthread_self(); diff --git a/src/legacy/lutimes.c b/src/legacy/lutimes.c index 2e5502d1e..dd465923a 100644 --- a/src/legacy/lutimes.c +++ b/src/legacy/lutimes.c @@ -6,9 +6,11 @@ int lutimes(const char *filename, const struct timeval tv[2]) { struct timespec times[2]; - times[0].tv_sec = tv[0].tv_sec; - times[0].tv_nsec = tv[0].tv_usec * 1000; - times[1].tv_sec = tv[1].tv_sec; - times[1].tv_nsec = tv[1].tv_usec * 1000; - return utimensat(AT_FDCWD, filename, times, AT_SYMLINK_NOFOLLOW); + if (tv) { + times[0].tv_sec = tv[0].tv_sec; + times[0].tv_nsec = tv[0].tv_usec * 1000; + times[1].tv_sec = tv[1].tv_sec; + times[1].tv_nsec = tv[1].tv_usec * 1000; + } + return utimensat(AT_FDCWD, filename, tv ? times : 0, AT_SYMLINK_NOFOLLOW); } diff --git a/src/linux/clock_adjtime.c b/src/linux/clock_adjtime.c index 23eb8729d..d4d03d24d 100644 --- a/src/linux/clock_adjtime.c +++ b/src/linux/clock_adjtime.c @@ -38,55 +38,52 @@ int clock_adjtime (clockid_t clock_id, struct timex *utx) { int r = -ENOSYS; #ifdef SYS_clock_adjtime64 - if (SYS_clock_adjtime == SYS_clock_adjtime64 || - (utx->modes & ADJ_SETOFFSET) && !IS32BIT(utx->time.tv_sec)) { - struct ktimex64 ktx = { - .modes = utx->modes, - .offset = utx->offset, - .freq = utx->freq, - .maxerror = utx->maxerror, - .esterror = utx->esterror, - .status = utx->status, - .constant = utx->constant, - .precision = utx->precision, - .tolerance = utx->tolerance, - .time_sec = utx->time.tv_sec, - .time_usec = utx->time.tv_usec, - .tick = utx->tick, - .ppsfreq = utx->ppsfreq, - .jitter = utx->jitter, - .shift = utx->shift, - .stabil = utx->stabil, - .jitcnt = utx->jitcnt, - .calcnt = utx->calcnt, - .errcnt = utx->errcnt, - .stbcnt = utx->stbcnt, - .tai = utx->tai, - }; - r = __syscall(SYS_clock_adjtime, clock_id, &ktx); - if (r>=0) { - utx->modes = ktx.modes; - utx->offset = ktx.offset; - utx->freq = ktx.freq; - utx->maxerror = ktx.maxerror; - utx->esterror = ktx.esterror; - utx->status = ktx.status; - utx->constant = ktx.constant; - utx->precision = ktx.precision; - utx->tolerance = ktx.tolerance; - utx->time.tv_sec = ktx.time_sec; - utx->time.tv_usec = ktx.time_usec; - utx->tick = ktx.tick; - utx->ppsfreq = ktx.ppsfreq; - utx->jitter = ktx.jitter; - utx->shift = ktx.shift; - utx->stabil = ktx.stabil; - utx->jitcnt = ktx.jitcnt; - utx->calcnt = ktx.calcnt; - utx->errcnt = ktx.errcnt; - utx->stbcnt = ktx.stbcnt; - utx->tai = ktx.tai; - } + struct ktimex64 ktx = { + .modes = utx->modes, + .offset = utx->offset, + .freq = utx->freq, + .maxerror = utx->maxerror, + .esterror = utx->esterror, + .status = utx->status, + .constant = utx->constant, + .precision = utx->precision, + .tolerance = utx->tolerance, + .time_sec = utx->time.tv_sec, + .time_usec = utx->time.tv_usec, + .tick = utx->tick, + .ppsfreq = utx->ppsfreq, + .jitter = utx->jitter, + .shift = utx->shift, + .stabil = utx->stabil, + .jitcnt = utx->jitcnt, + .calcnt = utx->calcnt, + .errcnt = utx->errcnt, + .stbcnt = utx->stbcnt, + .tai = utx->tai, + }; + r = __syscall(SYS_clock_adjtime64, clock_id, &ktx); + if (r>=0) { + utx->modes = ktx.modes; + utx->offset = ktx.offset; + utx->freq = ktx.freq; + utx->maxerror = ktx.maxerror; + utx->esterror = ktx.esterror; + utx->status = ktx.status; + utx->constant = ktx.constant; + utx->precision = ktx.precision; + utx->tolerance = ktx.tolerance; + utx->time.tv_sec = ktx.time_sec; + utx->time.tv_usec = ktx.time_usec; + utx->tick = ktx.tick; + utx->ppsfreq = ktx.ppsfreq; + utx->jitter = ktx.jitter; + utx->shift = ktx.shift; + utx->stabil = ktx.stabil; + utx->jitcnt = ktx.jitcnt; + utx->calcnt = ktx.calcnt; + utx->errcnt = ktx.errcnt; + utx->stbcnt = ktx.stbcnt; + utx->tai = ktx.tai; } if (SYS_clock_adjtime == SYS_clock_adjtime64 || r!=-ENOSYS) return __syscall_ret(r); diff --git a/src/linux/gettid.c b/src/linux/gettid.c new file mode 100644 index 000000000..70767137e --- /dev/null +++ b/src/linux/gettid.c @@ -0,0 +1,8 @@ +#define _GNU_SOURCE +#include +#include "pthread_impl.h" + +pid_t gettid(void) +{ + return __pthread_self()->tid; +} diff --git a/src/linux/membarrier.c b/src/linux/membarrier.c index 9ebe906ed..343f7360e 100644 --- a/src/linux/membarrier.c +++ b/src/linux/membarrier.c @@ -9,13 +9,8 @@ static void dummy_0(void) { } -static void dummy_1(pthread_t t) -{ -} - weak_alias(dummy_0, __tl_lock); weak_alias(dummy_0, __tl_unlock); -weak_alias(dummy_1, __tl_sync); static sem_t barrier_sem; diff --git a/src/linux/setgroups.c b/src/linux/setgroups.c index 1248fdbfd..47142f141 100644 --- a/src/linux/setgroups.c +++ b/src/linux/setgroups.c @@ -1,8 +1,36 @@ #define _GNU_SOURCE #include +#include #include "syscall.h" +#include "libc.h" + +struct ctx { + size_t count; + const gid_t *list; + int ret; +}; + +static void do_setgroups(void *p) +{ + struct ctx *c = p; + if (c->ret<0) return; + int ret = __syscall(SYS_setgroups, c->count, c->list); + if (ret && !c->ret) { + /* If one thread fails to set groups after another has already + * succeeded, forcibly killing the process is the only safe + * thing to do. State is inconsistent and dangerous. Use + * SIGKILL because it is uncatchable. */ + __block_all_sigs(0); + __syscall(SYS_kill, __syscall(SYS_getpid), SIGKILL); + } + c->ret = ret; +} int setgroups(size_t count, const gid_t list[]) { - return syscall(SYS_setgroups, count, list); + /* ret is initially nonzero so that failure of the first thread does not + * trigger the safety kill above. */ + struct ctx c = { .count = count, .list = list, .ret = 1 }; + __synccall(do_setgroups, &c); + return __syscall_ret(c.ret); } diff --git a/src/locale/dcngettext.c b/src/locale/dcngettext.c index 4c3043938..d1e6c6d13 100644 --- a/src/locale/dcngettext.c +++ b/src/locale/dcngettext.c @@ -10,6 +10,12 @@ #include "atomic.h" #include "pleval.h" #include "lock.h" +#include "fork_impl.h" + +#define malloc __libc_malloc +#define calloc __libc_calloc +#define realloc undef +#define free undef struct binding { struct binding *next; @@ -34,9 +40,11 @@ static char *gettextdir(const char *domainname, size_t *dirlen) return 0; } +static volatile int lock[1]; +volatile int *const __gettext_lockptr = lock; + char *bindtextdomain(const char *domainname, const char *dirname) { - static volatile int lock[1]; struct binding *p, *q; if (!domainname) return 0; diff --git a/src/locale/freelocale.c b/src/locale/freelocale.c index 802b8bfe1..385d12069 100644 --- a/src/locale/freelocale.c +++ b/src/locale/freelocale.c @@ -1,6 +1,11 @@ #include #include "locale_impl.h" +#define malloc undef +#define calloc undef +#define realloc undef +#define free __libc_free + void freelocale(locale_t l) { if (__loc_is_allocated(l)) free(l); diff --git a/src/locale/locale_map.c b/src/locale/locale_map.c index 2321bac0e..da61f7fc0 100644 --- a/src/locale/locale_map.c +++ b/src/locale/locale_map.c @@ -1,9 +1,16 @@ #include #include #include +#include #include "locale_impl.h" #include "libc.h" #include "lock.h" +#include "fork_impl.h" + +#define malloc __libc_malloc +#define calloc undef +#define realloc undef +#define free undef const char *__lctrans_impl(const char *msg, const struct __locale_map *lm) { @@ -21,9 +28,11 @@ static const char envvars[][12] = { "LC_MESSAGES", }; +volatile int __locale_lock[1]; +volatile int *const __locale_lockptr = __locale_lock; + const struct __locale_map *__get_locale(int cat, const char *val) { - static volatile int lock[1]; static void *volatile loc_head; const struct __locale_map *p; struct __locale_map *new = 0; @@ -54,20 +63,12 @@ const struct __locale_map *__get_locale(int cat, const char *val) for (p=loc_head; p; p=p->next) if (!strcmp(val, p->name)) return p; - LOCK(lock); - - for (p=loc_head; p; p=p->next) - if (!strcmp(val, p->name)) { - UNLOCK(lock); - return p; - } - if (!libc.secure) path = getenv("MUSL_LOCPATH"); /* FIXME: add a default path? */ if (path) for (; *path; path=z+!!*z) { z = __strchrnul(path, ':'); - l = z - path - !!*z; + l = z - path; if (l >= sizeof buf - n - 2) continue; memcpy(buf, path, l); buf[l] = '/'; @@ -108,6 +109,5 @@ const struct __locale_map *__get_locale(int cat, const char *val) * requested name was "C" or "POSIX". */ if (!new && cat == LC_CTYPE) new = (void *)&__c_dot_utf8; - UNLOCK(lock); return new; } diff --git a/src/locale/newlocale.c b/src/locale/newlocale.c index d20a84898..9ac3cd386 100644 --- a/src/locale/newlocale.c +++ b/src/locale/newlocale.c @@ -2,16 +2,15 @@ #include #include #include "locale_impl.h" +#include "lock.h" -static pthread_once_t default_locale_once; -static struct __locale_struct default_locale, default_ctype_locale; +#define malloc __libc_malloc +#define calloc undef +#define realloc undef +#define free undef -static void default_locale_init(void) -{ - for (int i=0; i LC_ALL) return 0; - LOCK(lock); + LOCK(__locale_lock); /* For LC_ALL, setlocale is required to return a string which * encodes the current setting for all categories. The format of @@ -36,7 +35,7 @@ char *setlocale(int cat, const char *name) } lm = __get_locale(i, part); if (lm == LOC_MAP_FAILED) { - UNLOCK(lock); + UNLOCK(__locale_lock); return 0; } tmp_locale.cat[i] = lm; @@ -57,14 +56,14 @@ char *setlocale(int cat, const char *name) s += l+1; } *--s = 0; - UNLOCK(lock); + UNLOCK(__locale_lock); return same==LC_ALL ? (char *)part : buf; } if (name) { lm = __get_locale(cat, name); if (lm == LOC_MAP_FAILED) { - UNLOCK(lock); + UNLOCK(__locale_lock); return 0; } libc.global_locale.cat[cat] = lm; @@ -73,7 +72,7 @@ char *setlocale(int cat, const char *name) } char *ret = lm ? (char *)lm->name : "C"; - UNLOCK(lock); + UNLOCK(__locale_lock); return ret; } diff --git a/src/malloc/DESIGN b/src/malloc/DESIGN deleted file mode 100644 index 58b0523ff..000000000 --- a/src/malloc/DESIGN +++ /dev/null @@ -1,22 +0,0 @@ - - -In principle, this memory allocator is roughly equivalent to Doug -Lea's dlmalloc with fine-grained locking. - - - -malloc: - -Uses a freelist binned by chunk size, with a bitmap to optimize -searching for the smallest non-empty bin which can satisfy an -allocation. If no free chunks are available, it creates a new chunk of -the requested size and attempts to merge it with any existing free -chunk immediately below the newly created chunk. - -Whether the chunk was obtained from a bin or newly created, it's -likely to be larger than the requested allocation. malloc always -finishes its work by passing the new chunk to realloc, which will -split it into two chunks and free the tail portion. - - - diff --git a/src/malloc/aligned_alloc.c b/src/malloc/aligned_alloc.c deleted file mode 100644 index b6143f303..000000000 --- a/src/malloc/aligned_alloc.c +++ /dev/null @@ -1,7 +0,0 @@ -#include -#include "malloc_impl.h" - -void *aligned_alloc(size_t align, size_t len) -{ - return __memalign(align, len); -} diff --git a/src/malloc/calloc.c b/src/malloc/calloc.c new file mode 100644 index 000000000..bf6bddca3 --- /dev/null +++ b/src/malloc/calloc.c @@ -0,0 +1,45 @@ +#include +#include +#include +#include +#include "dynlink.h" + +static size_t mal0_clear(char *p, size_t n) +{ + const size_t pagesz = 4096; /* arbitrary */ + if (n < pagesz) return n; +#ifdef __GNUC__ + typedef uint64_t __attribute__((__may_alias__)) T; +#else + typedef unsigned char T; +#endif + char *pp = p + n; + size_t i = (uintptr_t)pp & (pagesz - 1); + for (;;) { + pp = memset(pp - i, 0, i); + if (pp - p < pagesz) return pp - p; + for (i = pagesz; i; i -= 2*sizeof(T), pp -= 2*sizeof(T)) + if (((T *)pp)[-1] | ((T *)pp)[-2]) + break; + } +} + +static int allzerop(void *p) +{ + return 0; +} +weak_alias(allzerop, __malloc_allzerop); + +void *calloc(size_t m, size_t n) +{ + if (n && m > (size_t)-1/n) { + errno = ENOMEM; + return 0; + } + n *= m; + void *p = malloc(n); + if (!p || (!__malloc_replaced && __malloc_allzerop(p))) + return p; + n = mal0_clear(p, n); + return memset(p, 0, n); +} diff --git a/src/malloc/expand_heap.c b/src/malloc/expand_heap.c deleted file mode 100644 index e6a3d7a00..000000000 --- a/src/malloc/expand_heap.c +++ /dev/null @@ -1,71 +0,0 @@ -#include -#include -#include -#include -#include "libc.h" -#include "syscall.h" -#include "malloc_impl.h" - -/* This function returns true if the interval [old,new] - * intersects the 'len'-sized interval below &libc.auxv - * (interpreted as the main-thread stack) or below &b - * (the current stack). It is used to defend against - * buggy brk implementations that can cross the stack. */ - -static int traverses_stack_p(uintptr_t old, uintptr_t new) -{ - const uintptr_t len = 8<<20; - uintptr_t a, b; - - b = (uintptr_t)libc.auxv; - a = b > len ? b-len : 0; - if (new>a && old len ? b-len : 0; - if (new>a && old SIZE_MAX/2 - PAGE_SIZE) { - errno = ENOMEM; - return 0; - } - n += -n & PAGE_SIZE-1; - - if (!brk) { - brk = __syscall(SYS_brk, 0); - brk += -brk & PAGE_SIZE-1; - } - - if (n < SIZE_MAX-brk && !traverses_stack_p(brk, brk+n) - && __syscall(SYS_brk, brk+n)==brk+n) { - *pn = n; - brk += n; - return (void *)(brk-n); - } - - size_t min = (size_t)PAGE_SIZE << mmap_step/2; - if (n < min) n = min; - void *area = __mmap(0, n, PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); - if (area == MAP_FAILED) return 0; - *pn = n; - mmap_step++; - return area; -} diff --git a/src/malloc/free.c b/src/malloc/free.c new file mode 100644 index 000000000..f17a952cb --- /dev/null +++ b/src/malloc/free.c @@ -0,0 +1,6 @@ +#include + +void free(void *p) +{ + return __libc_free(p); +} diff --git a/src/malloc/libc_calloc.c b/src/malloc/libc_calloc.c new file mode 100644 index 000000000..d25eabea4 --- /dev/null +++ b/src/malloc/libc_calloc.c @@ -0,0 +1,4 @@ +#define calloc __libc_calloc +#define malloc __libc_malloc + +#include "calloc.c" diff --git a/src/malloc/lite_malloc.c b/src/malloc/lite_malloc.c index 050d84f64..43a988fbb 100644 --- a/src/malloc/lite_malloc.c +++ b/src/malloc/lite_malloc.c @@ -2,58 +2,117 @@ #include #include #include +#include +#include "libc.h" #include "lock.h" -#include "malloc_impl.h" +#include "syscall.h" +#include "fork_impl.h" #define ALIGN 16 +/* This function returns true if the interval [old,new] + * intersects the 'len'-sized interval below &libc.auxv + * (interpreted as the main-thread stack) or below &b + * (the current stack). It is used to defend against + * buggy brk implementations that can cross the stack. */ + +static int traverses_stack_p(uintptr_t old, uintptr_t new) +{ + const uintptr_t len = 8<<20; + uintptr_t a, b; + + b = (uintptr_t)libc.auxv; + a = b > len ? b-len : 0; + if (new>a && old len ? b-len : 0; + if (new>a && old SIZE_MAX/2) { + errno = ENOMEM; + return 0; + } + if (!n) n++; while (align end-cur) { - size_t m = n; - char *new = __expand_heap(&m); - if (!new) { - UNLOCK(lock); - return 0; + size_t req = n - (end-cur) + PAGE_SIZE-1 & -PAGE_SIZE; + + if (!cur) { + brk = __syscall(SYS_brk, 0); + brk += -brk & PAGE_SIZE-1; + cur = end = brk; } - if (new != end) { - cur = new; - n -= pad; - pad = 0; + + if (brk == end && req < SIZE_MAX-brk + && !traverses_stack_p(brk, brk+req) + && __syscall(SYS_brk, brk+req)==brk+req) { + brk = end += req; + } else { + int new_area = 0; + req = n + PAGE_SIZE-1 & -PAGE_SIZE; + /* Only make a new area rather than individual mmap + * if wasted space would be over 1/8 of the map. */ + if (req-n > req/8) { + /* Geometric area size growth up to 64 pages, + * bounding waste by 1/8 of the area. */ + size_t min = PAGE_SIZE<<(mmap_step/2); + if (min-n > end-cur) { + if (req < min) { + req = min; + if (mmap_step < 12) + mmap_step++; + } + new_area = 1; + } + } + void *mem = __mmap(0, req, PROT_READ|PROT_WRITE, + MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); + if (mem == MAP_FAILED || !new_area) { + UNLOCK(lock); + return mem==MAP_FAILED ? 0 : mem; + } + cur = (uintptr_t)mem; + end = cur + req; } - end = new + m; } - p = cur + pad; + p = (void *)cur; cur += n; UNLOCK(lock); return p; } -weak_alias(__simple_malloc, malloc); +weak_alias(__simple_malloc, __libc_malloc_impl); -static void *__simple_calloc(size_t m, size_t n) +void *__libc_malloc(size_t n) { - if (n && m > (size_t)-1/n) { - errno = ENOMEM; - return 0; - } - return __simple_malloc(n * m); + return __libc_malloc_impl(n); +} + +static void *default_malloc(size_t n) +{ + return __libc_malloc_impl(n); } -weak_alias(__simple_calloc, calloc); +weak_alias(default_malloc, malloc); diff --git a/src/malloc/mallocng/aligned_alloc.c b/src/malloc/mallocng/aligned_alloc.c new file mode 100644 index 000000000..341168960 --- /dev/null +++ b/src/malloc/mallocng/aligned_alloc.c @@ -0,0 +1,57 @@ +#include +#include +#include "meta.h" + +void *aligned_alloc(size_t align, size_t len) +{ + if ((align & -align) != align) { + errno = EINVAL; + return 0; + } + + if (len > SIZE_MAX - align || align >= (1ULL<<31)*UNIT) { + errno = ENOMEM; + return 0; + } + + if (DISABLE_ALIGNED_ALLOC) { + errno = ENOMEM; + return 0; + } + + if (align <= UNIT) align = UNIT; + + unsigned char *p = malloc(len + align - UNIT); + struct meta *g = get_meta(p); + int idx = get_slot_index(p); + size_t stride = get_stride(g); + unsigned char *start = g->mem->storage + stride*idx; + unsigned char *end = g->mem->storage + stride*(idx+1) - IB; + size_t adj = -(uintptr_t)p & (align-1); + + if (!adj) { + set_size(p, end, len); + return p; + } + p += adj; + uint32_t offset = (size_t)(p-g->mem->storage)/UNIT; + if (offset <= 0xffff) { + *(uint16_t *)(p-2) = offset; + p[-4] = 0; + } else { + // use a 32-bit offset if 16-bit doesn't fit. for this, + // 16-bit field must be zero, [-4] byte nonzero. + *(uint16_t *)(p-2) = 0; + *(uint32_t *)(p-8) = offset; + p[-4] = 1; + } + p[-3] = idx; + set_size(p, end, len); + // store offset to aligned enframing. this facilitates cycling + // offset and also iteration of heap for debugging/measurement. + // for extreme overalignment it won't fit but these are classless + // allocations anyway. + *(uint16_t *)(start - 2) = (size_t)(p-start)/UNIT; + start[-3] = 7<<5; + return p; +} diff --git a/src/malloc/mallocng/donate.c b/src/malloc/mallocng/donate.c new file mode 100644 index 000000000..41d850f35 --- /dev/null +++ b/src/malloc/mallocng/donate.c @@ -0,0 +1,39 @@ +#include +#include +#include +#include +#include +#include + +#include "meta.h" + +static void donate(unsigned char *base, size_t len) +{ + uintptr_t a = (uintptr_t)base; + uintptr_t b = a + len; + a += -a & (UNIT-1); + b -= b & (UNIT-1); + memset(base, 0, len); + for (int sc=47; sc>0 && b>a; sc-=4) { + if (b-a < (size_classes[sc]+1)*UNIT) continue; + struct meta *m = alloc_meta(); + m->avail_mask = 0; + m->freed_mask = 1; + m->mem = (void *)a; + m->mem->meta = m; + m->last_idx = 0; + m->freeable = 0; + m->sizeclass = sc; + m->maplen = 0; + *((unsigned char *)m->mem+UNIT-4) = 0; + *((unsigned char *)m->mem+UNIT-3) = 255; + m->mem->storage[size_classes[sc]*UNIT-4] = 0; + queue(&ctx.active[sc], m); + a += (size_classes[sc]+1)*UNIT; + } +} + +void __malloc_donate(char *start, char *end) +{ + donate((void *)start, end-start); +} diff --git a/src/malloc/mallocng/free.c b/src/malloc/mallocng/free.c new file mode 100644 index 000000000..418a085c1 --- /dev/null +++ b/src/malloc/mallocng/free.c @@ -0,0 +1,151 @@ +#define _BSD_SOURCE +#include +#include + +#include "meta.h" + +struct mapinfo { + void *base; + size_t len; +}; + +static struct mapinfo nontrivial_free(struct meta *, int); + +static struct mapinfo free_group(struct meta *g) +{ + struct mapinfo mi = { 0 }; + int sc = g->sizeclass; + if (sc < 48) { + ctx.usage_by_class[sc] -= g->last_idx+1; + } + if (g->maplen) { + step_seq(); + record_seq(sc); + mi.base = g->mem; + mi.len = g->maplen*4096UL; + } else { + void *p = g->mem; + struct meta *m = get_meta(p); + int idx = get_slot_index(p); + g->mem->meta = 0; + // not checking size/reserved here; it's intentionally invalid + mi = nontrivial_free(m, idx); + } + free_meta(g); + return mi; +} + +static int okay_to_free(struct meta *g) +{ + int sc = g->sizeclass; + + if (!g->freeable) return 0; + + // always free individual mmaps not suitable for reuse + if (sc >= 48 || get_stride(g) < UNIT*size_classes[sc]) + return 1; + + // always free groups allocated inside another group's slot + // since recreating them should not be expensive and they + // might be blocking freeing of a much larger group. + if (!g->maplen) return 1; + + // if there is another non-full group, free this one to + // consolidate future allocations, reduce fragmentation. + if (g->next != g) return 1; + + // free any group in a size class that's not bouncing + if (!is_bouncing(sc)) return 1; + + size_t cnt = g->last_idx+1; + size_t usage = ctx.usage_by_class[sc]; + + // if usage is high enough that a larger count should be + // used, free the low-count group so a new one will be made. + if (9*cnt <= usage && cnt < 20) + return 1; + + // otherwise, keep the last group in a bouncing class. + return 0; +} + +static struct mapinfo nontrivial_free(struct meta *g, int i) +{ + uint32_t self = 1u<sizeclass; + uint32_t mask = g->freed_mask | g->avail_mask; + + if (mask+self == (2u<last_idx)-1 && okay_to_free(g)) { + // any multi-slot group is necessarily on an active list + // here, but single-slot groups might or might not be. + if (g->next) { + assert(sc < 48); + int activate_new = (ctx.active[sc]==g); + dequeue(&ctx.active[sc], g); + if (activate_new && ctx.active[sc]) + activate_group(ctx.active[sc]); + } + return free_group(g); + } else if (!mask) { + assert(sc < 48); + // might still be active if there were no allocations + // after last available slot was taken. + if (ctx.active[sc] != g) { + queue(&ctx.active[sc], g); + } + } + a_or(&g->freed_mask, self); + return (struct mapinfo){ 0 }; +} + +void free(void *p) +{ + if (!p) return; + + struct meta *g = get_meta(p); + int idx = get_slot_index(p); + size_t stride = get_stride(g); + unsigned char *start = g->mem->storage + stride*idx; + unsigned char *end = start + stride - IB; + get_nominal_size(p, end); + uint32_t self = 1u<last_idx)-1; + ((unsigned char *)p)[-3] = 255; + // invalidate offset to group header, and cycle offset of + // used region within slot if current offset is zero. + *(uint16_t *)((char *)p-2) = 0; + + // release any whole pages contained in the slot to be freed + // unless it's a single-slot group that will be unmapped. + if (((uintptr_t)(start-1) ^ (uintptr_t)end) >= 2*PGSZ && g->last_idx) { + unsigned char *base = start + (-(uintptr_t)start & (PGSZ-1)); + size_t len = (end-base) & -PGSZ; + if (len) { + int e = errno; + madvise(base, len, MADV_FREE); + errno = e; + } + } + + // atomic free without locking if this is neither first or last slot + for (;;) { + uint32_t freed = g->freed_mask; + uint32_t avail = g->avail_mask; + uint32_t mask = freed | avail; + assert(!(mask&self)); + if (!freed || mask+self==all) break; + if (!MT) + g->freed_mask = freed+self; + else if (a_cas(&g->freed_mask, freed, freed+self)!=freed) + continue; + return; + } + + wrlock(); + struct mapinfo mi = nontrivial_free(g, idx); + unlock(); + if (mi.len) { + int e = errno; + munmap(mi.base, mi.len); + errno = e; + } +} diff --git a/src/malloc/mallocng/glue.h b/src/malloc/mallocng/glue.h new file mode 100644 index 000000000..151c48b80 --- /dev/null +++ b/src/malloc/mallocng/glue.h @@ -0,0 +1,93 @@ +#ifndef MALLOC_GLUE_H +#define MALLOC_GLUE_H + +#include +#include +#include +#include +#include +#include +#include "atomic.h" +#include "syscall.h" +#include "libc.h" +#include "lock.h" +#include "dynlink.h" + +// use macros to appropriately namespace these. +#define size_classes __malloc_size_classes +#define ctx __malloc_context +#define alloc_meta __malloc_alloc_meta +#define is_allzero __malloc_allzerop +#define dump_heap __dump_heap + +#define malloc __libc_malloc_impl +#define realloc __libc_realloc +#define free __libc_free + +#if USE_REAL_ASSERT +#include +#else +#undef assert +#define assert(x) do { if (!(x)) a_crash(); } while(0) +#endif + +#define brk(p) ((uintptr_t)__syscall(SYS_brk, p)) + +#define mmap __mmap +#define madvise __madvise +#define mremap __mremap + +#define DISABLE_ALIGNED_ALLOC (__malloc_replaced && !__aligned_alloc_replaced) + +static inline uint64_t get_random_secret() +{ + uint64_t secret = (uintptr_t)&secret * 1103515245; + for (size_t i=0; libc.auxv[i]; i+=2) + if (libc.auxv[i]==AT_RANDOM) + memcpy(&secret, (char *)libc.auxv[i+1]+8, sizeof secret); + return secret; +} + +#ifndef PAGESIZE +#define PAGESIZE PAGE_SIZE +#endif + +#define MT (libc.need_locks) + +#define RDLOCK_IS_EXCLUSIVE 1 + +__attribute__((__visibility__("hidden"))) +extern int __malloc_lock[1]; + +#define LOCK_OBJ_DEF \ +int __malloc_lock[1]; \ +void __malloc_atfork(int who) { malloc_atfork(who); } + +static inline void rdlock() +{ + if (MT) LOCK(__malloc_lock); +} +static inline void wrlock() +{ + if (MT) LOCK(__malloc_lock); +} +static inline void unlock() +{ + UNLOCK(__malloc_lock); +} +static inline void upgradelock() +{ +} +static inline void resetlock() +{ + __malloc_lock[0] = 0; +} + +static inline void malloc_atfork(int who) +{ + if (who<0) rdlock(); + else if (who>0) resetlock(); + else unlock(); +} + +#endif diff --git a/src/malloc/mallocng/malloc.c b/src/malloc/mallocng/malloc.c new file mode 100644 index 000000000..d695ab8ec --- /dev/null +++ b/src/malloc/mallocng/malloc.c @@ -0,0 +1,387 @@ +#include +#include +#include +#include +#include +#include + +#include "meta.h" + +LOCK_OBJ_DEF; + +const uint16_t size_classes[] = { + 1, 2, 3, 4, 5, 6, 7, 8, + 9, 10, 12, 15, + 18, 20, 25, 31, + 36, 42, 50, 63, + 72, 84, 102, 127, + 146, 170, 204, 255, + 292, 340, 409, 511, + 584, 682, 818, 1023, + 1169, 1364, 1637, 2047, + 2340, 2730, 3276, 4095, + 4680, 5460, 6552, 8191, +}; + +static const uint8_t small_cnt_tab[][3] = { + { 30, 30, 30 }, + { 31, 15, 15 }, + { 20, 10, 10 }, + { 31, 15, 7 }, + { 25, 12, 6 }, + { 21, 10, 5 }, + { 18, 8, 4 }, + { 31, 15, 7 }, + { 28, 14, 6 }, +}; + +static const uint8_t med_cnt_tab[4] = { 28, 24, 20, 32 }; + +struct malloc_context ctx = { 0 }; + +struct meta *alloc_meta(void) +{ + struct meta *m; + unsigned char *p; + if (!ctx.init_done) { +#ifndef PAGESIZE + ctx.pagesize = get_page_size(); +#endif + ctx.secret = get_random_secret(); + ctx.init_done = 1; + } + size_t pagesize = PGSZ; + if (pagesize < 4096) pagesize = 4096; + if ((m = dequeue_head(&ctx.free_meta_head))) return m; + if (!ctx.avail_meta_count) { + int need_unprotect = 1; + if (!ctx.avail_meta_area_count && ctx.brk!=-1) { + uintptr_t new = ctx.brk + pagesize; + int need_guard = 0; + if (!ctx.brk) { + need_guard = 1; + ctx.brk = brk(0); + // some ancient kernels returned _ebss + // instead of next page as initial brk. + ctx.brk += -ctx.brk & (pagesize-1); + new = ctx.brk + 2*pagesize; + } + if (brk(new) != new) { + ctx.brk = -1; + } else { + if (need_guard) mmap((void *)ctx.brk, pagesize, + PROT_NONE, MAP_ANON|MAP_PRIVATE|MAP_FIXED, -1, 0); + ctx.brk = new; + ctx.avail_meta_areas = (void *)(new - pagesize); + ctx.avail_meta_area_count = pagesize>>12; + need_unprotect = 0; + } + } + if (!ctx.avail_meta_area_count) { + size_t n = 2UL << ctx.meta_alloc_shift; + p = mmap(0, n*pagesize, PROT_NONE, + MAP_PRIVATE|MAP_ANON, -1, 0); + if (p==MAP_FAILED) return 0; + ctx.avail_meta_areas = p + pagesize; + ctx.avail_meta_area_count = (n-1)*(pagesize>>12); + ctx.meta_alloc_shift++; + } + p = ctx.avail_meta_areas; + if ((uintptr_t)p & (pagesize-1)) need_unprotect = 0; + if (need_unprotect) + if (mprotect(p, pagesize, PROT_READ|PROT_WRITE) + && errno != ENOSYS) + return 0; + ctx.avail_meta_area_count--; + ctx.avail_meta_areas = p + 4096; + if (ctx.meta_area_tail) { + ctx.meta_area_tail->next = (void *)p; + } else { + ctx.meta_area_head = (void *)p; + } + ctx.meta_area_tail = (void *)p; + ctx.meta_area_tail->check = ctx.secret; + ctx.avail_meta_count = ctx.meta_area_tail->nslots + = (4096-sizeof(struct meta_area))/sizeof *m; + ctx.avail_meta = ctx.meta_area_tail->slots; + } + ctx.avail_meta_count--; + m = ctx.avail_meta++; + m->prev = m->next = 0; + return m; +} + +static uint32_t try_avail(struct meta **pm) +{ + struct meta *m = *pm; + uint32_t first; + if (!m) return 0; + uint32_t mask = m->avail_mask; + if (!mask) { + if (!m) return 0; + if (!m->freed_mask) { + dequeue(pm, m); + m = *pm; + if (!m) return 0; + } else { + m = m->next; + *pm = m; + } + + mask = m->freed_mask; + + // skip fully-free group unless it's the only one + // or it's a permanently non-freeable group + if (mask == (2u<last_idx)-1 && m->freeable) { + m = m->next; + *pm = m; + mask = m->freed_mask; + } + + // activate more slots in a not-fully-active group + // if needed, but only as a last resort. prefer using + // any other group with free slots. this avoids + // touching & dirtying as-yet-unused pages. + if (!(mask & ((2u<mem->active_idx)-1))) { + if (m->next != m) { + m = m->next; + *pm = m; + } else { + int cnt = m->mem->active_idx + 2; + int size = size_classes[m->sizeclass]*UNIT; + int span = UNIT + size*cnt; + // activate up to next 4k boundary + while ((span^(span+size-1)) < 4096) { + cnt++; + span += size; + } + if (cnt > m->last_idx+1) + cnt = m->last_idx+1; + m->mem->active_idx = cnt-1; + } + } + mask = activate_group(m); + assert(mask); + decay_bounces(m->sizeclass); + } + first = mask&-mask; + m->avail_mask = mask-first; + return first; +} + +static int alloc_slot(int, size_t); + +static struct meta *alloc_group(int sc, size_t req) +{ + size_t size = UNIT*size_classes[sc]; + int i = 0, cnt; + unsigned char *p; + struct meta *m = alloc_meta(); + if (!m) return 0; + size_t usage = ctx.usage_by_class[sc]; + size_t pagesize = PGSZ; + int active_idx; + if (sc < 9) { + while (i<2 && 4*small_cnt_tab[sc][i] > usage) + i++; + cnt = small_cnt_tab[sc][i]; + } else { + // lookup max number of slots fitting in power-of-two size + // from a table, along with number of factors of two we + // can divide out without a remainder or reaching 1. + cnt = med_cnt_tab[sc&3]; + + // reduce cnt to avoid excessive eagar allocation. + while (!(cnt&1) && 4*cnt > usage) + cnt >>= 1; + + // data structures don't support groups whose slot offsets + // in units don't fit in 16 bits. + while (size*cnt >= 65536*UNIT) + cnt >>= 1; + } + + // If we selected a count of 1 above but it's not sufficient to use + // mmap, increase to 2. Then it might be; if not it will nest. + if (cnt==1 && size*cnt+UNIT <= pagesize/2) cnt = 2; + + // All choices of size*cnt are "just below" a power of two, so anything + // larger than half the page size should be allocated as whole pages. + if (size*cnt+UNIT > pagesize/2) { + // check/update bounce counter to start/increase retention + // of freed maps, and inhibit use of low-count, odd-size + // small mappings and single-slot groups if activated. + int nosmall = is_bouncing(sc); + account_bounce(sc); + step_seq(); + + // since the following count reduction opportunities have + // an absolute memory usage cost, don't overdo them. count + // coarse usage as part of usage. + if (!(sc&1) && sc<32) usage += ctx.usage_by_class[sc+1]; + + // try to drop to a lower count if the one found above + // increases usage by more than 25%. these reduced counts + // roughly fill an integral number of pages, just not a + // power of two, limiting amount of unusable space. + if (4*cnt > usage && !nosmall) { + if (0); + else if ((sc&3)==1 && size*cnt>8*pagesize) cnt = 2; + else if ((sc&3)==2 && size*cnt>4*pagesize) cnt = 3; + else if ((sc&3)==0 && size*cnt>8*pagesize) cnt = 3; + else if ((sc&3)==0 && size*cnt>2*pagesize) cnt = 5; + } + size_t needed = size*cnt + UNIT; + needed += -needed & (pagesize-1); + + // produce an individually-mmapped allocation if usage is low, + // bounce counter hasn't triggered, and either it saves memory + // or it avoids eagar slot allocation without wasting too much. + if (!nosmall && cnt<=7) { + req += IB + UNIT; + req += -req & (pagesize-1); + if (req=4*pagesize && 2*cnt>usage)) { + cnt = 1; + needed = req; + } + } + + p = mmap(0, needed, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANON, -1, 0); + if (p==MAP_FAILED) { + free_meta(m); + return 0; + } + m->maplen = needed>>12; + ctx.mmap_counter++; + active_idx = (4096-UNIT)/size-1; + if (active_idx > cnt-1) active_idx = cnt-1; + if (active_idx < 0) active_idx = 0; + } else { + int j = size_to_class(UNIT+cnt*size-IB); + int idx = alloc_slot(j, UNIT+cnt*size-IB); + if (idx < 0) { + free_meta(m); + return 0; + } + struct meta *g = ctx.active[j]; + p = enframe(g, idx, UNIT*size_classes[j]-IB, ctx.mmap_counter); + m->maplen = 0; + p[-3] = (p[-3]&31) | (6<<5); + for (int i=0; i<=cnt; i++) + p[UNIT+i*size-4] = 0; + active_idx = cnt-1; + } + ctx.usage_by_class[sc] += cnt; + m->avail_mask = (2u<freed_mask = (2u<<(cnt-1))-1 - m->avail_mask; + m->mem = (void *)p; + m->mem->meta = m; + m->mem->active_idx = active_idx; + m->last_idx = cnt-1; + m->freeable = 1; + m->sizeclass = sc; + return m; +} + +static int alloc_slot(int sc, size_t req) +{ + uint32_t first = try_avail(&ctx.active[sc]); + if (first) return a_ctz_32(first); + + struct meta *g = alloc_group(sc, req); + if (!g) return -1; + + g->avail_mask--; + queue(&ctx.active[sc], g); + return 0; +} + +void *malloc(size_t n) +{ + if (size_overflows(n)) return 0; + struct meta *g; + uint32_t mask, first; + int sc; + int idx; + int ctr; + + if (n >= MMAP_THRESHOLD) { + size_t needed = n + IB + UNIT; + void *p = mmap(0, needed, PROT_READ|PROT_WRITE, + MAP_PRIVATE|MAP_ANON, -1, 0); + if (p==MAP_FAILED) return 0; + wrlock(); + step_seq(); + g = alloc_meta(); + if (!g) { + unlock(); + munmap(p, needed); + return 0; + } + g->mem = p; + g->mem->meta = g; + g->last_idx = 0; + g->freeable = 1; + g->sizeclass = 63; + g->maplen = (needed+4095)/4096; + g->avail_mask = g->freed_mask = 0; + // use a global counter to cycle offset in + // individually-mmapped allocations. + ctx.mmap_counter++; + idx = 0; + goto success; + } + + sc = size_to_class(n); + + rdlock(); + g = ctx.active[sc]; + + // use coarse size classes initially when there are not yet + // any groups of desired size. this allows counts of 2 or 3 + // to be allocated at first rather than having to start with + // 7 or 5, the min counts for even size classes. + if (!g && sc>=4 && sc<32 && sc!=6 && !(sc&1) && !ctx.usage_by_class[sc]) { + size_t usage = ctx.usage_by_class[sc|1]; + // if a new group may be allocated, count it toward + // usage in deciding if we can use coarse class. + if (!ctx.active[sc|1] || (!ctx.active[sc|1]->avail_mask + && !ctx.active[sc|1]->freed_mask)) + usage += 3; + if (usage <= 12) + sc |= 1; + g = ctx.active[sc]; + } + + for (;;) { + mask = g ? g->avail_mask : 0; + first = mask&-mask; + if (!first) break; + if (RDLOCK_IS_EXCLUSIVE || !MT) + g->avail_mask = mask-first; + else if (a_cas(&g->avail_mask, mask, mask-first)!=mask) + continue; + idx = a_ctz_32(first); + goto success; + } + upgradelock(); + + idx = alloc_slot(sc, n); + if (idx < 0) { + unlock(); + return 0; + } + g = ctx.active[sc]; + +success: + ctr = ctx.mmap_counter; + unlock(); + return enframe(g, idx, n, ctr); +} + +int is_allzero(void *p) +{ + struct meta *g = get_meta(p); + return g->sizeclass >= 48 || + get_stride(g) < UNIT*size_classes[g->sizeclass]; +} diff --git a/src/malloc/mallocng/malloc_usable_size.c b/src/malloc/mallocng/malloc_usable_size.c new file mode 100644 index 000000000..ce6a960c6 --- /dev/null +++ b/src/malloc/mallocng/malloc_usable_size.c @@ -0,0 +1,13 @@ +#include +#include "meta.h" + +size_t malloc_usable_size(void *p) +{ + if (!p) return 0; + struct meta *g = get_meta(p); + int idx = get_slot_index(p); + size_t stride = get_stride(g); + unsigned char *start = g->mem->storage + stride*idx; + unsigned char *end = start + stride - IB; + return get_nominal_size(p, end); +} diff --git a/src/malloc/mallocng/meta.h b/src/malloc/mallocng/meta.h new file mode 100644 index 000000000..61ec53f9a --- /dev/null +++ b/src/malloc/mallocng/meta.h @@ -0,0 +1,288 @@ +#ifndef MALLOC_META_H +#define MALLOC_META_H + +#include +#include +#include +#include "glue.h" + +__attribute__((__visibility__("hidden"))) +extern const uint16_t size_classes[]; + +#define MMAP_THRESHOLD 131052 + +#define UNIT 16 +#define IB 4 + +struct group { + struct meta *meta; + unsigned char active_idx:5; + char pad[UNIT - sizeof(struct meta *) - 1]; + unsigned char storage[]; +}; + +struct meta { + struct meta *prev, *next; + struct group *mem; + volatile int avail_mask, freed_mask; + uintptr_t last_idx:5; + uintptr_t freeable:1; + uintptr_t sizeclass:6; + uintptr_t maplen:8*sizeof(uintptr_t)-12; +}; + +struct meta_area { + uint64_t check; + struct meta_area *next; + int nslots; + struct meta slots[]; +}; + +struct malloc_context { + uint64_t secret; +#ifndef PAGESIZE + size_t pagesize; +#endif + int init_done; + unsigned mmap_counter; + struct meta *free_meta_head; + struct meta *avail_meta; + size_t avail_meta_count, avail_meta_area_count, meta_alloc_shift; + struct meta_area *meta_area_head, *meta_area_tail; + unsigned char *avail_meta_areas; + struct meta *active[48]; + size_t usage_by_class[48]; + uint8_t unmap_seq[32], bounces[32]; + uint8_t seq; + uintptr_t brk; +}; + +__attribute__((__visibility__("hidden"))) +extern struct malloc_context ctx; + +#ifdef PAGESIZE +#define PGSZ PAGESIZE +#else +#define PGSZ ctx.pagesize +#endif + +__attribute__((__visibility__("hidden"))) +struct meta *alloc_meta(void); + +__attribute__((__visibility__("hidden"))) +int is_allzero(void *); + +static inline void queue(struct meta **phead, struct meta *m) +{ + assert(!m->next); + assert(!m->prev); + if (*phead) { + struct meta *head = *phead; + m->next = head; + m->prev = head->prev; + m->next->prev = m->prev->next = m; + } else { + m->prev = m->next = m; + *phead = m; + } +} + +static inline void dequeue(struct meta **phead, struct meta *m) +{ + if (m->next != m) { + m->prev->next = m->next; + m->next->prev = m->prev; + if (*phead == m) *phead = m->next; + } else { + *phead = 0; + } + m->prev = m->next = 0; +} + +static inline struct meta *dequeue_head(struct meta **phead) +{ + struct meta *m = *phead; + if (m) dequeue(phead, m); + return m; +} + +static inline void free_meta(struct meta *m) +{ + *m = (struct meta){0}; + queue(&ctx.free_meta_head, m); +} + +static inline uint32_t activate_group(struct meta *m) +{ + assert(!m->avail_mask); + uint32_t mask, act = (2u<mem->active_idx)-1; + do mask = m->freed_mask; + while (a_cas(&m->freed_mask, mask, mask&~act)!=mask); + return m->avail_mask = mask & act; +} + +static inline int get_slot_index(const unsigned char *p) +{ + return p[-3] & 31; +} + +static inline struct meta *get_meta(const unsigned char *p) +{ + assert(!((uintptr_t)p & 15)); + int offset = *(const uint16_t *)(p - 2); + int index = get_slot_index(p); + if (p[-4]) { + assert(!offset); + offset = *(uint32_t *)(p - 8); + assert(offset > 0xffff); + } + const struct group *base = (const void *)(p - UNIT*offset - UNIT); + const struct meta *meta = base->meta; + assert(meta->mem == base); + assert(index <= meta->last_idx); + assert(!(meta->avail_mask & (1u<freed_mask & (1u<check == ctx.secret); + if (meta->sizeclass < 48) { + assert(offset >= size_classes[meta->sizeclass]*index); + assert(offset < size_classes[meta->sizeclass]*(index+1)); + } else { + assert(meta->sizeclass == 63); + } + if (meta->maplen) { + assert(offset <= meta->maplen*4096UL/UNIT - 1); + } + return (struct meta *)meta; +} + +static inline size_t get_nominal_size(const unsigned char *p, const unsigned char *end) +{ + size_t reserved = p[-3] >> 5; + if (reserved >= 5) { + assert(reserved == 5); + reserved = *(const uint32_t *)(end-4); + assert(reserved >= 5); + assert(!end[-5]); + } + assert(reserved <= end-p); + assert(!*(end-reserved)); + // also check the slot's overflow byte + assert(!*end); + return end-reserved-p; +} + +static inline size_t get_stride(const struct meta *g) +{ + if (!g->last_idx && g->maplen) { + return g->maplen*4096UL - UNIT; + } else { + return UNIT*size_classes[g->sizeclass]; + } +} + +static inline void set_size(unsigned char *p, unsigned char *end, size_t n) +{ + int reserved = end-p-n; + if (reserved) end[-reserved] = 0; + if (reserved >= 5) { + *(uint32_t *)(end-4) = reserved; + end[-5] = 0; + reserved = 5; + } + p[-3] = (p[-3]&31) + (reserved<<5); +} + +static inline void *enframe(struct meta *g, int idx, size_t n, int ctr) +{ + size_t stride = get_stride(g); + size_t slack = (stride-IB-n)/UNIT; + unsigned char *p = g->mem->storage + stride*idx; + unsigned char *end = p+stride-IB; + // cycle offset within slot to increase interval to address + // reuse, facilitate trapping double-free. + int off = (p[-3] ? *(uint16_t *)(p-2) + 1 : ctr) & 255; + assert(!p[-4]); + if (off > slack) { + size_t m = slack; + m |= m>>1; m |= m>>2; m |= m>>4; + off &= m; + if (off > slack) off -= slack+1; + assert(off <= slack); + } + if (off) { + // store offset in unused header at offset zero + // if enframing at non-zero offset. + *(uint16_t *)(p-2) = off; + p[-3] = 7<<5; + p += UNIT*off; + // for nonzero offset there is no permanent check + // byte, so make one. + p[-4] = 0; + } + *(uint16_t *)(p-2) = (size_t)(p-g->mem->storage)/UNIT; + p[-3] = idx; + set_size(p, end, n); + return p; +} + +static inline int size_to_class(size_t n) +{ + n = (n+IB-1)>>4; + if (n<10) return n; + n++; + int i = (28-a_clz_32(n))*4 + 8; + if (n>size_classes[i+1]) i+=2; + if (n>size_classes[i]) i++; + return i; +} + +static inline int size_overflows(size_t n) +{ + if (n >= SIZE_MAX/2 - 4096) { + errno = ENOMEM; + return 1; + } + return 0; +} + +static inline void step_seq(void) +{ + if (ctx.seq==255) { + for (int i=0; i<32; i++) ctx.unmap_seq[i] = 0; + ctx.seq = 1; + } else { + ctx.seq++; + } +} + +static inline void record_seq(int sc) +{ + if (sc-7U < 32) ctx.unmap_seq[sc-7] = ctx.seq; +} + +static inline void account_bounce(int sc) +{ + if (sc-7U < 32) { + int seq = ctx.unmap_seq[sc-7]; + if (seq && ctx.seq-seq < 10) { + if (ctx.bounces[sc-7]+1 < 100) + ctx.bounces[sc-7]++; + else + ctx.bounces[sc-7] = 150; + } + } +} + +static inline void decay_bounces(int sc) +{ + if (sc-7U < 32 && ctx.bounces[sc-7]) + ctx.bounces[sc-7]--; +} + +static inline int is_bouncing(int sc) +{ + return (sc-7U < 32 && ctx.bounces[sc-7] >= 100); +} + +#endif diff --git a/src/malloc/mallocng/realloc.c b/src/malloc/mallocng/realloc.c new file mode 100644 index 000000000..18769f42d --- /dev/null +++ b/src/malloc/mallocng/realloc.c @@ -0,0 +1,51 @@ +#define _GNU_SOURCE +#include +#include +#include +#include "meta.h" + +void *realloc(void *p, size_t n) +{ + if (!p) return malloc(n); + if (size_overflows(n)) return 0; + + struct meta *g = get_meta(p); + int idx = get_slot_index(p); + size_t stride = get_stride(g); + unsigned char *start = g->mem->storage + stride*idx; + unsigned char *end = start + stride - IB; + size_t old_size = get_nominal_size(p, end); + size_t avail_size = end-(unsigned char *)p; + void *new; + + // only resize in-place if size class matches + if (n <= avail_size && n= g->sizeclass) { + set_size(p, end, n); + return p; + } + + // use mremap if old and new size are both mmap-worthy + if (g->sizeclass>=48 && n>=MMAP_THRESHOLD) { + assert(g->sizeclass==63); + size_t base = (unsigned char *)p-start; + size_t needed = (n + base + UNIT + IB + 4095) & -4096; + new = g->maplen*4096UL == needed ? g->mem : + mremap(g->mem, g->maplen*4096UL, needed, MREMAP_MAYMOVE); + if (new!=MAP_FAILED) { + g->mem = new; + g->maplen = needed/4096; + p = g->mem->storage + base; + end = g->mem->storage + (needed - UNIT) - IB; + *end = 0; + set_size(p, end, n); + return p; + } + } + + new = malloc(n); + if (!new) return 0; + memcpy(new, p, n < old_size ? n : old_size); + free(p); + return new; +} diff --git a/src/malloc/memalign.c b/src/malloc/memalign.c index cf9dfbda6..32cd87d81 100644 --- a/src/malloc/memalign.c +++ b/src/malloc/memalign.c @@ -1,54 +1,7 @@ +#define _BSD_SOURCE #include -#include -#include -#include "malloc_impl.h" -void *__memalign(size_t align, size_t len) +void *memalign(size_t align, size_t len) { - unsigned char *mem, *new; - - if ((align & -align) != align) { - errno = EINVAL; - return 0; - } - - if (len > SIZE_MAX - align || __malloc_replaced) { - errno = ENOMEM; - return 0; - } - - if (align <= SIZE_ALIGN) - return malloc(len); - - if (!(mem = malloc(len + align-1))) - return 0; - - new = (void *)((uintptr_t)mem + align-1 & -align); - if (new == mem) return mem; - - struct chunk *c = MEM_TO_CHUNK(mem); - struct chunk *n = MEM_TO_CHUNK(new); - - if (IS_MMAPPED(c)) { - /* Apply difference between aligned and original - * address to the "extra" field of mmapped chunk. */ - n->psize = c->psize + (new-mem); - n->csize = c->csize - (new-mem); - return new; - } - - struct chunk *t = NEXT_CHUNK(c); - - /* Split the allocated chunk into two chunks. The aligned part - * that will be used has the size in its footer reduced by the - * difference between the aligned and original addresses, and - * the resulting size copied to its header. A new header and - * footer are written for the split-off part to be freed. */ - n->psize = c->csize = C_INUSE | (new-mem); - n->csize = t->psize -= new-mem; - - __bin_chunk(c); - return new; + return aligned_alloc(align, len); } - -weak_alias(__memalign, memalign); diff --git a/src/malloc/oldmalloc/aligned_alloc.c b/src/malloc/oldmalloc/aligned_alloc.c new file mode 100644 index 000000000..4adca3b4f --- /dev/null +++ b/src/malloc/oldmalloc/aligned_alloc.c @@ -0,0 +1,53 @@ +#include +#include +#include +#include "malloc_impl.h" + +void *aligned_alloc(size_t align, size_t len) +{ + unsigned char *mem, *new; + + if ((align & -align) != align) { + errno = EINVAL; + return 0; + } + + if (len > SIZE_MAX - align || + (__malloc_replaced && !__aligned_alloc_replaced)) { + errno = ENOMEM; + return 0; + } + + if (align <= SIZE_ALIGN) + return malloc(len); + + if (!(mem = malloc(len + align-1))) + return 0; + + new = (void *)((uintptr_t)mem + align-1 & -align); + if (new == mem) return mem; + + struct chunk *c = MEM_TO_CHUNK(mem); + struct chunk *n = MEM_TO_CHUNK(new); + + if (IS_MMAPPED(c)) { + /* Apply difference between aligned and original + * address to the "extra" field of mmapped chunk. */ + n->psize = c->psize + (new-mem); + n->csize = c->csize - (new-mem); + return new; + } + + struct chunk *t = NEXT_CHUNK(c); + + /* Split the allocated chunk into two chunks. The aligned part + * that will be used has the size in its footer reduced by the + * difference between the aligned and original addresses, and + * the resulting size copied to its header. A new header and + * footer are written for the split-off part to be freed. */ + n->psize = c->csize = C_INUSE | (new-mem); + n->csize = t->psize -= new-mem; + + __bin_chunk(c); + return new; +} diff --git a/src/malloc/malloc.c b/src/malloc/oldmalloc/malloc.c similarity index 64% rename from src/malloc/malloc.c rename to src/malloc/oldmalloc/malloc.c index 96982596b..25d00d44d 100644 --- a/src/malloc/malloc.c +++ b/src/malloc/oldmalloc/malloc.c @@ -9,6 +9,11 @@ #include "atomic.h" #include "pthread_impl.h" #include "malloc_impl.h" +#include "fork_impl.h" + +#define malloc __libc_malloc_impl +#define realloc __libc_realloc +#define free __libc_free #if defined(__GNUC__) && defined(__PIC__) #define inline inline __attribute__((always_inline)) @@ -17,17 +22,18 @@ static struct { volatile uint64_t binmap; struct bin bins[64]; - volatile int free_lock[2]; + volatile int split_merge_lock[2]; } mal; -int __malloc_replaced; - /* Synchronization tools */ static inline void lock(volatile int *lk) { - if (libc.threads_minus_1) + int need_locks = libc.need_locks; + if (need_locks) { while(a_swap(lk, 1)) __wait(lk, lk+1, 1, 1); + if (need_locks < 0) libc.need_locks = 0; + } } static inline void unlock(volatile int *lk) @@ -123,9 +129,72 @@ void __dump_heap(int x) } #endif +/* This function returns true if the interval [old,new] + * intersects the 'len'-sized interval below &libc.auxv + * (interpreted as the main-thread stack) or below &b + * (the current stack). It is used to defend against + * buggy brk implementations that can cross the stack. */ + +static int traverses_stack_p(uintptr_t old, uintptr_t new) +{ + const uintptr_t len = 8<<20; + uintptr_t a, b; + + b = (uintptr_t)libc.auxv; + a = b > len ? b-len : 0; + if (new>a && old len ? b-len : 0; + if (new>a && old SIZE_MAX/2 - PAGE_SIZE) { + errno = ENOMEM; + return 0; + } + n += -n & PAGE_SIZE-1; + + if (!brk) { + brk = __syscall(SYS_brk, 0); + brk += -brk & PAGE_SIZE-1; + } + + if (n < SIZE_MAX-brk && !traverses_stack_p(brk, brk+n) + && __syscall(SYS_brk, brk+n)==brk+n) { + *pn = n; + brk += n; + return (void *)(brk-n); + } + + size_t min = (size_t)PAGE_SIZE << mmap_step/2; + if (n < min) n = min; + void *area = __mmap(0, n, PROT_READ|PROT_WRITE, + MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); + if (area == MAP_FAILED) return 0; + *pn = n; + mmap_step++; + return area; +} + static struct chunk *expand_heap(size_t n) { - static int heap_lock[2]; static void *end; void *p; struct chunk *w; @@ -135,13 +204,8 @@ static struct chunk *expand_heap(size_t n) * we need room for an extra zero-sized sentinel chunk. */ n += SIZE_ALIGN; - lock(heap_lock); - p = __expand_heap(&n); - if (!p) { - unlock(heap_lock); - return 0; - } + if (!p) return 0; /* If not just expanding existing space, we need to make a * new sentinel chunk below the allocated space. */ @@ -164,8 +228,6 @@ static struct chunk *expand_heap(size_t n) w = MEM_TO_CHUNK(p); w->csize = n | C_INUSE; - unlock(heap_lock); - return w; } @@ -195,96 +257,44 @@ static void unbin(struct chunk *c, int i) NEXT_CHUNK(c)->psize |= C_INUSE; } -static int alloc_fwd(struct chunk *c) +static void bin_chunk(struct chunk *self, int i) { - int i; - size_t k; - while (!((k=c->csize) & C_INUSE)) { - i = bin_index(k); - lock_bin(i); - if (c->csize == k) { - unbin(c, i); - unlock_bin(i); - return 1; - } - unlock_bin(i); - } - return 0; -} - -static int alloc_rev(struct chunk *c) -{ - int i; - size_t k; - while (!((k=c->psize) & C_INUSE)) { - i = bin_index(k); - lock_bin(i); - if (c->psize == k) { - unbin(PREV_CHUNK(c), i); - unlock_bin(i); - return 1; - } - unlock_bin(i); - } - return 0; + self->next = BIN_TO_CHUNK(i); + self->prev = mal.bins[i].tail; + self->next->prev = self; + self->prev->next = self; + if (self->prev == BIN_TO_CHUNK(i)) + a_or_64(&mal.binmap, 1ULL<= n1 - DONTCARE) return; next = NEXT_CHUNK(self); split = (void *)((char *)self + n); - split->prev = self->prev; - split->next = self->next; - split->prev->next = split; - split->next->prev = split; split->psize = n | C_INUSE; split->csize = n1-n; next->psize = n1-n; self->csize = n | C_INUSE; - return 1; -} -static void trim(struct chunk *self, size_t n) -{ - size_t n1 = CHUNK_SIZE(self); - struct chunk *next, *split; + int i = bin_index(n1-n); + lock_bin(i); - if (n >= n1 - DONTCARE) return; + bin_chunk(split, i); - next = NEXT_CHUNK(self); - split = (void *)((char *)self + n); - - split->psize = n | C_INUSE; - split->csize = n1-n | C_INUSE; - next->psize = n1-n | C_INUSE; - self->csize = n | C_INUSE; - - __bin_chunk(split); + unlock_bin(i); } void *malloc(size_t n) { struct chunk *c; int i, j; + uint64_t mask; if (adjust_size(&n) < 0) return 0; @@ -300,70 +310,43 @@ void *malloc(size_t n) } i = bin_index_up(n); - for (;;) { - uint64_t mask = mal.binmap & -(1ULL<psize = c->csize = - x->csize + CHUNK_SIZE(c); - } - break; + if (i<63 && (mal.binmap & (1ULL< (size_t)-1/n) { - errno = ENOMEM; - return 0; - } - n *= m; - void *p = malloc(n); - if (!p) return p; - if (!__malloc_replaced) { - if (IS_MMAPPED(MEM_TO_CHUNK(p))) - return p; - if (n >= PAGE_SIZE) - n = mal0_clear(p, PAGE_SIZE, n); - } - return memset(p, 0, n); + return IS_MMAPPED(MEM_TO_CHUNK(p)); } void *realloc(void *p, size_t n) @@ -379,6 +362,8 @@ void *realloc(void *p, size_t n) self = MEM_TO_CHUNK(p); n1 = n0 = CHUNK_SIZE(self); + if (n<=n0 && n0-n<=DONTCARE) return p; + if (IS_MMAPPED(self)) { size_t extra = self->psize; char *base = (char *)self - extra; @@ -405,34 +390,43 @@ void *realloc(void *p, size_t n) /* Crash on corrupted footer (likely from buffer overflow) */ if (next->psize != self->csize) a_crash(); - /* Merge adjacent chunks if we need more space. This is not - * a waste of time even if we fail to get enough space, because our - * subsequent call to free would otherwise have to do the merge. */ - if (n > n1 && alloc_fwd(next)) { - n1 += CHUNK_SIZE(next); - next = NEXT_CHUNK(next); - } - /* FIXME: find what's wrong here and reenable it..? */ - if (0 && n > n1 && alloc_rev(self)) { - self = PREV_CHUNK(self); - n1 += CHUNK_SIZE(self); + if (n < n0) { + int i = bin_index_up(n); + int j = bin_index(n0); + if (icsize = split->psize = n | C_INUSE; + split->csize = next->psize = n0-n | C_INUSE; + __bin_chunk(split); + return CHUNK_TO_MEM(self); } - self->csize = n1 | C_INUSE; - next->psize = n1 | C_INUSE; - /* If we got enough space, split off the excess and return */ - if (n <= n1) { - //memmove(CHUNK_TO_MEM(self), p, n0-OVERHEAD); - trim(self, n); - return CHUNK_TO_MEM(self); + lock(mal.split_merge_lock); + + size_t nsize = next->csize & C_INUSE ? 0 : CHUNK_SIZE(next); + if (n0+nsize >= n) { + int i = bin_index(nsize); + lock_bin(i); + if (!(next->csize & C_INUSE)) { + unbin(next, i); + unlock_bin(i); + next = NEXT_CHUNK(next); + self->csize = next->psize = n0+nsize | C_INUSE; + trim(self, n); + unlock(mal.split_merge_lock); + return CHUNK_TO_MEM(self); + } + unlock_bin(i); } + unlock(mal.split_merge_lock); copy_realloc: /* As a last resort, allocate a new chunk and copy to it. */ new = malloc(n-OVERHEAD); if (!new) return 0; copy_free_ret: - memcpy(new, p, n0-OVERHEAD); + memcpy(new, p, (npsize != self->csize) a_crash(); - for (;;) { - if (self->psize & next->csize & C_INUSE) { - self->csize = final_size | C_INUSE; - next->psize = final_size | C_INUSE; - i = bin_index(final_size); - lock_bin(i); - lock(mal.free_lock); - if (self->psize & next->csize & C_INUSE) - break; - unlock(mal.free_lock); - unlock_bin(i); - } + lock(mal.split_merge_lock); - if (alloc_rev(self)) { - self = PREV_CHUNK(self); - size = CHUNK_SIZE(self); - final_size += size; - if (new_size+size > RECLAIM && (new_size+size^size) > size) - reclaim = 1; - } + size_t osize = CHUNK_SIZE(self), size = osize; + + /* Since we hold split_merge_lock, only transition from free to + * in-use can race; in-use to free is impossible */ + size_t psize = self->psize & C_INUSE ? 0 : CHUNK_PSIZE(self); + size_t nsize = next->csize & C_INUSE ? 0 : CHUNK_SIZE(next); - if (alloc_fwd(next)) { - size = CHUNK_SIZE(next); - final_size += size; - if (new_size+size > RECLAIM && (new_size+size^size) > size) - reclaim = 1; + if (psize) { + int i = bin_index(psize); + lock_bin(i); + if (!(self->psize & C_INUSE)) { + struct chunk *prev = PREV_CHUNK(self); + unbin(prev, i); + self = prev; + size += psize; + } + unlock_bin(i); + } + if (nsize) { + int i = bin_index(nsize); + lock_bin(i); + if (!(next->csize & C_INUSE)) { + unbin(next, i); next = NEXT_CHUNK(next); + size += nsize; } + unlock_bin(i); } - if (!(mal.binmap & 1ULL<csize = final_size; - next->psize = final_size; - unlock(mal.free_lock); - - self->next = BIN_TO_CHUNK(i); - self->prev = mal.bins[i].tail; - self->next->prev = self; - self->prev->next = self; + self->csize = size; + next->psize = size; + bin_chunk(self, i); + unlock(mal.split_merge_lock); /* Replace middle of large chunks with fresh zero pages */ - if (reclaim) { + if (size > RECLAIM && (size^(size-osize)) > size-osize) { uintptr_t a = (uintptr_t)self + SIZE_ALIGN+PAGE_SIZE-1 & -PAGE_SIZE; uintptr_t b = (uintptr_t)next - SIZE_ALIGN & -PAGE_SIZE; + int e = errno; #if 1 __madvise((void *)a, b-a, MADV_DONTNEED); #else __mmap((void *)a, b-a, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED, -1, 0); #endif + errno = e; } unlock_bin(i); @@ -513,7 +501,9 @@ static void unmap_chunk(struct chunk *self) size_t len = CHUNK_SIZE(self) + extra; /* Crash on double free */ if (extra & 1) a_crash(); + int e = errno; __munmap(base, len); + errno = e; } void free(void *p) @@ -546,3 +536,21 @@ void __malloc_donate(char *start, char *end) c->csize = n->psize = C_INUSE | (end-start); __bin_chunk(c); } + +void __malloc_atfork(int who) +{ + if (who<0) { + lock(mal.split_merge_lock); + for (int i=0; i<64; i++) + lock(mal.bins[i].lock); + } else if (!who) { + for (int i=0; i<64; i++) + unlock(mal.bins[i].lock); + unlock(mal.split_merge_lock); + } else { + for (int i=0; i<64; i++) + mal.bins[i].lock[0] = mal.bins[i].lock[1] = 0; + mal.split_merge_lock[1] = 0; + mal.split_merge_lock[0] = 0; + } +} diff --git a/src/internal/malloc_impl.h b/src/malloc/oldmalloc/malloc_impl.h similarity index 85% rename from src/internal/malloc_impl.h rename to src/malloc/oldmalloc/malloc_impl.h index 59785a7fe..e1cf4774c 100644 --- a/src/internal/malloc_impl.h +++ b/src/malloc/oldmalloc/malloc_impl.h @@ -2,12 +2,7 @@ #define MALLOC_IMPL_H #include - -hidden void *__expand_heap(size_t *); - -hidden void __malloc_donate(char *, char *); - -hidden void *__memalign(size_t, size_t); +#include "dynlink.h" struct chunk { size_t psize, csize; @@ -41,6 +36,4 @@ struct bin { hidden void __bin_chunk(struct chunk *); -hidden extern int __malloc_replaced; - #endif diff --git a/src/malloc/malloc_usable_size.c b/src/malloc/oldmalloc/malloc_usable_size.c similarity index 100% rename from src/malloc/malloc_usable_size.c rename to src/malloc/oldmalloc/malloc_usable_size.c diff --git a/src/malloc/posix_memalign.c b/src/malloc/posix_memalign.c index 2ea8bd8a4..ad4d8f473 100644 --- a/src/malloc/posix_memalign.c +++ b/src/malloc/posix_memalign.c @@ -1,11 +1,10 @@ #include #include -#include "malloc_impl.h" int posix_memalign(void **res, size_t align, size_t len) { if (align < sizeof(void *)) return EINVAL; - void *mem = __memalign(align, len); + void *mem = aligned_alloc(align, len); if (!mem) return errno; *res = mem; return 0; diff --git a/src/malloc/realloc.c b/src/malloc/realloc.c new file mode 100644 index 000000000..fb0e8b7c4 --- /dev/null +++ b/src/malloc/realloc.c @@ -0,0 +1,6 @@ +#include + +void *realloc(void *p, size_t n) +{ + return __libc_realloc(p, n); +} diff --git a/src/malloc/reallocarray.c b/src/malloc/reallocarray.c new file mode 100644 index 000000000..4a6ebe460 --- /dev/null +++ b/src/malloc/reallocarray.c @@ -0,0 +1,13 @@ +#define _BSD_SOURCE +#include +#include + +void *reallocarray(void *ptr, size_t m, size_t n) +{ + if (n && m > -1 / n) { + errno = ENOMEM; + return 0; + } + + return realloc(ptr, m * n); +} diff --git a/src/malloc/replaced.c b/src/malloc/replaced.c new file mode 100644 index 000000000..07fce61ec --- /dev/null +++ b/src/malloc/replaced.c @@ -0,0 +1,4 @@ +#include "dynlink.h" + +int __malloc_replaced; +int __aligned_alloc_replaced; diff --git a/src/math/__math_invalidl.c b/src/math/__math_invalidl.c new file mode 100644 index 000000000..1fca99de4 --- /dev/null +++ b/src/math/__math_invalidl.c @@ -0,0 +1,9 @@ +#include +#include "libm.h" + +#if LDBL_MANT_DIG != DBL_MANT_DIG +long double __math_invalidl(long double x) +{ + return (x - x) / (x - x); +} +#endif diff --git a/src/math/arm/fabs.c b/src/math/arm/fabs.c index f890520a5..6e1d367d3 100644 --- a/src/math/arm/fabs.c +++ b/src/math/arm/fabs.c @@ -1,6 +1,6 @@ #include -#if __ARM_PCS_VFP +#if __ARM_PCS_VFP && __ARM_FP&8 double fabs(double x) { diff --git a/src/math/arm/sqrt.c b/src/math/arm/sqrt.c index 874af9606..567e2e910 100644 --- a/src/math/arm/sqrt.c +++ b/src/math/arm/sqrt.c @@ -1,6 +1,6 @@ #include -#if __ARM_PCS_VFP || (__VFP_FP__ && !__SOFTFP__) +#if (__ARM_PCS_VFP || (__VFP_FP__ && !__SOFTFP__)) && (__ARM_FP&8) double sqrt(double x) { diff --git a/src/math/i386/fabs.c b/src/math/i386/fabs.c new file mode 100644 index 000000000..396727863 --- /dev/null +++ b/src/math/i386/fabs.c @@ -0,0 +1,7 @@ +#include + +double fabs(double x) +{ + __asm__ ("fabs" : "+t"(x)); + return x; +} diff --git a/src/math/i386/fabs.s b/src/math/i386/fabs.s deleted file mode 100644 index d66ea9a19..000000000 --- a/src/math/i386/fabs.s +++ /dev/null @@ -1,6 +0,0 @@ -.global fabs -.type fabs,@function -fabs: - fldl 4(%esp) - fabs - ret diff --git a/src/math/i386/fabsf.c b/src/math/i386/fabsf.c new file mode 100644 index 000000000..d882eee34 --- /dev/null +++ b/src/math/i386/fabsf.c @@ -0,0 +1,7 @@ +#include + +float fabsf(float x) +{ + __asm__ ("fabs" : "+t"(x)); + return x; +} diff --git a/src/math/i386/fabsf.s b/src/math/i386/fabsf.s deleted file mode 100644 index a981c4222..000000000 --- a/src/math/i386/fabsf.s +++ /dev/null @@ -1,6 +0,0 @@ -.global fabsf -.type fabsf,@function -fabsf: - flds 4(%esp) - fabs - ret diff --git a/src/math/i386/fabsl.c b/src/math/i386/fabsl.c new file mode 100644 index 000000000..cc1c9ed9c --- /dev/null +++ b/src/math/i386/fabsl.c @@ -0,0 +1,7 @@ +#include + +long double fabsl(long double x) +{ + __asm__ ("fabs" : "+t"(x)); + return x; +} diff --git a/src/math/i386/fabsl.s b/src/math/i386/fabsl.s deleted file mode 100644 index ceef9e4cc..000000000 --- a/src/math/i386/fabsl.s +++ /dev/null @@ -1,6 +0,0 @@ -.global fabsl -.type fabsl,@function -fabsl: - fldt 4(%esp) - fabs - ret diff --git a/src/math/i386/fmod.c b/src/math/i386/fmod.c new file mode 100644 index 000000000..ea0c58d9b --- /dev/null +++ b/src/math/i386/fmod.c @@ -0,0 +1,10 @@ +#include + +double fmod(double x, double y) +{ + unsigned short fpsr; + // fprem does not introduce excess precision into x + do __asm__ ("fprem; fnstsw %%ax" : "+t"(x), "=a"(fpsr) : "u"(y)); + while (fpsr & 0x400); + return x; +} diff --git a/src/math/i386/fmod.s b/src/math/i386/fmod.s deleted file mode 100644 index 2113b3c54..000000000 --- a/src/math/i386/fmod.s +++ /dev/null @@ -1,11 +0,0 @@ -.global fmod -.type fmod,@function -fmod: - fldl 12(%esp) - fldl 4(%esp) -1: fprem - fnstsw %ax - sahf - jp 1b - fstp %st(1) - ret diff --git a/src/math/i386/fmodf.c b/src/math/i386/fmodf.c new file mode 100644 index 000000000..90b56ab0f --- /dev/null +++ b/src/math/i386/fmodf.c @@ -0,0 +1,10 @@ +#include + +float fmodf(float x, float y) +{ + unsigned short fpsr; + // fprem does not introduce excess precision into x + do __asm__ ("fprem; fnstsw %%ax" : "+t"(x), "=a"(fpsr) : "u"(y)); + while (fpsr & 0x400); + return x; +} diff --git a/src/math/i386/fmodf.s b/src/math/i386/fmodf.s deleted file mode 100644 index e04e2a567..000000000 --- a/src/math/i386/fmodf.s +++ /dev/null @@ -1,11 +0,0 @@ -.global fmodf -.type fmodf,@function -fmodf: - flds 8(%esp) - flds 4(%esp) -1: fprem - fnstsw %ax - sahf - jp 1b - fstp %st(1) - ret diff --git a/src/math/i386/fmodl.c b/src/math/i386/fmodl.c new file mode 100644 index 000000000..3daeab060 --- /dev/null +++ b/src/math/i386/fmodl.c @@ -0,0 +1,9 @@ +#include + +long double fmodl(long double x, long double y) +{ + unsigned short fpsr; + do __asm__ ("fprem; fnstsw %%ax" : "+t"(x), "=a"(fpsr) : "u"(y)); + while (fpsr & 0x400); + return x; +} diff --git a/src/math/i386/fmodl.s b/src/math/i386/fmodl.s deleted file mode 100644 index 0cb3fe9ba..000000000 --- a/src/math/i386/fmodl.s +++ /dev/null @@ -1,11 +0,0 @@ -.global fmodl -.type fmodl,@function -fmodl: - fldt 16(%esp) - fldt 4(%esp) -1: fprem - fnstsw %ax - sahf - jp 1b - fstp %st(1) - ret diff --git a/src/math/i386/llrint.c b/src/math/i386/llrint.c new file mode 100644 index 000000000..aa4008171 --- /dev/null +++ b/src/math/i386/llrint.c @@ -0,0 +1,8 @@ +#include + +long long llrint(double x) +{ + long long r; + __asm__ ("fistpll %0" : "=m"(r) : "t"(x) : "st"); + return r; +} diff --git a/src/math/i386/llrint.s b/src/math/i386/llrint.s deleted file mode 100644 index 8e89cd913..000000000 --- a/src/math/i386/llrint.s +++ /dev/null @@ -1,8 +0,0 @@ -.global llrint -.type llrint,@function -llrint: - fldl 4(%esp) - fistpll 4(%esp) - mov 4(%esp),%eax - mov 8(%esp),%edx - ret diff --git a/src/math/i386/llrintf.c b/src/math/i386/llrintf.c new file mode 100644 index 000000000..c41a317bd --- /dev/null +++ b/src/math/i386/llrintf.c @@ -0,0 +1,8 @@ +#include + +long long llrintf(float x) +{ + long long r; + __asm__ ("fistpll %0" : "=m"(r) : "t"(x) : "st"); + return r; +} diff --git a/src/math/i386/llrintf.s b/src/math/i386/llrintf.s deleted file mode 100644 index aa850c6cd..000000000 --- a/src/math/i386/llrintf.s +++ /dev/null @@ -1,9 +0,0 @@ -.global llrintf -.type llrintf,@function -llrintf: - sub $8,%esp - flds 12(%esp) - fistpll (%esp) - pop %eax - pop %edx - ret diff --git a/src/math/i386/llrintl.c b/src/math/i386/llrintl.c new file mode 100644 index 000000000..c439ef28d --- /dev/null +++ b/src/math/i386/llrintl.c @@ -0,0 +1,8 @@ +#include + +long long llrintl(long double x) +{ + long long r; + __asm__ ("fistpll %0" : "=m"(r) : "t"(x) : "st"); + return r; +} diff --git a/src/math/i386/llrintl.s b/src/math/i386/llrintl.s deleted file mode 100644 index 1cfb56f15..000000000 --- a/src/math/i386/llrintl.s +++ /dev/null @@ -1,8 +0,0 @@ -.global llrintl -.type llrintl,@function -llrintl: - fldt 4(%esp) - fistpll 4(%esp) - mov 4(%esp),%eax - mov 8(%esp),%edx - ret diff --git a/src/math/i386/lrint.c b/src/math/i386/lrint.c new file mode 100644 index 000000000..89563ab26 --- /dev/null +++ b/src/math/i386/lrint.c @@ -0,0 +1,8 @@ +#include + +long lrint(double x) +{ + long r; + __asm__ ("fistpl %0" : "=m"(r) : "t"(x) : "st"); + return r; +} diff --git a/src/math/i386/lrint.s b/src/math/i386/lrint.s deleted file mode 100644 index 02b83d9ff..000000000 --- a/src/math/i386/lrint.s +++ /dev/null @@ -1,7 +0,0 @@ -.global lrint -.type lrint,@function -lrint: - fldl 4(%esp) - fistpl 4(%esp) - mov 4(%esp),%eax - ret diff --git a/src/math/i386/lrintf.c b/src/math/i386/lrintf.c new file mode 100644 index 000000000..0bbf29de0 --- /dev/null +++ b/src/math/i386/lrintf.c @@ -0,0 +1,8 @@ +#include + +long lrintf(float x) +{ + long r; + __asm__ ("fistpl %0" : "=m"(r) : "t"(x) : "st"); + return r; +} diff --git a/src/math/i386/lrintf.s b/src/math/i386/lrintf.s deleted file mode 100644 index 907aac291..000000000 --- a/src/math/i386/lrintf.s +++ /dev/null @@ -1,7 +0,0 @@ -.global lrintf -.type lrintf,@function -lrintf: - flds 4(%esp) - fistpl 4(%esp) - mov 4(%esp),%eax - ret diff --git a/src/math/i386/lrintl.c b/src/math/i386/lrintl.c new file mode 100644 index 000000000..eb8c09028 --- /dev/null +++ b/src/math/i386/lrintl.c @@ -0,0 +1,8 @@ +#include + +long lrintl(long double x) +{ + long r; + __asm__ ("fistpl %0" : "=m"(r) : "t"(x) : "st"); + return r; +} diff --git a/src/math/i386/lrintl.s b/src/math/i386/lrintl.s deleted file mode 100644 index 3ae05aac2..000000000 --- a/src/math/i386/lrintl.s +++ /dev/null @@ -1,7 +0,0 @@ -.global lrintl -.type lrintl,@function -lrintl: - fldt 4(%esp) - fistpl 4(%esp) - mov 4(%esp),%eax - ret diff --git a/src/math/i386/remainder.c b/src/math/i386/remainder.c new file mode 100644 index 000000000..c083df904 --- /dev/null +++ b/src/math/i386/remainder.c @@ -0,0 +1,12 @@ +#include + +double remainder(double x, double y) +{ + unsigned short fpsr; + // fprem1 does not introduce excess precision into x + do __asm__ ("fprem1; fnstsw %%ax" : "+t"(x), "=a"(fpsr) : "u"(y)); + while (fpsr & 0x400); + return x; +} + +weak_alias(remainder, drem); diff --git a/src/math/i386/remainder.s b/src/math/i386/remainder.s deleted file mode 100644 index ab1da95dd..000000000 --- a/src/math/i386/remainder.s +++ /dev/null @@ -1,14 +0,0 @@ -.global remainder -.type remainder,@function -remainder: -.weak drem -.type drem,@function -drem: - fldl 12(%esp) - fldl 4(%esp) -1: fprem1 - fnstsw %ax - sahf - jp 1b - fstp %st(1) - ret diff --git a/src/math/i386/remainderf.c b/src/math/i386/remainderf.c new file mode 100644 index 000000000..280207d26 --- /dev/null +++ b/src/math/i386/remainderf.c @@ -0,0 +1,12 @@ +#include + +float remainderf(float x, float y) +{ + unsigned short fpsr; + // fprem1 does not introduce excess precision into x + do __asm__ ("fprem1; fnstsw %%ax" : "+t"(x), "=a"(fpsr) : "u"(y)); + while (fpsr & 0x400); + return x; +} + +weak_alias(remainderf, dremf); diff --git a/src/math/i386/remainderf.s b/src/math/i386/remainderf.s deleted file mode 100644 index 6a7378a34..000000000 --- a/src/math/i386/remainderf.s +++ /dev/null @@ -1,14 +0,0 @@ -.global remainderf -.type remainderf,@function -remainderf: -.weak dremf -.type dremf,@function -dremf: - flds 8(%esp) - flds 4(%esp) -1: fprem1 - fnstsw %ax - sahf - jp 1b - fstp %st(1) - ret diff --git a/src/math/i386/remainderl.c b/src/math/i386/remainderl.c new file mode 100644 index 000000000..8cf75071e --- /dev/null +++ b/src/math/i386/remainderl.c @@ -0,0 +1,9 @@ +#include + +long double remainderl(long double x, long double y) +{ + unsigned short fpsr; + do __asm__ ("fprem1; fnstsw %%ax" : "+t"(x), "=a"(fpsr) : "u"(y)); + while (fpsr & 0x400); + return x; +} diff --git a/src/math/i386/remainderl.s b/src/math/i386/remainderl.s deleted file mode 100644 index b41518ede..000000000 --- a/src/math/i386/remainderl.s +++ /dev/null @@ -1,11 +0,0 @@ -.global remainderl -.type remainderl,@function -remainderl: - fldt 16(%esp) - fldt 4(%esp) -1: fprem1 - fnstsw %ax - sahf - jp 1b - fstp %st(1) - ret diff --git a/src/math/i386/rint.c b/src/math/i386/rint.c new file mode 100644 index 000000000..a5276a60d --- /dev/null +++ b/src/math/i386/rint.c @@ -0,0 +1,7 @@ +#include + +double rint(double x) +{ + __asm__ ("frndint" : "+t"(x)); + return x; +} diff --git a/src/math/i386/rint.s b/src/math/i386/rint.s deleted file mode 100644 index bb99a11c3..000000000 --- a/src/math/i386/rint.s +++ /dev/null @@ -1,6 +0,0 @@ -.global rint -.type rint,@function -rint: - fldl 4(%esp) - frndint - ret diff --git a/src/math/i386/rintf.c b/src/math/i386/rintf.c new file mode 100644 index 000000000..bb4121a4e --- /dev/null +++ b/src/math/i386/rintf.c @@ -0,0 +1,7 @@ +#include + +float rintf(float x) +{ + __asm__ ("frndint" : "+t"(x)); + return x; +} diff --git a/src/math/i386/rintf.s b/src/math/i386/rintf.s deleted file mode 100644 index bce4c5a60..000000000 --- a/src/math/i386/rintf.s +++ /dev/null @@ -1,6 +0,0 @@ -.global rintf -.type rintf,@function -rintf: - flds 4(%esp) - frndint - ret diff --git a/src/math/i386/rintl.c b/src/math/i386/rintl.c new file mode 100644 index 000000000..e1a92077f --- /dev/null +++ b/src/math/i386/rintl.c @@ -0,0 +1,7 @@ +#include + +long double rintl(long double x) +{ + __asm__ ("frndint" : "+t"(x)); + return x; +} diff --git a/src/math/i386/rintl.s b/src/math/i386/rintl.s deleted file mode 100644 index cd2bf9a98..000000000 --- a/src/math/i386/rintl.s +++ /dev/null @@ -1,6 +0,0 @@ -.global rintl -.type rintl,@function -rintl: - fldt 4(%esp) - frndint - ret diff --git a/src/math/i386/sqrt.c b/src/math/i386/sqrt.c new file mode 100644 index 000000000..934fbccab --- /dev/null +++ b/src/math/i386/sqrt.c @@ -0,0 +1,15 @@ +#include "libm.h" + +double sqrt(double x) +{ + union ldshape ux; + unsigned fpsr; + __asm__ ("fsqrt; fnstsw %%ax": "=t"(ux.f), "=a"(fpsr) : "0"(x)); + if ((ux.i.m & 0x7ff) != 0x400) + return (double)ux.f; + /* Rounding to double would have encountered an exact halfway case. + Adjust mantissa downwards if fsqrt rounded up, else upwards. + (result of fsqrt could not have been exact) */ + ux.i.m ^= (fpsr & 0x200) + 0x300; + return (double)ux.f; +} diff --git a/src/math/i386/sqrt.s b/src/math/i386/sqrt.s deleted file mode 100644 index 57837e256..000000000 --- a/src/math/i386/sqrt.s +++ /dev/null @@ -1,21 +0,0 @@ -.global sqrt -.type sqrt,@function -sqrt: fldl 4(%esp) - fsqrt - fnstsw %ax - sub $12,%esp - fld %st(0) - fstpt (%esp) - mov (%esp),%ecx - and $0x7ff,%ecx - cmp $0x400,%ecx - jnz 1f - and $0x200,%eax - sub $0x100,%eax - sub %eax,(%esp) - fstp %st(0) - fldt (%esp) -1: add $12,%esp - fstpl 4(%esp) - fldl 4(%esp) - ret diff --git a/src/math/i386/sqrtf.c b/src/math/i386/sqrtf.c new file mode 100644 index 000000000..41c65c2bd --- /dev/null +++ b/src/math/i386/sqrtf.c @@ -0,0 +1,12 @@ +#include + +float sqrtf(float x) +{ + long double t; + /* The long double result has sufficient precision so that + * second rounding to float still keeps the returned value + * correctly rounded, see Pierre Roux, "Innocuous Double + * Rounding of Basic Arithmetic Operations". */ + __asm__ ("fsqrt" : "=t"(t) : "0"(x)); + return (float)t; +} diff --git a/src/math/i386/sqrtf.s b/src/math/i386/sqrtf.s deleted file mode 100644 index 9e944f456..000000000 --- a/src/math/i386/sqrtf.s +++ /dev/null @@ -1,7 +0,0 @@ -.global sqrtf -.type sqrtf,@function -sqrtf: flds 4(%esp) - fsqrt - fstps 4(%esp) - flds 4(%esp) - ret diff --git a/src/math/i386/sqrtl.c b/src/math/i386/sqrtl.c new file mode 100644 index 000000000..864cfcc4f --- /dev/null +++ b/src/math/i386/sqrtl.c @@ -0,0 +1,7 @@ +#include + +long double sqrtl(long double x) +{ + __asm__ ("fsqrt" : "+t"(x)); + return x; +} diff --git a/src/math/i386/sqrtl.s b/src/math/i386/sqrtl.s deleted file mode 100644 index e0d426168..000000000 --- a/src/math/i386/sqrtl.s +++ /dev/null @@ -1,5 +0,0 @@ -.global sqrtl -.type sqrtl,@function -sqrtl: fldt 4(%esp) - fsqrt - ret diff --git a/src/math/m68k/sqrtl.c b/src/math/m68k/sqrtl.c new file mode 100644 index 000000000..b1c303c7e --- /dev/null +++ b/src/math/m68k/sqrtl.c @@ -0,0 +1,15 @@ +#include + +#if __HAVE_68881__ + +long double sqrtl(long double x) +{ + __asm__ ("fsqrt.x %1,%0" : "=f"(x) : "fm"(x)); + return x; +} + +#else + +#include "../sqrtl.c" + +#endif diff --git a/src/math/sqrt.c b/src/math/sqrt.c index f1f6d76c7..5ba265596 100644 --- a/src/math/sqrt.c +++ b/src/math/sqrt.c @@ -1,184 +1,158 @@ -/* origin: FreeBSD /usr/src/lib/msun/src/e_sqrt.c */ -/* - * ==================================================== - * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. - * - * Developed at SunSoft, a Sun Microsystems, Inc. business. - * Permission to use, copy, modify, and distribute this - * software is freely granted, provided that this notice - * is preserved. - * ==================================================== - */ -/* sqrt(x) - * Return correctly rounded sqrt. - * ------------------------------------------ - * | Use the hardware sqrt if you have one | - * ------------------------------------------ - * Method: - * Bit by bit method using integer arithmetic. (Slow, but portable) - * 1. Normalization - * Scale x to y in [1,4) with even powers of 2: - * find an integer k such that 1 <= (y=x*2^(2k)) < 4, then - * sqrt(x) = 2^k * sqrt(y) - * 2. Bit by bit computation - * Let q = sqrt(y) truncated to i bit after binary point (q = 1), - * i 0 - * i+1 2 - * s = 2*q , and y = 2 * ( y - q ). (1) - * i i i i - * - * To compute q from q , one checks whether - * i+1 i - * - * -(i+1) 2 - * (q + 2 ) <= y. (2) - * i - * -(i+1) - * If (2) is false, then q = q ; otherwise q = q + 2 . - * i+1 i i+1 i - * - * With some algebric manipulation, it is not difficult to see - * that (2) is equivalent to - * -(i+1) - * s + 2 <= y (3) - * i i - * - * The advantage of (3) is that s and y can be computed by - * i i - * the following recurrence formula: - * if (3) is false - * - * s = s , y = y ; (4) - * i+1 i i+1 i - * - * otherwise, - * -i -(i+1) - * s = s + 2 , y = y - s - 2 (5) - * i+1 i i+1 i i - * - * One may easily use induction to prove (4) and (5). - * Note. Since the left hand side of (3) contain only i+2 bits, - * it does not necessary to do a full (53-bit) comparison - * in (3). - * 3. Final rounding - * After generating the 53 bits result, we compute one more bit. - * Together with the remainder, we can decide whether the - * result is exact, bigger than 1/2ulp, or less than 1/2ulp - * (it will never equal to 1/2ulp). - * The rounding mode can be detected by checking whether - * huge + tiny is equal to huge, and whether huge - tiny is - * equal to huge for some floating point number "huge" and "tiny". - * - * Special cases: - * sqrt(+-0) = +-0 ... exact - * sqrt(inf) = inf - * sqrt(-ve) = NaN ... with invalid signal - * sqrt(NaN) = NaN ... with invalid signal for signaling NaN - */ - +#include +#include #include "libm.h" +#include "sqrt_data.h" -static const double tiny = 1.0e-300; +#define FENV_SUPPORT 1 -double sqrt(double x) +/* returns a*b*2^-32 - e, with error 0 <= e < 1. */ +static inline uint32_t mul32(uint32_t a, uint32_t b) { - double z; - int32_t sign = (int)0x80000000; - int32_t ix0,s0,q,m,t,i; - uint32_t r,t1,s1,ix1,q1; + return (uint64_t)a*b >> 32; +} - EXTRACT_WORDS(ix0, ix1, x); +/* returns a*b*2^-64 - e, with error 0 <= e < 3. */ +static inline uint64_t mul64(uint64_t a, uint64_t b) +{ + uint64_t ahi = a>>32; + uint64_t alo = a&0xffffffff; + uint64_t bhi = b>>32; + uint64_t blo = b&0xffffffff; + return ahi*bhi + (ahi*blo >> 32) + (alo*bhi >> 32); +} - /* take care of Inf and NaN */ - if ((ix0&0x7ff00000) == 0x7ff00000) { - return x*x + x; /* sqrt(NaN)=NaN, sqrt(+inf)=+inf, sqrt(-inf)=sNaN */ - } - /* take care of zero */ - if (ix0 <= 0) { - if (((ix0&~sign)|ix1) == 0) - return x; /* sqrt(+-0) = +-0 */ - if (ix0 < 0) - return (x-x)/(x-x); /* sqrt(-ve) = sNaN */ - } - /* normalize x */ - m = ix0>>20; - if (m == 0) { /* subnormal x */ - while (ix0 == 0) { - m -= 21; - ix0 |= (ix1>>11); - ix1 <<= 21; - } - for (i=0; (ix0&0x00100000) == 0; i++) - ix0<<=1; - m -= i - 1; - ix0 |= ix1>>(32-i); - ix1 <<= i; - } - m -= 1023; /* unbias exponent */ - ix0 = (ix0&0x000fffff)|0x00100000; - if (m & 1) { /* odd m, double x to make it even */ - ix0 += ix0 + ((ix1&sign)>>31); - ix1 += ix1; - } - m >>= 1; /* m = [m/2] */ - - /* generate sqrt(x) bit by bit */ - ix0 += ix0 + ((ix1&sign)>>31); - ix1 += ix1; - q = q1 = s0 = s1 = 0; /* [q,q1] = sqrt(x) */ - r = 0x00200000; /* r = moving bit from right to left */ - - while (r != 0) { - t = s0 + r; - if (t <= ix0) { - s0 = t + r; - ix0 -= t; - q += r; - } - ix0 += ix0 + ((ix1&sign)>>31); - ix1 += ix1; - r >>= 1; - } +double sqrt(double x) +{ + uint64_t ix, top, m; - r = sign; - while (r != 0) { - t1 = s1 + r; - t = s0; - if (t < ix0 || (t == ix0 && t1 <= ix1)) { - s1 = t1 + r; - if ((t1&sign) == sign && (s1&sign) == 0) - s0++; - ix0 -= t; - if (ix1 < t1) - ix0--; - ix1 -= t1; - q1 += r; - } - ix0 += ix0 + ((ix1&sign)>>31); - ix1 += ix1; - r >>= 1; + /* special case handling. */ + ix = asuint64(x); + top = ix >> 52; + if (predict_false(top - 0x001 >= 0x7ff - 0x001)) { + /* x < 0x1p-1022 or inf or nan. */ + if (ix * 2 == 0) + return x; + if (ix == 0x7ff0000000000000) + return x; + if (ix > 0x7ff0000000000000) + return __math_invalid(x); + /* x is subnormal, normalize it. */ + ix = asuint64(x * 0x1p52); + top = ix >> 52; + top -= 52; } - /* use floating add to find out rounding direction */ - if ((ix0|ix1) != 0) { - z = 1.0 - tiny; /* raise inexact flag */ - if (z >= 1.0) { - z = 1.0 + tiny; - if (q1 == (uint32_t)0xffffffff) { - q1 = 0; - q++; - } else if (z > 1.0) { - if (q1 == (uint32_t)0xfffffffe) - q++; - q1 += 2; - } else - q1 += q1 & 1; - } + /* argument reduction: + x = 4^e m; with integer e, and m in [1, 4) + m: fixed point representation [2.62] + 2^e is the exponent part of the result. */ + int even = top & 1; + m = (ix << 11) | 0x8000000000000000; + if (even) m >>= 1; + top = (top + 0x3ff) >> 1; + + /* approximate r ~ 1/sqrt(m) and s ~ sqrt(m) when m in [1,4) + + initial estimate: + 7bit table lookup (1bit exponent and 6bit significand). + + iterative approximation: + using 2 goldschmidt iterations with 32bit int arithmetics + and a final iteration with 64bit int arithmetics. + + details: + + the relative error (e = r0 sqrt(m)-1) of a linear estimate + (r0 = a m + b) is |e| < 0.085955 ~ 0x1.6p-4 at best, + a table lookup is faster and needs one less iteration + 6 bit lookup table (128b) gives |e| < 0x1.f9p-8 + 7 bit lookup table (256b) gives |e| < 0x1.fdp-9 + for single and double prec 6bit is enough but for quad + prec 7bit is needed (or modified iterations). to avoid + one more iteration >=13bit table would be needed (16k). + + a newton-raphson iteration for r is + w = r*r + u = 3 - m*w + r = r*u/2 + can use a goldschmidt iteration for s at the end or + s = m*r + + first goldschmidt iteration is + s = m*r + u = 3 - s*r + r = r*u/2 + s = s*u/2 + next goldschmidt iteration is + u = 3 - s*r + r = r*u/2 + s = s*u/2 + and at the end r is not computed only s. + + they use the same amount of operations and converge at the + same quadratic rate, i.e. if + r1 sqrt(m) - 1 = e, then + r2 sqrt(m) - 1 = -3/2 e^2 - 1/2 e^3 + the advantage of goldschmidt is that the mul for s and r + are independent (computed in parallel), however it is not + "self synchronizing": it only uses the input m in the + first iteration so rounding errors accumulate. at the end + or when switching to larger precision arithmetics rounding + errors dominate so the first iteration should be used. + + the fixed point representations are + m: 2.30 r: 0.32, s: 2.30, d: 2.30, u: 2.30, three: 2.30 + and after switching to 64 bit + m: 2.62 r: 0.64, s: 2.62, d: 2.62, u: 2.62, three: 2.62 */ + + static const uint64_t three = 0xc0000000; + uint64_t r, s, d, u, i; + + i = (ix >> 46) % 128; + r = (uint32_t)__rsqrt_tab[i] << 16; + /* |r sqrt(m) - 1| < 0x1.fdp-9 */ + s = mul32(m>>32, r); + /* |s/sqrt(m) - 1| < 0x1.fdp-9 */ + d = mul32(s, r); + u = three - d; + r = mul32(r, u) << 1; + /* |r sqrt(m) - 1| < 0x1.7bp-16 */ + s = mul32(s, u) << 1; + /* |s/sqrt(m) - 1| < 0x1.7bp-16 */ + d = mul32(s, r); + u = three - d; + r = mul32(r, u) << 1; + /* |r sqrt(m) - 1| < 0x1.3704p-29 (measured worst-case) */ + r = r << 32; + s = mul64(m, r); + d = mul64(s, r); + u = (three<<32) - d; + s = mul64(s, u); /* repr: 3.61 */ + /* -0x1p-57 < s - sqrt(m) < 0x1.8001p-61 */ + s = (s - 2) >> 9; /* repr: 12.52 */ + /* -0x1.09p-52 < s - sqrt(m) < -0x1.fffcp-63 */ + + /* s < sqrt(m) < s + 0x1.09p-52, + compute nearest rounded result: + the nearest result to 52 bits is either s or s+0x1p-52, + we can decide by comparing (2^52 s + 0.5)^2 to 2^104 m. */ + uint64_t d0, d1, d2; + double y, t; + d0 = (m << 42) - s*s; + d1 = s - d0; + d2 = d1 + s + 1; + s += d1 >> 63; + s &= 0x000fffffffffffff; + s |= top << 52; + y = asdouble(s); + if (FENV_SUPPORT) { + /* handle rounding modes and inexact exception: + only (s+1)^2 == 2^42 m case is exact otherwise + add a tiny value to cause the fenv effects. */ + uint64_t tiny = predict_false(d2==0) ? 0 : 0x0010000000000000; + tiny |= (d1^d2) & 0x8000000000000000; + t = asdouble(tiny); + y = eval_as_double(y + t); } - ix0 = (q>>1) + 0x3fe00000; - ix1 = q1>>1; - if (q&1) - ix1 |= sign; - INSERT_WORDS(z, ix0 + ((uint32_t)m << 20), ix1); - return z; + return y; } diff --git a/src/math/sqrt_data.c b/src/math/sqrt_data.c new file mode 100644 index 000000000..61bc22f43 --- /dev/null +++ b/src/math/sqrt_data.c @@ -0,0 +1,19 @@ +#include "sqrt_data.h" +const uint16_t __rsqrt_tab[128] = { +0xb451,0xb2f0,0xb196,0xb044,0xaef9,0xadb6,0xac79,0xab43, +0xaa14,0xa8eb,0xa7c8,0xa6aa,0xa592,0xa480,0xa373,0xa26b, +0xa168,0xa06a,0x9f70,0x9e7b,0x9d8a,0x9c9d,0x9bb5,0x9ad1, +0x99f0,0x9913,0x983a,0x9765,0x9693,0x95c4,0x94f8,0x9430, +0x936b,0x92a9,0x91ea,0x912e,0x9075,0x8fbe,0x8f0a,0x8e59, +0x8daa,0x8cfe,0x8c54,0x8bac,0x8b07,0x8a64,0x89c4,0x8925, +0x8889,0x87ee,0x8756,0x86c0,0x862b,0x8599,0x8508,0x8479, +0x83ec,0x8361,0x82d8,0x8250,0x81c9,0x8145,0x80c2,0x8040, +0xff02,0xfd0e,0xfb25,0xf947,0xf773,0xf5aa,0xf3ea,0xf234, +0xf087,0xeee3,0xed47,0xebb3,0xea27,0xe8a3,0xe727,0xe5b2, +0xe443,0xe2dc,0xe17a,0xe020,0xdecb,0xdd7d,0xdc34,0xdaf1, +0xd9b3,0xd87b,0xd748,0xd61a,0xd4f1,0xd3cd,0xd2ad,0xd192, +0xd07b,0xcf69,0xce5b,0xcd51,0xcc4a,0xcb48,0xca4a,0xc94f, +0xc858,0xc764,0xc674,0xc587,0xc49d,0xc3b7,0xc2d4,0xc1f4, +0xc116,0xc03c,0xbf65,0xbe90,0xbdbe,0xbcef,0xbc23,0xbb59, +0xba91,0xb9cc,0xb90a,0xb84a,0xb78c,0xb6d0,0xb617,0xb560, +}; diff --git a/src/math/sqrt_data.h b/src/math/sqrt_data.h new file mode 100644 index 000000000..260c7f9c2 --- /dev/null +++ b/src/math/sqrt_data.h @@ -0,0 +1,13 @@ +#ifndef _SQRT_DATA_H +#define _SQRT_DATA_H + +#include +#include + +/* if x in [1,2): i = (int)(64*x); + if x in [2,4): i = (int)(32*x-64); + __rsqrt_tab[i]*2^-16 is estimating 1/sqrt(x) with small relative error: + |__rsqrt_tab[i]*0x1p-16*sqrt(x) - 1| < -0x1.fdp-9 < 2^-8 */ +extern hidden const uint16_t __rsqrt_tab[128]; + +#endif diff --git a/src/math/sqrtf.c b/src/math/sqrtf.c index d6ace38aa..740d81cba 100644 --- a/src/math/sqrtf.c +++ b/src/math/sqrtf.c @@ -1,83 +1,83 @@ -/* origin: FreeBSD /usr/src/lib/msun/src/e_sqrtf.c */ -/* - * Conversion to float by Ian Lance Taylor, Cygnus Support, ian@cygnus.com. - */ -/* - * ==================================================== - * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. - * - * Developed at SunPro, a Sun Microsystems, Inc. business. - * Permission to use, copy, modify, and distribute this - * software is freely granted, provided that this notice - * is preserved. - * ==================================================== - */ - +#include +#include #include "libm.h" +#include "sqrt_data.h" -static const float tiny = 1.0e-30; +#define FENV_SUPPORT 1 -float sqrtf(float x) +static inline uint32_t mul32(uint32_t a, uint32_t b) { - float z; - int32_t sign = (int)0x80000000; - int32_t ix,s,q,m,t,i; - uint32_t r; + return (uint64_t)a*b >> 32; +} - GET_FLOAT_WORD(ix, x); +/* see sqrt.c for more detailed comments. */ - /* take care of Inf and NaN */ - if ((ix&0x7f800000) == 0x7f800000) - return x*x + x; /* sqrt(NaN)=NaN, sqrt(+inf)=+inf, sqrt(-inf)=sNaN */ +float sqrtf(float x) +{ + uint32_t ix, m, m1, m0, even, ey; - /* take care of zero */ - if (ix <= 0) { - if ((ix&~sign) == 0) - return x; /* sqrt(+-0) = +-0 */ - if (ix < 0) - return (x-x)/(x-x); /* sqrt(-ve) = sNaN */ - } - /* normalize x */ - m = ix>>23; - if (m == 0) { /* subnormal x */ - for (i = 0; (ix&0x00800000) == 0; i++) - ix<<=1; - m -= i - 1; + ix = asuint(x); + if (predict_false(ix - 0x00800000 >= 0x7f800000 - 0x00800000)) { + /* x < 0x1p-126 or inf or nan. */ + if (ix * 2 == 0) + return x; + if (ix == 0x7f800000) + return x; + if (ix > 0x7f800000) + return __math_invalidf(x); + /* x is subnormal, normalize it. */ + ix = asuint(x * 0x1p23f); + ix -= 23 << 23; } - m -= 127; /* unbias exponent */ - ix = (ix&0x007fffff)|0x00800000; - if (m&1) /* odd m, double x to make it even */ - ix += ix; - m >>= 1; /* m = [m/2] */ - /* generate sqrt(x) bit by bit */ - ix += ix; - q = s = 0; /* q = sqrt(x) */ - r = 0x01000000; /* r = moving bit from right to left */ + /* x = 4^e m; with int e and m in [1, 4). */ + even = ix & 0x00800000; + m1 = (ix << 8) | 0x80000000; + m0 = (ix << 7) & 0x7fffffff; + m = even ? m0 : m1; - while (r != 0) { - t = s + r; - if (t <= ix) { - s = t+r; - ix -= t; - q += r; - } - ix += ix; - r >>= 1; - } + /* 2^e is the exponent part of the return value. */ + ey = ix >> 1; + ey += 0x3f800000 >> 1; + ey &= 0x7f800000; + + /* compute r ~ 1/sqrt(m), s ~ sqrt(m) with 2 goldschmidt iterations. */ + static const uint32_t three = 0xc0000000; + uint32_t r, s, d, u, i; + i = (ix >> 17) % 128; + r = (uint32_t)__rsqrt_tab[i] << 16; + /* |r*sqrt(m) - 1| < 0x1p-8 */ + s = mul32(m, r); + /* |s/sqrt(m) - 1| < 0x1p-8 */ + d = mul32(s, r); + u = three - d; + r = mul32(r, u) << 1; + /* |r*sqrt(m) - 1| < 0x1.7bp-16 */ + s = mul32(s, u) << 1; + /* |s/sqrt(m) - 1| < 0x1.7bp-16 */ + d = mul32(s, r); + u = three - d; + s = mul32(s, u); + /* -0x1.03p-28 < s/sqrt(m) - 1 < 0x1.fp-31 */ + s = (s - 1)>>6; + /* s < sqrt(m) < s + 0x1.08p-23 */ - /* use floating add to find out rounding direction */ - if (ix != 0) { - z = 1.0f - tiny; /* raise inexact flag */ - if (z >= 1.0f) { - z = 1.0f + tiny; - if (z > 1.0f) - q += 2; - else - q += q & 1; - } + /* compute nearest rounded result. */ + uint32_t d0, d1, d2; + float y, t; + d0 = (m << 16) - s*s; + d1 = s - d0; + d2 = d1 + s + 1; + s += d1 >> 31; + s &= 0x007fffff; + s |= ey; + y = asfloat(s); + if (FENV_SUPPORT) { + /* handle rounding and inexact exception. */ + uint32_t tiny = predict_false(d2==0) ? 0 : 0x01000000; + tiny |= (d1^d2) & 0x80000000; + t = asfloat(tiny); + y = eval_as_float(y + t); } - ix = (q>>1) + 0x3f000000; - SET_FLOAT_WORD(z, ix + ((uint32_t)m << 23)); - return z; + return y; } diff --git a/src/math/sqrtl.c b/src/math/sqrtl.c index 83a8f80c9..1b9f19c7d 100644 --- a/src/math/sqrtl.c +++ b/src/math/sqrtl.c @@ -1,7 +1,259 @@ +#include #include +#include +#include "libm.h" +#if LDBL_MANT_DIG == 53 && LDBL_MAX_EXP == 1024 long double sqrtl(long double x) { - /* FIXME: implement in C, this is for LDBL_MANT_DIG == 64 only */ return sqrt(x); } +#elif (LDBL_MANT_DIG == 113 || LDBL_MANT_DIG == 64) && LDBL_MAX_EXP == 16384 +#include "sqrt_data.h" + +#define FENV_SUPPORT 1 + +typedef struct { + uint64_t hi; + uint64_t lo; +} u128; + +/* top: 16 bit sign+exponent, x: significand. */ +static inline long double mkldbl(uint64_t top, u128 x) +{ + union ldshape u; +#if LDBL_MANT_DIG == 113 + u.i2.hi = x.hi; + u.i2.lo = x.lo; + u.i2.hi &= 0x0000ffffffffffff; + u.i2.hi |= top << 48; +#elif LDBL_MANT_DIG == 64 + u.i.se = top; + u.i.m = x.lo; + /* force the top bit on non-zero (and non-subnormal) results. */ + if (top & 0x7fff) + u.i.m |= 0x8000000000000000; +#endif + return u.f; +} + +/* return: top 16 bit is sign+exp and following bits are the significand. */ +static inline u128 asu128(long double x) +{ + union ldshape u = {.f=x}; + u128 r; +#if LDBL_MANT_DIG == 113 + r.hi = u.i2.hi; + r.lo = u.i2.lo; +#elif LDBL_MANT_DIG == 64 + r.lo = u.i.m<<49; + /* ignore the top bit: pseudo numbers are not handled. */ + r.hi = u.i.m>>15; + r.hi &= 0x0000ffffffffffff; + r.hi |= (uint64_t)u.i.se << 48; +#endif + return r; +} + +/* returns a*b*2^-32 - e, with error 0 <= e < 1. */ +static inline uint32_t mul32(uint32_t a, uint32_t b) +{ + return (uint64_t)a*b >> 32; +} + +/* returns a*b*2^-64 - e, with error 0 <= e < 3. */ +static inline uint64_t mul64(uint64_t a, uint64_t b) +{ + uint64_t ahi = a>>32; + uint64_t alo = a&0xffffffff; + uint64_t bhi = b>>32; + uint64_t blo = b&0xffffffff; + return ahi*bhi + (ahi*blo >> 32) + (alo*bhi >> 32); +} + +static inline u128 add64(u128 a, uint64_t b) +{ + u128 r; + r.lo = a.lo + b; + r.hi = a.hi; + if (r.lo < a.lo) + r.hi++; + return r; +} + +static inline u128 add128(u128 a, u128 b) +{ + u128 r; + r.lo = a.lo + b.lo; + r.hi = a.hi + b.hi; + if (r.lo < a.lo) + r.hi++; + return r; +} + +static inline u128 sub64(u128 a, uint64_t b) +{ + u128 r; + r.lo = a.lo - b; + r.hi = a.hi; + if (a.lo < b) + r.hi--; + return r; +} + +static inline u128 sub128(u128 a, u128 b) +{ + u128 r; + r.lo = a.lo - b.lo; + r.hi = a.hi - b.hi; + if (a.lo < b.lo) + r.hi--; + return r; +} + +/* a<= 64) { + a.hi = a.lo<<(n-64); + a.lo = 0; + } else { + a.hi = (a.hi<>(64-n)); + a.lo = a.lo<>n, 0 <= n <= 127 */ +static inline u128 rsh(u128 a, int n) +{ + if (n == 0) + return a; + if (n >= 64) { + a.lo = a.hi>>(n-64); + a.hi = 0; + } else { + a.lo = (a.lo>>n) | (a.hi<<(64-n)); + a.hi = a.hi>>n; + } + return a; +} + +/* returns a*b exactly. */ +static inline u128 mul64_128(uint64_t a, uint64_t b) +{ + u128 r; + uint64_t ahi = a>>32; + uint64_t alo = a&0xffffffff; + uint64_t bhi = b>>32; + uint64_t blo = b&0xffffffff; + uint64_t lo1 = ((ahi*blo)&0xffffffff) + ((alo*bhi)&0xffffffff) + (alo*blo>>32); + uint64_t lo2 = (alo*blo)&0xffffffff; + r.hi = ahi*bhi + (ahi*blo>>32) + (alo*bhi>>32) + (lo1>>32); + r.lo = (lo1<<32) + lo2; + return r; +} + +/* returns a*b*2^-128 - e, with error 0 <= e < 7. */ +static inline u128 mul128(u128 a, u128 b) +{ + u128 hi = mul64_128(a.hi, b.hi); + uint64_t m1 = mul64(a.hi, b.lo); + uint64_t m2 = mul64(a.lo, b.hi); + return add64(add64(hi, m1), m2); +} + +/* returns a*b % 2^128. */ +static inline u128 mul128_tail(u128 a, u128 b) +{ + u128 lo = mul64_128(a.lo, b.lo); + lo.hi += a.hi*b.lo + a.lo*b.hi; + return lo; +} + + +/* see sqrt.c for detailed comments. */ + +long double sqrtl(long double x) +{ + u128 ix, ml; + uint64_t top; + + ix = asu128(x); + top = ix.hi >> 48; + if (predict_false(top - 0x0001 >= 0x7fff - 0x0001)) { + /* x < 0x1p-16382 or inf or nan. */ + if (2*ix.hi == 0 && ix.lo == 0) + return x; + if (ix.hi == 0x7fff000000000000 && ix.lo == 0) + return x; + if (top >= 0x7fff) + return __math_invalidl(x); + /* x is subnormal, normalize it. */ + ix = asu128(x * 0x1p112); + top = ix.hi >> 48; + top -= 112; + } + + /* x = 4^e m; with int e and m in [1, 4) */ + int even = top & 1; + ml = lsh(ix, 15); + ml.hi |= 0x8000000000000000; + if (even) ml = rsh(ml, 1); + top = (top + 0x3fff) >> 1; + + /* r ~ 1/sqrt(m) */ + static const uint64_t three = 0xc0000000; + uint64_t r, s, d, u, i; + i = (ix.hi >> 42) % 128; + r = (uint32_t)__rsqrt_tab[i] << 16; + /* |r sqrt(m) - 1| < 0x1p-8 */ + s = mul32(ml.hi>>32, r); + d = mul32(s, r); + u = three - d; + r = mul32(u, r) << 1; + /* |r sqrt(m) - 1| < 0x1.7bp-16, switch to 64bit */ + r = r<<32; + s = mul64(ml.hi, r); + d = mul64(s, r); + u = (three<<32) - d; + r = mul64(u, r) << 1; + /* |r sqrt(m) - 1| < 0x1.a5p-31 */ + s = mul64(u, s) << 1; + d = mul64(s, r); + u = (three<<32) - d; + r = mul64(u, r) << 1; + /* |r sqrt(m) - 1| < 0x1.c001p-59, switch to 128bit */ + + static const u128 threel = {.hi=three<<32, .lo=0}; + u128 rl, sl, dl, ul; + rl.hi = r; + rl.lo = 0; + sl = mul128(ml, rl); + dl = mul128(sl, rl); + ul = sub128(threel, dl); + sl = mul128(ul, sl); /* repr: 3.125 */ + /* -0x1p-116 < s - sqrt(m) < 0x3.8001p-125 */ + sl = rsh(sub64(sl, 4), 125-(LDBL_MANT_DIG-1)); + /* s < sqrt(m) < s + 1 ULP + tiny */ + + long double y; + u128 d2, d1, d0; + d0 = sub128(lsh(ml, 2*(LDBL_MANT_DIG-1)-126), mul128_tail(sl,sl)); + d1 = sub128(sl, d0); + d2 = add128(add64(sl, 1), d1); + sl = add64(sl, d1.hi >> 63); + y = mkldbl(top, sl); + if (FENV_SUPPORT) { + /* handle rounding modes and inexact exception. */ + top = predict_false((d2.hi|d2.lo)==0) ? 0 : 1; + top |= ((d1.hi^d2.hi)&0x8000000000000000) >> 48; + y += mkldbl(top, (u128){0}); + } + return y; +} +#else +#error unsupported long double format +#endif diff --git a/src/math/x86_64/fabs.c b/src/math/x86_64/fabs.c new file mode 100644 index 000000000..165624777 --- /dev/null +++ b/src/math/x86_64/fabs.c @@ -0,0 +1,10 @@ +#include + +double fabs(double x) +{ + double t; + __asm__ ("pcmpeqd %0, %0" : "=x"(t)); // t = ~0 + __asm__ ("psrlq $1, %0" : "+x"(t)); // t >>= 1 + __asm__ ("andps %1, %0" : "+x"(x) : "x"(t)); // x &= t + return x; +} diff --git a/src/math/x86_64/fabs.s b/src/math/x86_64/fabs.s deleted file mode 100644 index 5715005e3..000000000 --- a/src/math/x86_64/fabs.s +++ /dev/null @@ -1,9 +0,0 @@ -.global fabs -.type fabs,@function -fabs: - xor %eax,%eax - dec %rax - shr %rax - movq %rax,%xmm1 - andpd %xmm1,%xmm0 - ret diff --git a/src/math/x86_64/fabsf.c b/src/math/x86_64/fabsf.c new file mode 100644 index 000000000..36ea7481f --- /dev/null +++ b/src/math/x86_64/fabsf.c @@ -0,0 +1,10 @@ +#include + +float fabsf(float x) +{ + float t; + __asm__ ("pcmpeqd %0, %0" : "=x"(t)); // t = ~0 + __asm__ ("psrld $1, %0" : "+x"(t)); // t >>= 1 + __asm__ ("andps %1, %0" : "+x"(x) : "x"(t)); // x &= t + return x; +} diff --git a/src/math/x86_64/fabsf.s b/src/math/x86_64/fabsf.s deleted file mode 100644 index 501a1f175..000000000 --- a/src/math/x86_64/fabsf.s +++ /dev/null @@ -1,7 +0,0 @@ -.global fabsf -.type fabsf,@function -fabsf: - mov $0x7fffffff,%eax - movq %rax,%xmm1 - andps %xmm1,%xmm0 - ret diff --git a/src/math/x86_64/fabsl.c b/src/math/x86_64/fabsl.c new file mode 100644 index 000000000..cc1c9ed9c --- /dev/null +++ b/src/math/x86_64/fabsl.c @@ -0,0 +1,7 @@ +#include + +long double fabsl(long double x) +{ + __asm__ ("fabs" : "+t"(x)); + return x; +} diff --git a/src/math/x86_64/fabsl.s b/src/math/x86_64/fabsl.s deleted file mode 100644 index 4e7ab525e..000000000 --- a/src/math/x86_64/fabsl.s +++ /dev/null @@ -1,6 +0,0 @@ -.global fabsl -.type fabsl,@function -fabsl: - fldt 8(%rsp) - fabs - ret diff --git a/src/math/x86_64/fmodl.c b/src/math/x86_64/fmodl.c new file mode 100644 index 000000000..3daeab060 --- /dev/null +++ b/src/math/x86_64/fmodl.c @@ -0,0 +1,9 @@ +#include + +long double fmodl(long double x, long double y) +{ + unsigned short fpsr; + do __asm__ ("fprem; fnstsw %%ax" : "+t"(x), "=a"(fpsr) : "u"(y)); + while (fpsr & 0x400); + return x; +} diff --git a/src/math/x86_64/fmodl.s b/src/math/x86_64/fmodl.s deleted file mode 100644 index ea07b402f..000000000 --- a/src/math/x86_64/fmodl.s +++ /dev/null @@ -1,11 +0,0 @@ -.global fmodl -.type fmodl,@function -fmodl: - fldt 24(%rsp) - fldt 8(%rsp) -1: fprem - fnstsw %ax - testb $4,%ah - jnz 1b - fstp %st(1) - ret diff --git a/src/math/x86_64/llrint.c b/src/math/x86_64/llrint.c new file mode 100644 index 000000000..dd38a7223 --- /dev/null +++ b/src/math/x86_64/llrint.c @@ -0,0 +1,8 @@ +#include + +long long llrint(double x) +{ + long long r; + __asm__ ("cvtsd2si %1, %0" : "=r"(r) : "x"(x)); + return r; +} diff --git a/src/math/x86_64/llrint.s b/src/math/x86_64/llrint.s deleted file mode 100644 index bf4764983..000000000 --- a/src/math/x86_64/llrint.s +++ /dev/null @@ -1,5 +0,0 @@ -.global llrint -.type llrint,@function -llrint: - cvtsd2si %xmm0,%rax - ret diff --git a/src/math/x86_64/llrintf.c b/src/math/x86_64/llrintf.c new file mode 100644 index 000000000..fc8625e88 --- /dev/null +++ b/src/math/x86_64/llrintf.c @@ -0,0 +1,8 @@ +#include + +long long llrintf(float x) +{ + long long r; + __asm__ ("cvtss2si %1, %0" : "=r"(r) : "x"(x)); + return r; +} diff --git a/src/math/x86_64/llrintf.s b/src/math/x86_64/llrintf.s deleted file mode 100644 index d7204ac0c..000000000 --- a/src/math/x86_64/llrintf.s +++ /dev/null @@ -1,5 +0,0 @@ -.global llrintf -.type llrintf,@function -llrintf: - cvtss2si %xmm0,%rax - ret diff --git a/src/math/x86_64/llrintl.c b/src/math/x86_64/llrintl.c new file mode 100644 index 000000000..c439ef28d --- /dev/null +++ b/src/math/x86_64/llrintl.c @@ -0,0 +1,8 @@ +#include + +long long llrintl(long double x) +{ + long long r; + __asm__ ("fistpll %0" : "=m"(r) : "t"(x) : "st"); + return r; +} diff --git a/src/math/x86_64/llrintl.s b/src/math/x86_64/llrintl.s deleted file mode 100644 index 1ec0817d3..000000000 --- a/src/math/x86_64/llrintl.s +++ /dev/null @@ -1,7 +0,0 @@ -.global llrintl -.type llrintl,@function -llrintl: - fldt 8(%rsp) - fistpll 8(%rsp) - mov 8(%rsp),%rax - ret diff --git a/src/math/x86_64/lrint.c b/src/math/x86_64/lrint.c new file mode 100644 index 000000000..a742fec64 --- /dev/null +++ b/src/math/x86_64/lrint.c @@ -0,0 +1,8 @@ +#include + +long lrint(double x) +{ + long r; + __asm__ ("cvtsd2si %1, %0" : "=r"(r) : "x"(x)); + return r; +} diff --git a/src/math/x86_64/lrint.s b/src/math/x86_64/lrint.s deleted file mode 100644 index 15fc2454b..000000000 --- a/src/math/x86_64/lrint.s +++ /dev/null @@ -1,5 +0,0 @@ -.global lrint -.type lrint,@function -lrint: - cvtsd2si %xmm0,%rax - ret diff --git a/src/math/x86_64/lrintf.c b/src/math/x86_64/lrintf.c new file mode 100644 index 000000000..2ba5639dc --- /dev/null +++ b/src/math/x86_64/lrintf.c @@ -0,0 +1,8 @@ +#include + +long lrintf(float x) +{ + long r; + __asm__ ("cvtss2si %1, %0" : "=r"(r) : "x"(x)); + return r; +} diff --git a/src/math/x86_64/lrintf.s b/src/math/x86_64/lrintf.s deleted file mode 100644 index 488423d21..000000000 --- a/src/math/x86_64/lrintf.s +++ /dev/null @@ -1,5 +0,0 @@ -.global lrintf -.type lrintf,@function -lrintf: - cvtss2si %xmm0,%rax - ret diff --git a/src/math/x86_64/lrintl.c b/src/math/x86_64/lrintl.c new file mode 100644 index 000000000..068e2e4d6 --- /dev/null +++ b/src/math/x86_64/lrintl.c @@ -0,0 +1,8 @@ +#include + +long lrintl(long double x) +{ + long r; + __asm__ ("fistpll %0" : "=m"(r) : "t"(x) : "st"); + return r; +} diff --git a/src/math/x86_64/lrintl.s b/src/math/x86_64/lrintl.s deleted file mode 100644 index d587b12b5..000000000 --- a/src/math/x86_64/lrintl.s +++ /dev/null @@ -1,7 +0,0 @@ -.global lrintl -.type lrintl,@function -lrintl: - fldt 8(%rsp) - fistpll 8(%rsp) - mov 8(%rsp),%rax - ret diff --git a/src/math/x86_64/remainderl.c b/src/math/x86_64/remainderl.c new file mode 100644 index 000000000..8cf75071e --- /dev/null +++ b/src/math/x86_64/remainderl.c @@ -0,0 +1,9 @@ +#include + +long double remainderl(long double x, long double y) +{ + unsigned short fpsr; + do __asm__ ("fprem1; fnstsw %%ax" : "+t"(x), "=a"(fpsr) : "u"(y)); + while (fpsr & 0x400); + return x; +} diff --git a/src/math/x86_64/remainderl.s b/src/math/x86_64/remainderl.s deleted file mode 100644 index cb3857b48..000000000 --- a/src/math/x86_64/remainderl.s +++ /dev/null @@ -1,11 +0,0 @@ -.global remainderl -.type remainderl,@function -remainderl: - fldt 24(%rsp) - fldt 8(%rsp) -1: fprem1 - fnstsw %ax - testb $4,%ah - jnz 1b - fstp %st(1) - ret diff --git a/src/math/x86_64/remquol.c b/src/math/x86_64/remquol.c new file mode 100644 index 000000000..60eef089f --- /dev/null +++ b/src/math/x86_64/remquol.c @@ -0,0 +1,32 @@ +#include + +long double remquol(long double x, long double y, int *quo) +{ + signed char *cx = (void *)&x, *cy = (void *)&y; + /* By ensuring that addresses of x and y cannot be discarded, + * this empty asm guides GCC into representing extraction of + * their sign bits as memory loads rather than making x and y + * not-address-taken internally and using bitfield operations, + * which in the end wouldn't work out, as extraction from FPU + * registers needs to go through memory anyway. This way GCC + * should manage to use incoming stack slots without spills. */ + __asm__ ("" :: "X"(cx), "X"(cy)); + + long double t = x; + unsigned fpsr; + do __asm__ ("fprem1; fnstsw %%ax" : "+t"(t), "=a"(fpsr) : "u"(y)); + while (fpsr & 0x400); + /* C0, C1, C3 flags in x87 status word carry low bits of quotient: + * 15 14 13 12 11 10 9 8 + * . C3 . . . C2 C1 C0 + * . b1 . . . 0 b0 b2 */ + unsigned char i = fpsr >> 8; + i = i>>4 | i<<4; + /* i[5:2] is now {b0 b2 ? b1}. Retrieve {0 b2 b1 b0} via + * in-register table lookup. */ + unsigned qbits = 0x7575313164642020 >> (i & 60); + qbits &= 7; + + *quo = (cx[9]^cy[9]) < 0 ? -qbits : qbits; + return t; +} diff --git a/src/math/x86_64/rintl.c b/src/math/x86_64/rintl.c new file mode 100644 index 000000000..e1a92077f --- /dev/null +++ b/src/math/x86_64/rintl.c @@ -0,0 +1,7 @@ +#include + +long double rintl(long double x) +{ + __asm__ ("frndint" : "+t"(x)); + return x; +} diff --git a/src/math/x86_64/rintl.s b/src/math/x86_64/rintl.s deleted file mode 100644 index 64e663cd2..000000000 --- a/src/math/x86_64/rintl.s +++ /dev/null @@ -1,6 +0,0 @@ -.global rintl -.type rintl,@function -rintl: - fldt 8(%rsp) - frndint - ret diff --git a/src/math/x86_64/sqrt.c b/src/math/x86_64/sqrt.c new file mode 100644 index 000000000..657e09e3b --- /dev/null +++ b/src/math/x86_64/sqrt.c @@ -0,0 +1,7 @@ +#include + +double sqrt(double x) +{ + __asm__ ("sqrtsd %1, %0" : "=x"(x) : "x"(x)); + return x; +} diff --git a/src/math/x86_64/sqrt.s b/src/math/x86_64/sqrt.s deleted file mode 100644 index d3c609f9f..000000000 --- a/src/math/x86_64/sqrt.s +++ /dev/null @@ -1,4 +0,0 @@ -.global sqrt -.type sqrt,@function -sqrt: sqrtsd %xmm0, %xmm0 - ret diff --git a/src/math/x86_64/sqrtf.c b/src/math/x86_64/sqrtf.c new file mode 100644 index 000000000..720baec60 --- /dev/null +++ b/src/math/x86_64/sqrtf.c @@ -0,0 +1,7 @@ +#include + +float sqrtf(float x) +{ + __asm__ ("sqrtss %1, %0" : "=x"(x) : "x"(x)); + return x; +} diff --git a/src/math/x86_64/sqrtf.s b/src/math/x86_64/sqrtf.s deleted file mode 100644 index eec48c609..000000000 --- a/src/math/x86_64/sqrtf.s +++ /dev/null @@ -1,4 +0,0 @@ -.global sqrtf -.type sqrtf,@function -sqrtf: sqrtss %xmm0, %xmm0 - ret diff --git a/src/math/x86_64/sqrtl.c b/src/math/x86_64/sqrtl.c new file mode 100644 index 000000000..864cfcc4f --- /dev/null +++ b/src/math/x86_64/sqrtl.c @@ -0,0 +1,7 @@ +#include + +long double sqrtl(long double x) +{ + __asm__ ("fsqrt" : "+t"(x)); + return x; +} diff --git a/src/math/x86_64/sqrtl.s b/src/math/x86_64/sqrtl.s deleted file mode 100644 index 23cd687d0..000000000 --- a/src/math/x86_64/sqrtl.s +++ /dev/null @@ -1,5 +0,0 @@ -.global sqrtl -.type sqrtl,@function -sqrtl: fldt 8(%rsp) - fsqrt - ret diff --git a/src/misc/getentropy.c b/src/misc/getentropy.c index d2f282ce8..651ea95f1 100644 --- a/src/misc/getentropy.c +++ b/src/misc/getentropy.c @@ -6,7 +6,7 @@ int getentropy(void *buffer, size_t len) { - int cs, ret; + int cs, ret = 0; char *pos = buffer; if (len > 256) { diff --git a/src/misc/ioctl.c b/src/misc/ioctl.c index 894775114..492828119 100644 --- a/src/misc/ioctl.c +++ b/src/misc/ioctl.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include "syscall.h" @@ -28,6 +29,12 @@ struct ioctl_compat_map { * number producing macros; only size of result is meaningful. */ #define new_misaligned(n) struct { int i; time_t t; char c[(n)-4]; } +struct v4l2_event { + uint32_t a; + uint64_t b[8]; + uint32_t c[2], ts[2], d[9]; +}; + static const struct ioctl_compat_map compat_map[] = { { SIOCGSTAMP, SIOCGSTAMP_OLD, 8, R, 0, OFFS(0, 4) }, { SIOCGSTAMPNS, SIOCGSTAMPNS_OLD, 8, R, 0, OFFS(0, 4) }, @@ -49,13 +56,14 @@ static const struct ioctl_compat_map compat_map[] = { { 0, 0, 8, WR, 1, OFFS(0,4) }, /* snd_pcm_mmap_control */ /* VIDIOC_QUERYBUF, VIDIOC_QBUF, VIDIOC_DQBUF, VIDIOC_PREPARE_BUF */ - { _IOWR('V', 9, new_misaligned(72)), _IOWR('V', 9, char[72]), 72, WR, 0, OFFS(20) }, - { _IOWR('V', 15, new_misaligned(72)), _IOWR('V', 15, char[72]), 72, WR, 0, OFFS(20) }, - { _IOWR('V', 17, new_misaligned(72)), _IOWR('V', 17, char[72]), 72, WR, 0, OFFS(20) }, - { _IOWR('V', 93, new_misaligned(72)), _IOWR('V', 93, char[72]), 72, WR, 0, OFFS(20) }, + { _IOWR('V', 9, new_misaligned(68)), _IOWR('V', 9, char[68]), 68, WR, 1, OFFS(20, 24) }, + { _IOWR('V', 15, new_misaligned(68)), _IOWR('V', 15, char[68]), 68, WR, 1, OFFS(20, 24) }, + { _IOWR('V', 17, new_misaligned(68)), _IOWR('V', 17, char[68]), 68, WR, 1, OFFS(20, 24) }, + { _IOWR('V', 93, new_misaligned(68)), _IOWR('V', 93, char[68]), 68, WR, 1, OFFS(20, 24) }, /* VIDIOC_DQEVENT */ - { _IOR('V', 89, new_misaligned(96)), _IOR('V', 89, char[96]), 96, R, 0, OFFS(76,80) }, + { _IOR('V', 89, new_misaligned(120)), _IOR('V', 89, struct v4l2_event), sizeof(struct v4l2_event), + R, 0, OFFS(offsetof(struct v4l2_event, ts[0]), offsetof(struct v4l2_event, ts[1])) }, /* VIDIOC_OMAP3ISP_STAT_REQ */ { _IOWR('V', 192+6, char[32]), _IOWR('V', 192+6, char[24]), 22, WR, 0, OFFS(0,4) }, diff --git a/src/misc/nftw.c b/src/misc/nftw.c index 0a4641007..8dcff7fef 100644 --- a/src/misc/nftw.c +++ b/src/misc/nftw.c @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -26,6 +27,8 @@ static int do_nftw(char *path, int (*fn)(const char *, const struct stat *, int, struct history new; int type; int r; + int dfd; + int err; struct FTW lev; if ((flags & FTW_PHYS) ? lstat(path, &st) : stat(path, &st) < 0) { @@ -34,8 +37,7 @@ static int do_nftw(char *path, int (*fn)(const char *, const struct stat *, int, else if (errno != EACCES) return -1; else type = FTW_NS; } else if (S_ISDIR(st.st_mode)) { - if (access(path, R_OK) < 0) type = FTW_DNR; - else if (flags & FTW_DEPTH) type = FTW_DP; + if (flags & FTW_DEPTH) type = FTW_DP; else type = FTW_D; } else if (S_ISLNK(st.st_mode)) { if (flags & FTW_PHYS) type = FTW_SL; @@ -63,6 +65,13 @@ static int do_nftw(char *path, int (*fn)(const char *, const struct stat *, int, lev.base = k; } + if (type == FTW_D || type == FTW_DP) { + dfd = open(path, O_RDONLY); + err = errno; + if (dfd < 0 && err == EACCES) type = FTW_DNR; + if (!fd_limit) close(dfd); + } + if (!(flags & FTW_DEPTH) && (r=fn(path, &st, type, &lev))) return r; @@ -71,7 +80,11 @@ static int do_nftw(char *path, int (*fn)(const char *, const struct stat *, int, return 0; if ((type == FTW_D || type == FTW_DP) && fd_limit) { - DIR *d = opendir(path); + if (dfd < 0) { + errno = err; + return -1; + } + DIR *d = fdopendir(dfd); if (d) { struct dirent *de; while ((de = readdir(d))) { @@ -92,7 +105,8 @@ static int do_nftw(char *path, int (*fn)(const char *, const struct stat *, int, } } closedir(d); - } else if (errno != EACCES) { + } else { + close(dfd); return -1; } } diff --git a/src/misc/realpath.c b/src/misc/realpath.c index d2708e59d..db8b74dc8 100644 --- a/src/misc/realpath.c +++ b/src/misc/realpath.c @@ -1,43 +1,156 @@ #include #include -#include -#include #include #include #include -#include "syscall.h" + +static size_t slash_len(const char *s) +{ + const char *s0 = s; + while (*s == '/') s++; + return s-s0; +} char *realpath(const char *restrict filename, char *restrict resolved) { - int fd; - ssize_t r; - struct stat st1, st2; - char buf[15+3*sizeof(int)]; - char tmp[PATH_MAX]; + char stack[PATH_MAX+1]; + char output[PATH_MAX]; + size_t p, q, l, l0, cnt=0, nup=0; + int check_dir=0; if (!filename) { errno = EINVAL; return 0; } + l = strnlen(filename, sizeof stack); + if (!l) { + errno = ENOENT; + return 0; + } + if (l >= PATH_MAX) goto toolong; + p = sizeof stack - l - 1; + q = 0; + memcpy(stack+p, filename, l+1); + + /* Main loop. Each iteration pops the next part from stack of + * remaining path components and consumes any slashes that follow. + * If not a link, it's moved to output; if a link, contents are + * pushed to the stack. */ +restart: + for (; ; p+=slash_len(stack+p)) { + /* If stack starts with /, the whole component is / or // + * and the output state must be reset. */ + if (stack[p] == '/') { + check_dir=0; + nup=0; + q=0; + output[q++] = '/'; + p++; + /* Initial // is special. */ + if (stack[p] == '/' && stack[p+1] != '/') + output[q++] = '/'; + continue; + } + + char *z = __strchrnul(stack+p, '/'); + l0 = l = z-(stack+p); - fd = sys_open(filename, O_PATH|O_NONBLOCK|O_CLOEXEC); - if (fd < 0) return 0; - __procfdname(buf, fd); + if (!l && !check_dir) break; - r = readlink(buf, tmp, sizeof tmp - 1); - if (r < 0) goto err; - tmp[r] = 0; + /* Skip any . component but preserve check_dir status. */ + if (l==1 && stack[p]=='.') { + p += l; + continue; + } - fstat(fd, &st1); - r = stat(tmp, &st2); - if (r<0 || st1.st_dev != st2.st_dev || st1.st_ino != st2.st_ino) { - if (!r) errno = ELOOP; - goto err; + /* Copy next component onto output at least temporarily, to + * call readlink, but wait to advance output position until + * determining it's not a link. */ + if (q && output[q-1] != '/') { + if (!p) goto toolong; + stack[--p] = '/'; + l++; + } + if (q+l >= PATH_MAX) goto toolong; + memcpy(output+q, stack+p, l); + output[q+l] = 0; + p += l; + + int up = 0; + if (l0==2 && stack[p-2]=='.' && stack[p-1]=='.') { + up = 1; + /* Any non-.. path components we could cancel start + * after nup repetitions of the 3-byte string "../"; + * if there are none, accumulate .. components to + * later apply to cwd, if needed. */ + if (q <= 3*nup) { + nup++; + q += l; + continue; + } + /* When previous components are already known to be + * directories, processing .. can skip readlink. */ + if (!check_dir) goto skip_readlink; + } + ssize_t k = readlink(output, stack, p); + if (k==p) goto toolong; + if (!k) { + errno = ENOENT; + return 0; + } + if (k<0) { + if (errno != EINVAL) return 0; +skip_readlink: + check_dir = 0; + if (up) { + while(q && output[q-1]!='/') q--; + if (q>1 && (q>2 || output[0]!='/')) q--; + continue; + } + if (l0) q += l; + check_dir = stack[p]; + continue; + } + if (++cnt == SYMLOOP_MAX) { + errno = ELOOP; + return 0; + } + + /* If link contents end in /, strip any slashes already on + * stack to avoid /->// or //->/// or spurious toolong. */ + if (stack[k-1]=='/') while (stack[p]=='/') p++; + p -= k; + memmove(stack+p, stack, k); + + /* Skip the stack advancement in case we have a new + * absolute base path. */ + goto restart; } - __syscall(SYS_close, fd); - return resolved ? strcpy(resolved, tmp) : strdup(tmp); -err: - __syscall(SYS_close, fd); + output[q] = 0; + + if (output[0] != '/') { + if (!getcwd(stack, sizeof stack)) return 0; + l = strlen(stack); + /* Cancel any initial .. components. */ + p = 0; + while (nup--) { + while(l>1 && stack[l-1]!='/') l--; + if (l>1) l--; + p += 2; + if (p= PATH_MAX) goto toolong; + memmove(output + l, output + p, q - p + 1); + memcpy(output, stack, l); + q = l + q-p; + } + + if (resolved) return memcpy(resolved, output, q+1); + else return strdup(output); + +toolong: + errno = ENAMETOOLONG; return 0; } diff --git a/src/misc/setrlimit.c b/src/misc/setrlimit.c index 7a66ab297..8340aee09 100644 --- a/src/misc/setrlimit.c +++ b/src/misc/setrlimit.c @@ -6,25 +6,8 @@ #define MIN(a, b) ((a)<(b) ? (a) : (b)) #define FIX(x) do{ if ((x)>=SYSCALL_RLIM_INFINITY) (x)=RLIM_INFINITY; }while(0) -static int __setrlimit(int resource, const struct rlimit *rlim) -{ - unsigned long k_rlim[2]; - struct rlimit tmp; - if (SYSCALL_RLIM_INFINITY != RLIM_INFINITY) { - tmp = *rlim; - FIX(tmp.rlim_cur); - FIX(tmp.rlim_max); - rlim = &tmp; - } - int ret = __syscall(SYS_prlimit64, 0, resource, rlim, 0); - if (ret != -ENOSYS) return ret; - k_rlim[0] = MIN(rlim->rlim_cur, MIN(-1UL, SYSCALL_RLIM_INFINITY)); - k_rlim[1] = MIN(rlim->rlim_max, MIN(-1UL, SYSCALL_RLIM_INFINITY)); - return __syscall(SYS_setrlimit, resource, k_rlim); -} - struct ctx { - const struct rlimit *rlim; + unsigned long lim[2]; int res; int err; }; @@ -33,12 +16,26 @@ static void do_setrlimit(void *p) { struct ctx *c = p; if (c->err>0) return; - c->err = -__setrlimit(c->res, c->rlim); + c->err = -__syscall(SYS_setrlimit, c->res, c->lim); } int setrlimit(int resource, const struct rlimit *rlim) { - struct ctx c = { .res = resource, .rlim = rlim, .err = -1 }; + struct rlimit tmp; + if (SYSCALL_RLIM_INFINITY != RLIM_INFINITY) { + tmp = *rlim; + FIX(tmp.rlim_cur); + FIX(tmp.rlim_max); + rlim = &tmp; + } + int ret = __syscall(SYS_prlimit64, 0, resource, rlim, 0); + if (ret != -ENOSYS) return __syscall_ret(ret); + + struct ctx c = { + .lim[0] = MIN(rlim->rlim_cur, MIN(-1UL, SYSCALL_RLIM_INFINITY)), + .lim[1] = MIN(rlim->rlim_max, MIN(-1UL, SYSCALL_RLIM_INFINITY)), + .res = resource, .err = -1 + }; __synccall(do_setrlimit, &c); if (c.err) { if (c.err>0) errno = c.err; diff --git a/src/misc/syslog.c b/src/misc/syslog.c index 13d4b0a6d..7dc0c1be5 100644 --- a/src/misc/syslog.c +++ b/src/misc/syslog.c @@ -10,6 +10,7 @@ #include #include #include "lock.h" +#include "fork_impl.h" static volatile int lock[1]; static char log_ident[32]; @@ -17,6 +18,7 @@ static int log_opt; static int log_facility = LOG_USER; static int log_mask = 0xff; static int log_fd = -1; +volatile int *const __syslog_lockptr = lock; int setlogmask(int maskpri) { diff --git a/src/multibyte/wcsnrtombs.c b/src/multibyte/wcsnrtombs.c index 676932b5d..95e25e708 100644 --- a/src/multibyte/wcsnrtombs.c +++ b/src/multibyte/wcsnrtombs.c @@ -1,41 +1,33 @@ #include +#include +#include size_t wcsnrtombs(char *restrict dst, const wchar_t **restrict wcs, size_t wn, size_t n, mbstate_t *restrict st) { - size_t l, cnt=0, n2; - char *s, buf[256]; const wchar_t *ws = *wcs; - const wchar_t *tmp_ws; - - if (!dst) s = buf, n = sizeof buf; - else s = dst; - - while ( ws && n && ( (n2=wn)>=n || n2>32 ) ) { - if (n2>=n) n2=n; - tmp_ws = ws; - l = wcsrtombs(s, &ws, n2, 0); - if (!(l+1)) { - cnt = l; - n = 0; + size_t cnt = 0; + if (!dst) n=0; + while (ws && wn) { + char tmp[MB_LEN_MAX]; + size_t l = wcrtomb(nn) break; + memcpy(dst, tmp, l); + } + dst += l; n -= l; } - wn = ws ? wn - (ws - tmp_ws) : 0; - cnt += l; - } - if (ws) while (n && wn) { - l = wcrtomb(s, *ws, 0); - if ((l+1)<=1) { - if (!l) ws = 0; - else cnt = l; + if (!*ws) { + ws = 0; break; } - ws++; wn--; - /* safe - this loop runs fewer than sizeof(buf) times */ - s+=l; n-=l; + ws++; + wn--; cnt += l; } if (dst) *wcs = ws; diff --git a/src/network/getnameinfo.c b/src/network/getnameinfo.c index f77e73ade..949e18115 100644 --- a/src/network/getnameinfo.c +++ b/src/network/getnameinfo.c @@ -158,6 +158,7 @@ int getnameinfo(const struct sockaddr *restrict sa, socklen_t sl, unsigned char query[18+PTR_MAX], reply[512]; int qlen = __res_mkquery(0, ptr, 1, RR_PTR, 0, 0, 0, query, sizeof query); + query[3] = 0; /* don't need AD flag */ int rlen = __res_send(query, qlen, reply, sizeof reply); buf[0] = 0; if (rlen > 0) diff --git a/src/network/h_errno.c b/src/network/h_errno.c index 4f700ceaf..638f77180 100644 --- a/src/network/h_errno.c +++ b/src/network/h_errno.c @@ -1,9 +1,11 @@ #include +#include "pthread_impl.h" #undef h_errno int h_errno; int *__h_errno_location(void) { - return &h_errno; + if (!__pthread_self()->stack) return &h_errno; + return &__pthread_self()->h_errno_val; } diff --git a/src/network/herror.c b/src/network/herror.c index 65f25ff3f..87f8cff4f 100644 --- a/src/network/herror.c +++ b/src/network/herror.c @@ -4,5 +4,5 @@ void herror(const char *msg) { - fprintf(stderr, "%s%s%s", msg?msg:"", msg?": ":"", hstrerror(h_errno)); + fprintf(stderr, "%s%s%s\n", msg?msg:"", msg?": ":"", hstrerror(h_errno)); } diff --git a/src/network/lookup_name.c b/src/network/lookup_name.c index c93263a9d..aa558c197 100644 --- a/src/network/lookup_name.c +++ b/src/network/lookup_name.c @@ -50,7 +50,7 @@ static int name_from_hosts(struct address buf[static MAXADDRS], char canon[stati { char line[512]; size_t l = strlen(name); - int cnt = 0, badfam = 0; + int cnt = 0, badfam = 0, have_canon = 0; unsigned char _buf[1032]; FILE _f, *f = __fopen_rb_ca("/etc/hosts", &_f, _buf, sizeof _buf); if (!f) switch (errno) { @@ -80,14 +80,19 @@ static int name_from_hosts(struct address buf[static MAXADDRS], char canon[stati continue; default: badfam = EAI_NONAME; - continue; + break; } + if (have_canon) continue; + /* Extract first name as canonical name */ for (; *p && isspace(*p); p++); for (z=p; *z && !isspace(*z); z++); *z = 0; - if (is_valid_hostname(p)) memcpy(canon, p, z-p+1); + if (is_valid_hostname(p)) { + have_canon = 1; + memcpy(canon, p, z-p+1); + } } __fclose_ca(f); return cnt ? cnt : badfam; @@ -149,6 +154,7 @@ static int name_from_dns(struct address buf[static MAXADDRS], char canon[static 0, 0, 0, qbuf[nq], sizeof *qbuf); if (qlens[nq] == -1) return EAI_NONAME; + qbuf[nq][3] = 0; /* don't need AD flag */ nq++; } } @@ -156,14 +162,17 @@ static int name_from_dns(struct address buf[static MAXADDRS], char canon[static if (__res_msend_rc(nq, qp, qlens, ap, alens, sizeof *abuf, conf) < 0) return EAI_SYSTEM; + for (i=0; i #include @@ -6,7 +7,20 @@ int res_query(const char *name, int class, int type, unsigned char *dest, int le unsigned char q[280]; int ql = __res_mkquery(0, name, class, type, 0, 0, 0, q, sizeof q); if (ql < 0) return ql; - return __res_send(q, ql, dest, len); + int r = __res_send(q, ql, dest, len); + if (r<12) { + h_errno = TRY_AGAIN; + return -1; + } + if ((dest[3] & 15) == 3) { + h_errno = HOST_NOT_FOUND; + return -1; + } + if ((dest[3] & 15) == 0 && !dest[6] && !dest[7]) { + h_errno = NO_DATA; + return -1; + } + return r; } weak_alias(res_query, res_search); diff --git a/src/network/res_send.c b/src/network/res_send.c index b9cea0bfd..ee4abf1f1 100644 --- a/src/network/res_send.c +++ b/src/network/res_send.c @@ -3,7 +3,7 @@ int __res_send(const unsigned char *msg, int msglen, unsigned char *answer, int anslen) { int r = __res_msend(1, &msg, &msglen, &answer, &anslen, anslen); - return r<0 ? r : anslen; + return r<0 || !anslen ? -1 : anslen; } weak_alias(__res_send, res_send); diff --git a/src/passwd/getgrouplist.c b/src/passwd/getgrouplist.c index 43e518245..301824cec 100644 --- a/src/passwd/getgrouplist.c +++ b/src/passwd/getgrouplist.c @@ -31,7 +31,8 @@ int getgrouplist(const char *user, gid_t gid, gid_t *groups, int *ngroups) if (resp[INITGRFOUND]) { nscdbuf = calloc(resp[INITGRNGRPS], sizeof(uint32_t)); if (!nscdbuf) goto cleanup; - if (!fread(nscdbuf, sizeof(*nscdbuf)*resp[INITGRNGRPS], 1, f)) { + size_t nbytes = sizeof(*nscdbuf)*resp[INITGRNGRPS]; + if (nbytes && !fread(nscdbuf, nbytes, 1, f)) { if (!ferror(f)) errno = EIO; goto cleanup; } diff --git a/src/prng/random.c b/src/prng/random.c index 633a17f69..d3780fa7e 100644 --- a/src/prng/random.c +++ b/src/prng/random.c @@ -1,6 +1,7 @@ #include #include #include "lock.h" +#include "fork_impl.h" /* this code uses the same lagged fibonacci generator as the @@ -23,6 +24,7 @@ static int i = 3; static int j = 0; static uint32_t *x = init+1; static volatile int lock[1]; +volatile int *const __random_lockptr = lock; static uint32_t lcg31(uint32_t x) { return (1103515245*x + 12345) & 0x7fffffff; diff --git a/src/process/_Fork.c b/src/process/_Fork.c new file mode 100644 index 000000000..da0638681 --- /dev/null +++ b/src/process/_Fork.c @@ -0,0 +1,38 @@ +#include +#include +#include "syscall.h" +#include "libc.h" +#include "lock.h" +#include "pthread_impl.h" +#include "aio_impl.h" + +static void dummy(int x) { } +weak_alias(dummy, __aio_atfork); + +pid_t _Fork(void) +{ + pid_t ret; + sigset_t set; + __block_all_sigs(&set); + __aio_atfork(-1); + LOCK(__abort_lock); +#ifdef SYS_fork + ret = __syscall(SYS_fork); +#else + ret = __syscall(SYS_clone, SIGCHLD, 0); +#endif + if (!ret) { + pthread_t self = __pthread_self(); + self->tid = __syscall(SYS_gettid); + self->robust_list.off = 0; + self->robust_list.pending = 0; + self->next = self->prev = self; + __thread_list_lock = 0; + libc.threads_minus_1 = 0; + if (libc.need_locks) libc.need_locks = -1; + } + UNLOCK(__abort_lock); + __aio_atfork(!ret); + __restore_sigs(&set); + return __syscall_ret(ret); +} diff --git a/src/process/fork.c b/src/process/fork.c index fb42478ae..54bc28920 100644 --- a/src/process/fork.c +++ b/src/process/fork.c @@ -1,37 +1,86 @@ #include -#include -#include -#include "syscall.h" +#include #include "libc.h" +#include "lock.h" #include "pthread_impl.h" +#include "fork_impl.h" -static void dummy(int x) -{ -} +static volatile int *const dummy_lockptr = 0; + +weak_alias(dummy_lockptr, __at_quick_exit_lockptr); +weak_alias(dummy_lockptr, __atexit_lockptr); +weak_alias(dummy_lockptr, __dlerror_lockptr); +weak_alias(dummy_lockptr, __gettext_lockptr); +weak_alias(dummy_lockptr, __locale_lockptr); +weak_alias(dummy_lockptr, __random_lockptr); +weak_alias(dummy_lockptr, __sem_open_lockptr); +weak_alias(dummy_lockptr, __stdio_ofl_lockptr); +weak_alias(dummy_lockptr, __syslog_lockptr); +weak_alias(dummy_lockptr, __timezone_lockptr); +weak_alias(dummy_lockptr, __bump_lockptr); + +weak_alias(dummy_lockptr, __vmlock_lockptr); +static volatile int *const *const atfork_locks[] = { + &__at_quick_exit_lockptr, + &__atexit_lockptr, + &__dlerror_lockptr, + &__gettext_lockptr, + &__locale_lockptr, + &__random_lockptr, + &__sem_open_lockptr, + &__stdio_ofl_lockptr, + &__syslog_lockptr, + &__timezone_lockptr, + &__bump_lockptr, +}; + +static void dummy(int x) { } weak_alias(dummy, __fork_handler); +weak_alias(dummy, __malloc_atfork); +weak_alias(dummy, __ldso_atfork); + +static void dummy_0(void) { } +weak_alias(dummy_0, __tl_lock); +weak_alias(dummy_0, __tl_unlock); pid_t fork(void) { - pid_t ret; sigset_t set; __fork_handler(-1); - __block_all_sigs(&set); -#ifdef SYS_fork - ret = __syscall(SYS_fork); -#else - ret = __syscall(SYS_clone, SIGCHLD, 0); -#endif - if (!ret) { - pthread_t self = __pthread_self(); - self->tid = __syscall(SYS_gettid); - self->robust_list.off = 0; - self->robust_list.pending = 0; - self->next = self->prev = self; - __thread_list_lock = 0; - libc.threads_minus_1 = 0; + __block_app_sigs(&set); + int need_locks = libc.need_locks > 0; + if (need_locks) { + __ldso_atfork(-1); + __inhibit_ptc(); + for (int i=0; inext; + pid_t ret = _Fork(); + int errno_save = errno; + if (need_locks) { + if (!ret) { + for (pthread_t td=next; td!=self; td=td->next) + td->tid = -1; + if (__vmlock_lockptr) { + __vmlock_lockptr[0] = 0; + __vmlock_lockptr[1] = 0; + } + } + __tl_unlock(); + __malloc_atfork(!ret); + for (int i=0; i #include #include "syscall.h" +#include "lock.h" #include "pthread_impl.h" #include "fdop.h" @@ -170,9 +171,6 @@ int posix_spawn(pid_t *restrict res, const char *restrict path, int ec=0, cs; struct args args; - if (pipe2(args.p, O_CLOEXEC)) - return errno; - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cs); args.path = path; @@ -182,9 +180,20 @@ int posix_spawn(pid_t *restrict res, const char *restrict path, args.envp = envp; pthread_sigmask(SIG_BLOCK, SIGALL_SET, &args.oldmask); + /* The lock guards both against seeing a SIGABRT disposition change + * by abort and against leaking the pipe fd to fork-without-exec. */ + LOCK(__abort_lock); + + if (pipe2(args.p, O_CLOEXEC)) { + UNLOCK(__abort_lock); + ec = errno; + goto fail; + } + pid = __clone(child, stack+sizeof stack, CLONE_VM|CLONE_VFORK|SIGCHLD, &args); close(args.p[1]); + UNLOCK(__abort_lock); if (pid > 0) { if (read(args.p[0], &ec, sizeof ec) != sizeof ec) ec = 0; @@ -197,6 +206,7 @@ int posix_spawn(pid_t *restrict res, const char *restrict path, if (!ec && res) *res = pid; +fail: pthread_sigmask(SIG_SETMASK, &args.oldmask, 0); pthread_setcancelstate(cs, 0); diff --git a/src/process/posix_spawn_file_actions_addclose.c b/src/process/posix_spawn_file_actions_addclose.c index cdda59799..0c2ef8fa3 100644 --- a/src/process/posix_spawn_file_actions_addclose.c +++ b/src/process/posix_spawn_file_actions_addclose.c @@ -5,6 +5,7 @@ int posix_spawn_file_actions_addclose(posix_spawn_file_actions_t *fa, int fd) { + if (fd < 0) return EBADF; struct fdop *op = malloc(sizeof *op); if (!op) return ENOMEM; op->cmd = FDOP_CLOSE; diff --git a/src/process/posix_spawn_file_actions_adddup2.c b/src/process/posix_spawn_file_actions_adddup2.c index 0367498fd..addca4d4f 100644 --- a/src/process/posix_spawn_file_actions_adddup2.c +++ b/src/process/posix_spawn_file_actions_adddup2.c @@ -5,6 +5,7 @@ int posix_spawn_file_actions_adddup2(posix_spawn_file_actions_t *fa, int srcfd, int fd) { + if (srcfd < 0 || fd < 0) return EBADF; struct fdop *op = malloc(sizeof *op); if (!op) return ENOMEM; op->cmd = FDOP_DUP2; diff --git a/src/process/posix_spawn_file_actions_addfchdir.c b/src/process/posix_spawn_file_actions_addfchdir.c index 436c683d2..e89ede8c3 100644 --- a/src/process/posix_spawn_file_actions_addfchdir.c +++ b/src/process/posix_spawn_file_actions_addfchdir.c @@ -6,6 +6,7 @@ int posix_spawn_file_actions_addfchdir_np(posix_spawn_file_actions_t *fa, int fd) { + if (fd < 0) return EBADF; struct fdop *op = malloc(sizeof *op); if (!op) return ENOMEM; op->cmd = FDOP_FCHDIR; diff --git a/src/process/posix_spawn_file_actions_addopen.c b/src/process/posix_spawn_file_actions_addopen.c index 368922c76..82bbcec9e 100644 --- a/src/process/posix_spawn_file_actions_addopen.c +++ b/src/process/posix_spawn_file_actions_addopen.c @@ -6,6 +6,7 @@ int posix_spawn_file_actions_addopen(posix_spawn_file_actions_t *restrict fa, int fd, const char *restrict path, int flags, mode_t mode) { + if (fd < 0) return EBADF; struct fdop *op = malloc(sizeof *op + strlen(path) + 1); if (!op) return ENOMEM; op->cmd = FDOP_OPEN; diff --git a/src/setjmp/aarch64/longjmp.s b/src/setjmp/aarch64/longjmp.s index 7c4655fa9..0af9c50ee 100644 --- a/src/setjmp/aarch64/longjmp.s +++ b/src/setjmp/aarch64/longjmp.s @@ -18,7 +18,6 @@ longjmp: ldp d12, d13, [x0,#144] ldp d14, d15, [x0,#160] - mov x0, x1 - cbnz x1, 1f - mov x0, #1 -1: br x30 + cmp w1, 0 + csinc w0, w1, wzr, ne + br x30 diff --git a/src/setjmp/i386/longjmp.s b/src/setjmp/i386/longjmp.s index 772d28ddb..8188f06bc 100644 --- a/src/setjmp/i386/longjmp.s +++ b/src/setjmp/i386/longjmp.s @@ -6,15 +6,11 @@ _longjmp: longjmp: mov 4(%esp),%edx mov 8(%esp),%eax - test %eax,%eax - jnz 1f - inc %eax -1: + cmp $1,%eax + adc $0, %al mov (%edx),%ebx mov 4(%edx),%esi mov 8(%edx),%edi mov 12(%edx),%ebp - mov 16(%edx),%ecx - mov %ecx,%esp - mov 20(%edx),%ecx - jmp *%ecx + mov 16(%edx),%esp + jmp *20(%edx) diff --git a/src/setjmp/x32/longjmp.s b/src/setjmp/x32/longjmp.s index e175a4b96..1b2661c3e 100644 --- a/src/setjmp/x32/longjmp.s +++ b/src/setjmp/x32/longjmp.s @@ -5,18 +5,14 @@ .type longjmp,@function _longjmp: longjmp: - mov %rsi,%rax /* val will be longjmp return */ - test %rax,%rax - jnz 1f - inc %rax /* if val==0, val=1 per longjmp semantics */ -1: + xor %eax,%eax + cmp $1,%esi /* CF = val ? 0 : 1 */ + adc %esi,%eax /* eax = val + !val */ mov (%rdi),%rbx /* rdi is the jmp_buf, restore regs from it */ mov 8(%rdi),%rbp mov 16(%rdi),%r12 mov 24(%rdi),%r13 mov 32(%rdi),%r14 mov 40(%rdi),%r15 - mov 48(%rdi),%rdx /* this ends up being the stack pointer */ - mov %rdx,%rsp - mov 56(%rdi),%rdx /* this is the instruction pointer */ - jmp *%rdx /* goto saved address without altering rsp */ + mov 48(%rdi),%rsp + jmp *56(%rdi) /* goto saved address without altering rsp */ diff --git a/src/setjmp/x32/setjmp.s b/src/setjmp/x32/setjmp.s index 98f58b8d6..d95e48535 100644 --- a/src/setjmp/x32/setjmp.s +++ b/src/setjmp/x32/setjmp.s @@ -18,5 +18,5 @@ setjmp: mov %rdx,48(%rdi) mov (%rsp),%rdx /* save return addr ptr for new rip */ mov %rdx,56(%rdi) - xor %rax,%rax /* always return 0 */ + xor %eax,%eax /* always return 0 */ ret diff --git a/src/setjmp/x86_64/longjmp.s b/src/setjmp/x86_64/longjmp.s index e175a4b96..1b2661c3e 100644 --- a/src/setjmp/x86_64/longjmp.s +++ b/src/setjmp/x86_64/longjmp.s @@ -5,18 +5,14 @@ .type longjmp,@function _longjmp: longjmp: - mov %rsi,%rax /* val will be longjmp return */ - test %rax,%rax - jnz 1f - inc %rax /* if val==0, val=1 per longjmp semantics */ -1: + xor %eax,%eax + cmp $1,%esi /* CF = val ? 0 : 1 */ + adc %esi,%eax /* eax = val + !val */ mov (%rdi),%rbx /* rdi is the jmp_buf, restore regs from it */ mov 8(%rdi),%rbp mov 16(%rdi),%r12 mov 24(%rdi),%r13 mov 32(%rdi),%r14 mov 40(%rdi),%r15 - mov 48(%rdi),%rdx /* this ends up being the stack pointer */ - mov %rdx,%rsp - mov 56(%rdi),%rdx /* this is the instruction pointer */ - jmp *%rdx /* goto saved address without altering rsp */ + mov 48(%rdi),%rsp + jmp *56(%rdi) /* goto saved address without altering rsp */ diff --git a/src/setjmp/x86_64/setjmp.s b/src/setjmp/x86_64/setjmp.s index 98f58b8d6..d95e48535 100644 --- a/src/setjmp/x86_64/setjmp.s +++ b/src/setjmp/x86_64/setjmp.s @@ -18,5 +18,5 @@ setjmp: mov %rdx,48(%rdi) mov (%rsp),%rdx /* save return addr ptr for new rip */ mov %rdx,56(%rdi) - xor %rax,%rax /* always return 0 */ + xor %eax,%eax /* always return 0 */ ret diff --git a/src/signal/sigaction.c b/src/signal/sigaction.c index c109bea0c..2203471b2 100644 --- a/src/signal/sigaction.c +++ b/src/signal/sigaction.c @@ -7,12 +7,6 @@ #include "lock.h" #include "ksigaction.h" -static volatile int dummy_lock[1] = { 0 }; - -extern hidden volatile int __abort_lock[1]; - -weak_alias(dummy_lock, __abort_lock); - static int unmask_done; static unsigned long handler_set[_NSIG/(8*sizeof(long))]; @@ -26,7 +20,6 @@ volatile int __eintr_valid_flag; int __libc_sigaction(int sig, const struct sigaction *restrict sa, struct sigaction *restrict old) { struct k_sigaction ksa, ksa_old; - unsigned long set[_NSIG/(8*sizeof(long))]; if (sa) { if ((uintptr_t)sa->sa_handler > 1UL) { a_or_l(handler_set+(sig-1)/(8*sizeof(long)), @@ -50,24 +43,12 @@ int __libc_sigaction(int sig, const struct sigaction *restrict sa, struct sigact a_store(&__eintr_valid_flag, 1); } } - /* Changing the disposition of SIGABRT to anything but - * SIG_DFL requires a lock, so that it cannot be changed - * while abort is terminating the process after simply - * calling raise(SIGABRT) failed to do so. */ - if (sa->sa_handler != SIG_DFL && sig == SIGABRT) { - __block_all_sigs(&set); - LOCK(__abort_lock); - } ksa.handler = sa->sa_handler; ksa.flags = sa->sa_flags | SA_RESTORER; ksa.restorer = (sa->sa_flags & SA_SIGINFO) ? __restore_rt : __restore; memcpy(&ksa.mask, &sa->sa_mask, _NSIG/8); } int r = __syscall(SYS_rt_sigaction, sig, sa?&ksa:0, old?&ksa_old:0, _NSIG/8); - if (sig == SIGABRT && sa && sa->sa_handler != SIG_DFL) { - UNLOCK(__abort_lock); - __restore_sigs(&set); - } if (old && !r) { old->sa_handler = ksa_old.handler; old->sa_flags = ksa_old.flags; @@ -78,11 +59,26 @@ int __libc_sigaction(int sig, const struct sigaction *restrict sa, struct sigact int __sigaction(int sig, const struct sigaction *restrict sa, struct sigaction *restrict old) { + unsigned long set[_NSIG/(8*sizeof(long))]; + if (sig-32U < 3 || sig-1U >= _NSIG-1) { errno = EINVAL; return -1; } - return __libc_sigaction(sig, sa, old); + + /* Doing anything with the disposition of SIGABRT requires a lock, + * so that it cannot be changed while abort is terminating the + * process and so any change made by abort can't be observed. */ + if (sig == SIGABRT) { + __block_all_sigs(&set); + LOCK(__abort_lock); + } + int r = __libc_sigaction(sig, sa, old); + if (sig == SIGABRT) { + UNLOCK(__abort_lock); + __restore_sigs(&set); + } + return r; } weak_alias(__sigaction, sigaction); diff --git a/src/stdio/__stdio_close.c b/src/stdio/__stdio_close.c index 79452bdb6..302913285 100644 --- a/src/stdio/__stdio_close.c +++ b/src/stdio/__stdio_close.c @@ -1,4 +1,5 @@ #include "stdio_impl.h" +#include "aio_impl.h" static int dummy(int fd) { diff --git a/src/stdio/__string_read.c b/src/stdio/__string_read.c deleted file mode 100644 index 7b50a7e11..000000000 --- a/src/stdio/__string_read.c +++ /dev/null @@ -1,16 +0,0 @@ -#include "stdio_impl.h" -#include - -size_t __string_read(FILE *f, unsigned char *buf, size_t len) -{ - char *src = f->cookie; - size_t k = len+256; - char *end = memchr(src, 0, k); - if (end) k = end-src; - if (k < len) len = k; - memcpy(buf, src, len); - f->rpos = (void *)(src+len); - f->rend = (void *)(src+k); - f->cookie = src+k; - return len; -} diff --git a/src/stdio/fmemopen.c b/src/stdio/fmemopen.c index 5685092e3..343e3e3fe 100644 --- a/src/stdio/fmemopen.c +++ b/src/stdio/fmemopen.c @@ -2,6 +2,7 @@ #include #include #include +#include #include #include "libc.h" @@ -95,18 +96,17 @@ FILE *fmemopen(void *restrict buf, size_t size, const char *restrict mode) f = malloc(sizeof *f + (buf?0:size)); if (!f) return 0; - memset(&f->f, 0, sizeof f->f); + memset(f, 0, offsetof(struct mem_FILE, buf)); f->f.cookie = &f->c; f->f.fd = -1; f->f.lbf = EOF; f->f.buf = f->buf + UNGET; f->f.buf_size = sizeof f->buf - UNGET; if (!buf) { - buf = f->buf2;; + buf = f->buf2; memset(buf, 0, size); } - memset(&f->c, 0, sizeof f->c); f->c.buf = buf; f->c.size = size; f->c.mode = *mode; diff --git a/src/stdio/ofl.c b/src/stdio/ofl.c index f2d3215a9..aad3d171d 100644 --- a/src/stdio/ofl.c +++ b/src/stdio/ofl.c @@ -1,8 +1,10 @@ #include "stdio_impl.h" #include "lock.h" +#include "fork_impl.h" static FILE *ofl_head; static volatile int ofl_lock[1]; +volatile int *const __stdio_ofl_lockptr = ofl_lock; FILE **__ofl_lock() { diff --git a/src/stdio/vfscanf.c b/src/stdio/vfscanf.c index 9e030fc44..b78a374d3 100644 --- a/src/stdio/vfscanf.c +++ b/src/stdio/vfscanf.c @@ -57,7 +57,7 @@ int vfscanf(FILE *restrict f, const char *restrict fmt, va_list ap) { int width; int size; - int alloc; + int alloc = 0; int base; const unsigned char *p; int c, t; @@ -76,6 +76,9 @@ int vfscanf(FILE *restrict f, const char *restrict fmt, va_list ap) FLOCK(f); + if (!f->rpos) __toread(f); + if (!f->rpos) goto input_fail; + for (p=(const unsigned char *)fmt; *p; p++) { alloc = 0; diff --git a/src/stdio/vsscanf.c b/src/stdio/vsscanf.c index 985002256..4d6d259b8 100644 --- a/src/stdio/vsscanf.c +++ b/src/stdio/vsscanf.c @@ -1,15 +1,25 @@ #include "stdio_impl.h" +#include -static size_t do_read(FILE *f, unsigned char *buf, size_t len) +static size_t string_read(FILE *f, unsigned char *buf, size_t len) { - return __string_read(f, buf, len); + char *src = f->cookie; + size_t k = len+256; + char *end = memchr(src, 0, k); + if (end) k = end-src; + if (k < len) len = k; + memcpy(buf, src, len); + f->rpos = (void *)(src+len); + f->rend = (void *)(src+k); + f->cookie = src+k; + return len; } int vsscanf(const char *restrict s, const char *restrict fmt, va_list ap) { FILE f = { .buf = (void *)s, .cookie = (void *)s, - .read = do_read, .lock = -1 + .read = string_read, .lock = -1 }; return vfscanf(&f, fmt, ap); } diff --git a/src/stdlib/wcstod.c b/src/stdlib/wcstod.c index 26fe9af8b..0deb7010b 100644 --- a/src/stdlib/wcstod.c +++ b/src/stdlib/wcstod.c @@ -33,8 +33,7 @@ static long double wcstox(const wchar_t *s, wchar_t **p, int prec) unsigned char buf[64]; FILE f = {0}; f.flags = 0; - f.rpos = f.rend = 0; - f.buf = buf + 4; + f.rpos = f.rend = f.buf = buf + 4; f.buf_size = sizeof buf - 4; f.lock = -1; f.read = do_read; diff --git a/src/stdlib/wcstol.c b/src/stdlib/wcstol.c index 4443f5772..1eeb495fd 100644 --- a/src/stdlib/wcstol.c +++ b/src/stdlib/wcstol.c @@ -35,8 +35,7 @@ static unsigned long long wcstox(const wchar_t *s, wchar_t **p, int base, unsign unsigned char buf[64]; FILE f = {0}; f.flags = 0; - f.rpos = f.rend = 0; - f.buf = buf + 4; + f.rpos = f.rend = f.buf = buf + 4; f.buf_size = sizeof buf - 4; f.lock = -1; f.read = do_read; diff --git a/src/string/aarch64/memcpy.S b/src/string/aarch64/memcpy.S new file mode 100644 index 000000000..48bb8a8d3 --- /dev/null +++ b/src/string/aarch64/memcpy.S @@ -0,0 +1,186 @@ +/* + * memcpy - copy memory area + * + * Copyright (c) 2012-2020, Arm Limited. + * SPDX-License-Identifier: MIT + */ + +/* Assumptions: + * + * ARMv8-a, AArch64, unaligned accesses. + * + */ + +#define dstin x0 +#define src x1 +#define count x2 +#define dst x3 +#define srcend x4 +#define dstend x5 +#define A_l x6 +#define A_lw w6 +#define A_h x7 +#define B_l x8 +#define B_lw w8 +#define B_h x9 +#define C_l x10 +#define C_lw w10 +#define C_h x11 +#define D_l x12 +#define D_h x13 +#define E_l x14 +#define E_h x15 +#define F_l x16 +#define F_h x17 +#define G_l count +#define G_h dst +#define H_l src +#define H_h srcend +#define tmp1 x14 + +/* This implementation of memcpy uses unaligned accesses and branchless + sequences to keep the code small, simple and improve performance. + + Copies are split into 3 main cases: small copies of up to 32 bytes, medium + copies of up to 128 bytes, and large copies. The overhead of the overlap + check is negligible since it is only required for large copies. + + Large copies use a software pipelined loop processing 64 bytes per iteration. + The destination pointer is 16-byte aligned to minimize unaligned accesses. + The loop tail is handled by always copying 64 bytes from the end. +*/ + +.global memcpy +.type memcpy,%function +memcpy: + add srcend, src, count + add dstend, dstin, count + cmp count, 128 + b.hi .Lcopy_long + cmp count, 32 + b.hi .Lcopy32_128 + + /* Small copies: 0..32 bytes. */ + cmp count, 16 + b.lo .Lcopy16 + ldp A_l, A_h, [src] + ldp D_l, D_h, [srcend, -16] + stp A_l, A_h, [dstin] + stp D_l, D_h, [dstend, -16] + ret + + /* Copy 8-15 bytes. */ +.Lcopy16: + tbz count, 3, .Lcopy8 + ldr A_l, [src] + ldr A_h, [srcend, -8] + str A_l, [dstin] + str A_h, [dstend, -8] + ret + + .p2align 3 + /* Copy 4-7 bytes. */ +.Lcopy8: + tbz count, 2, .Lcopy4 + ldr A_lw, [src] + ldr B_lw, [srcend, -4] + str A_lw, [dstin] + str B_lw, [dstend, -4] + ret + + /* Copy 0..3 bytes using a branchless sequence. */ +.Lcopy4: + cbz count, .Lcopy0 + lsr tmp1, count, 1 + ldrb A_lw, [src] + ldrb C_lw, [srcend, -1] + ldrb B_lw, [src, tmp1] + strb A_lw, [dstin] + strb B_lw, [dstin, tmp1] + strb C_lw, [dstend, -1] +.Lcopy0: + ret + + .p2align 4 + /* Medium copies: 33..128 bytes. */ +.Lcopy32_128: + ldp A_l, A_h, [src] + ldp B_l, B_h, [src, 16] + ldp C_l, C_h, [srcend, -32] + ldp D_l, D_h, [srcend, -16] + cmp count, 64 + b.hi .Lcopy128 + stp A_l, A_h, [dstin] + stp B_l, B_h, [dstin, 16] + stp C_l, C_h, [dstend, -32] + stp D_l, D_h, [dstend, -16] + ret + + .p2align 4 + /* Copy 65..128 bytes. */ +.Lcopy128: + ldp E_l, E_h, [src, 32] + ldp F_l, F_h, [src, 48] + cmp count, 96 + b.ls .Lcopy96 + ldp G_l, G_h, [srcend, -64] + ldp H_l, H_h, [srcend, -48] + stp G_l, G_h, [dstend, -64] + stp H_l, H_h, [dstend, -48] +.Lcopy96: + stp A_l, A_h, [dstin] + stp B_l, B_h, [dstin, 16] + stp E_l, E_h, [dstin, 32] + stp F_l, F_h, [dstin, 48] + stp C_l, C_h, [dstend, -32] + stp D_l, D_h, [dstend, -16] + ret + + .p2align 4 + /* Copy more than 128 bytes. */ +.Lcopy_long: + + /* Copy 16 bytes and then align dst to 16-byte alignment. */ + + ldp D_l, D_h, [src] + and tmp1, dstin, 15 + bic dst, dstin, 15 + sub src, src, tmp1 + add count, count, tmp1 /* Count is now 16 too large. */ + ldp A_l, A_h, [src, 16] + stp D_l, D_h, [dstin] + ldp B_l, B_h, [src, 32] + ldp C_l, C_h, [src, 48] + ldp D_l, D_h, [src, 64]! + subs count, count, 128 + 16 /* Test and readjust count. */ + b.ls .Lcopy64_from_end + +.Lloop64: + stp A_l, A_h, [dst, 16] + ldp A_l, A_h, [src, 16] + stp B_l, B_h, [dst, 32] + ldp B_l, B_h, [src, 32] + stp C_l, C_h, [dst, 48] + ldp C_l, C_h, [src, 48] + stp D_l, D_h, [dst, 64]! + ldp D_l, D_h, [src, 64]! + subs count, count, 64 + b.hi .Lloop64 + + /* Write the last iteration and copy 64 bytes from the end. */ +.Lcopy64_from_end: + ldp E_l, E_h, [srcend, -64] + stp A_l, A_h, [dst, 16] + ldp A_l, A_h, [srcend, -48] + stp B_l, B_h, [dst, 32] + ldp B_l, B_h, [srcend, -32] + stp C_l, C_h, [dst, 48] + ldp C_l, C_h, [srcend, -16] + stp D_l, D_h, [dst, 64] + stp E_l, E_h, [dstend, -64] + stp A_l, A_h, [dstend, -48] + stp B_l, B_h, [dstend, -32] + stp C_l, C_h, [dstend, -16] + ret + +.size memcpy,.-memcpy diff --git a/src/string/aarch64/memset.S b/src/string/aarch64/memset.S new file mode 100644 index 000000000..f0d29b7fa --- /dev/null +++ b/src/string/aarch64/memset.S @@ -0,0 +1,115 @@ +/* + * memset - fill memory with a constant byte + * + * Copyright (c) 2012-2020, Arm Limited. + * SPDX-License-Identifier: MIT + */ + +/* Assumptions: + * + * ARMv8-a, AArch64, Advanced SIMD, unaligned accesses. + * + */ + +#define dstin x0 +#define val x1 +#define valw w1 +#define count x2 +#define dst x3 +#define dstend x4 +#define zva_val x5 + +.global memset +.type memset,%function +memset: + + dup v0.16B, valw + add dstend, dstin, count + + cmp count, 96 + b.hi .Lset_long + cmp count, 16 + b.hs .Lset_medium + mov val, v0.D[0] + + /* Set 0..15 bytes. */ + tbz count, 3, 1f + str val, [dstin] + str val, [dstend, -8] + ret + nop +1: tbz count, 2, 2f + str valw, [dstin] + str valw, [dstend, -4] + ret +2: cbz count, 3f + strb valw, [dstin] + tbz count, 1, 3f + strh valw, [dstend, -2] +3: ret + + /* Set 17..96 bytes. */ +.Lset_medium: + str q0, [dstin] + tbnz count, 6, .Lset96 + str q0, [dstend, -16] + tbz count, 5, 1f + str q0, [dstin, 16] + str q0, [dstend, -32] +1: ret + + .p2align 4 + /* Set 64..96 bytes. Write 64 bytes from the start and + 32 bytes from the end. */ +.Lset96: + str q0, [dstin, 16] + stp q0, q0, [dstin, 32] + stp q0, q0, [dstend, -32] + ret + + .p2align 4 +.Lset_long: + and valw, valw, 255 + bic dst, dstin, 15 + str q0, [dstin] + cmp count, 160 + ccmp valw, 0, 0, hs + b.ne .Lno_zva + +#ifndef SKIP_ZVA_CHECK + mrs zva_val, dczid_el0 + and zva_val, zva_val, 31 + cmp zva_val, 4 /* ZVA size is 64 bytes. */ + b.ne .Lno_zva +#endif + str q0, [dst, 16] + stp q0, q0, [dst, 32] + bic dst, dst, 63 + sub count, dstend, dst /* Count is now 64 too large. */ + sub count, count, 128 /* Adjust count and bias for loop. */ + + .p2align 4 +.Lzva_loop: + add dst, dst, 64 + dc zva, dst + subs count, count, 64 + b.hi .Lzva_loop + stp q0, q0, [dstend, -64] + stp q0, q0, [dstend, -32] + ret + +.Lno_zva: + sub count, dstend, dst /* Count is 16 too large. */ + sub dst, dst, 16 /* Dst is biased by -32. */ + sub count, count, 64 + 16 /* Adjust count and bias for loop. */ +.Lno_zva_loop: + stp q0, q0, [dst, 32] + stp q0, q0, [dst, 64]! + subs count, count, 64 + b.hi .Lno_zva_loop + stp q0, q0, [dstend, -64] + stp q0, q0, [dstend, -32] + ret + +.size memset,.-memset + diff --git a/src/string/arm/memcpy_le.S b/src/string/arm/memcpy.S similarity index 82% rename from src/string/arm/memcpy_le.S rename to src/string/arm/memcpy.S index 7b35d305e..869e34481 100644 --- a/src/string/arm/memcpy_le.S +++ b/src/string/arm/memcpy.S @@ -1,5 +1,3 @@ -#if !__ARMEB__ - /* * Copyright (C) 2008 The Android Open Source Project * All rights reserved. @@ -42,7 +40,7 @@ * code safely callable from thumb mode, adjusting the return * instructions to be compatible with pre-thumb ARM cpus, removal of * prefetch code that is not compatible with older cpus and support for - * building as thumb 2. + * building as thumb 2 and big-endian. */ .syntax unified @@ -227,24 +225,45 @@ non_congruent: * becomes aligned to 32 bits (r5 = nb of words to copy for alignment) */ movs r5, r5, lsl #31 + +#if __ARMEB__ + movmi r3, r3, ror #24 + strbmi r3, [r0], #1 + movcs r3, r3, ror #24 + strbcs r3, [r0], #1 + movcs r3, r3, ror #24 + strbcs r3, [r0], #1 +#else strbmi r3, [r0], #1 movmi r3, r3, lsr #8 strbcs r3, [r0], #1 movcs r3, r3, lsr #8 strbcs r3, [r0], #1 movcs r3, r3, lsr #8 +#endif cmp r2, #4 blo partial_word_tail +#if __ARMEB__ + mov r3, r3, lsr r12 + mov r3, r3, lsl r12 +#endif + /* Align destination to 32 bytes (cache line boundary) */ 1: tst r0, #0x1c beq 2f ldr r5, [r1], #4 sub r2, r2, #4 +#if __ARMEB__ + mov r4, r5, lsr lr + orr r4, r4, r3 + mov r3, r5, lsl r12 +#else mov r4, r5, lsl lr orr r4, r4, r3 mov r3, r5, lsr r12 +#endif str r4, [r0], #4 cmp r2, #4 bhs 1b @@ -270,6 +289,25 @@ loop16: ldmia r1!, { r5,r6,r7, r8,r9,r10,r11} subs r2, r2, #32 ldrhs r12, [r1], #4 +#if __ARMEB__ + orr r3, r3, r4, lsr #16 + mov r4, r4, lsl #16 + orr r4, r4, r5, lsr #16 + mov r5, r5, lsl #16 + orr r5, r5, r6, lsr #16 + mov r6, r6, lsl #16 + orr r6, r6, r7, lsr #16 + mov r7, r7, lsl #16 + orr r7, r7, r8, lsr #16 + mov r8, r8, lsl #16 + orr r8, r8, r9, lsr #16 + mov r9, r9, lsl #16 + orr r9, r9, r10, lsr #16 + mov r10, r10, lsl #16 + orr r10, r10, r11, lsr #16 + stmia r0!, {r3,r4,r5,r6, r7,r8,r9,r10} + mov r3, r11, lsl #16 +#else orr r3, r3, r4, lsl #16 mov r4, r4, lsr #16 orr r4, r4, r5, lsl #16 @@ -287,6 +325,7 @@ loop16: orr r10, r10, r11, lsl #16 stmia r0!, {r3,r4,r5,r6, r7,r8,r9,r10} mov r3, r11, lsr #16 +#endif bhs 1b b less_than_thirtytwo @@ -296,6 +335,25 @@ loop8: ldmia r1!, { r5,r6,r7, r8,r9,r10,r11} subs r2, r2, #32 ldrhs r12, [r1], #4 +#if __ARMEB__ + orr r3, r3, r4, lsr #24 + mov r4, r4, lsl #8 + orr r4, r4, r5, lsr #24 + mov r5, r5, lsl #8 + orr r5, r5, r6, lsr #24 + mov r6, r6, lsl #8 + orr r6, r6, r7, lsr #24 + mov r7, r7, lsl #8 + orr r7, r7, r8, lsr #24 + mov r8, r8, lsl #8 + orr r8, r8, r9, lsr #24 + mov r9, r9, lsl #8 + orr r9, r9, r10, lsr #24 + mov r10, r10, lsl #8 + orr r10, r10, r11, lsr #24 + stmia r0!, {r3,r4,r5,r6, r7,r8,r9,r10} + mov r3, r11, lsl #8 +#else orr r3, r3, r4, lsl #24 mov r4, r4, lsr #8 orr r4, r4, r5, lsl #24 @@ -313,6 +371,7 @@ loop8: orr r10, r10, r11, lsl #24 stmia r0!, {r3,r4,r5,r6, r7,r8,r9,r10} mov r3, r11, lsr #8 +#endif bhs 1b b less_than_thirtytwo @@ -322,6 +381,25 @@ loop24: ldmia r1!, { r5,r6,r7, r8,r9,r10,r11} subs r2, r2, #32 ldrhs r12, [r1], #4 +#if __ARMEB__ + orr r3, r3, r4, lsr #8 + mov r4, r4, lsl #24 + orr r4, r4, r5, lsr #8 + mov r5, r5, lsl #24 + orr r5, r5, r6, lsr #8 + mov r6, r6, lsl #24 + orr r6, r6, r7, lsr #8 + mov r7, r7, lsl #24 + orr r7, r7, r8, lsr #8 + mov r8, r8, lsl #24 + orr r8, r8, r9, lsr #8 + mov r9, r9, lsl #24 + orr r9, r9, r10, lsr #8 + mov r10, r10, lsl #24 + orr r10, r10, r11, lsr #8 + stmia r0!, {r3,r4,r5,r6, r7,r8,r9,r10} + mov r3, r11, lsl #24 +#else orr r3, r3, r4, lsl #8 mov r4, r4, lsr #24 orr r4, r4, r5, lsl #8 @@ -339,6 +417,7 @@ loop24: orr r10, r10, r11, lsl #8 stmia r0!, {r3,r4,r5,r6, r7,r8,r9,r10} mov r3, r11, lsr #24 +#endif bhs 1b less_than_thirtytwo: @@ -350,9 +429,15 @@ less_than_thirtytwo: 1: ldr r5, [r1], #4 sub r2, r2, #4 +#if __ARMEB__ + mov r4, r5, lsr lr + orr r4, r4, r3 + mov r3, r5, lsl r12 +#else mov r4, r5, lsl lr orr r4, r4, r3 mov r3, r5, lsr r12 +#endif str r4, [r0], #4 cmp r2, #4 bhs 1b @@ -360,11 +445,20 @@ less_than_thirtytwo: partial_word_tail: /* we have a partial word in the input buffer */ movs r5, lr, lsl #(31-3) +#if __ARMEB__ + movmi r3, r3, ror #24 + strbmi r3, [r0], #1 + movcs r3, r3, ror #24 + strbcs r3, [r0], #1 + movcs r3, r3, ror #24 + strbcs r3, [r0], #1 +#else strbmi r3, [r0], #1 movmi r3, r3, lsr #8 strbcs r3, [r0], #1 movcs r3, r3, lsr #8 strbcs r3, [r0], #1 +#endif /* Refill spilled registers from the stack. Don't update sp. */ ldmfd sp, {r5-r11} @@ -383,4 +477,3 @@ copy_last_3_and_return: ldmfd sp!, {r0, r4, lr} bx lr -#endif diff --git a/src/string/arm/memcpy.c b/src/string/arm/memcpy.c deleted file mode 100644 index 041614f4b..000000000 --- a/src/string/arm/memcpy.c +++ /dev/null @@ -1,3 +0,0 @@ -#if __ARMEB__ -#include "../memcpy.c" -#endif diff --git a/src/string/memmem.c b/src/string/memmem.c index 58a21fcd6..11eff86e4 100644 --- a/src/string/memmem.c +++ b/src/string/memmem.c @@ -12,8 +12,8 @@ static char *twobyte_memmem(const unsigned char *h, size_t k, const unsigned cha static char *threebyte_memmem(const unsigned char *h, size_t k, const unsigned char *n) { - uint32_t nw = n[0]<<24 | n[1]<<16 | n[2]<<8; - uint32_t hw = h[0]<<24 | h[1]<<16 | h[2]<<8; + uint32_t nw = (uint32_t)n[0]<<24 | n[1]<<16 | n[2]<<8; + uint32_t hw = (uint32_t)h[0]<<24 | h[1]<<16 | h[2]<<8; for (h+=3, k-=3; k; k--, hw = (hw|*h++)<<8) if (hw == nw) return (char *)h-3; return hw == nw ? (char *)h-3 : 0; @@ -21,8 +21,8 @@ static char *threebyte_memmem(const unsigned char *h, size_t k, const unsigned c static char *fourbyte_memmem(const unsigned char *h, size_t k, const unsigned char *n) { - uint32_t nw = n[0]<<24 | n[1]<<16 | n[2]<<8 | n[3]; - uint32_t hw = h[0]<<24 | h[1]<<16 | h[2]<<8 | h[3]; + uint32_t nw = (uint32_t)n[0]<<24 | n[1]<<16 | n[2]<<8 | n[3]; + uint32_t hw = (uint32_t)h[0]<<24 | h[1]<<16 | h[2]<<8 | h[3]; for (h+=4, k-=4; k; k--, hw = hw<<8 | *h++) if (hw == nw) return (char *)h-4; return hw == nw ? (char *)h-4 : 0; diff --git a/src/string/strsignal.c b/src/string/strsignal.c index 96bfe841f..5156366e6 100644 --- a/src/string/strsignal.c +++ b/src/string/strsignal.c @@ -31,7 +31,11 @@ static const char map[] = { [SIGPIPE] = 13, [SIGALRM] = 14, [SIGTERM] = 15, +#if defined(SIGSTKFLT) [SIGSTKFLT] = 16, +#elif defined(SIGEMT) + [SIGEMT] = 16, +#endif [SIGCHLD] = 17, [SIGCONT] = 18, [SIGSTOP] = 19, @@ -70,7 +74,13 @@ static const char strings[] = "Broken pipe\0" "Alarm clock\0" "Terminated\0" +#if defined(SIGSTKFLT) "Stack fault\0" +#elif defined(SIGEMT) + "Emulator trap\0" +#else + "Unknown signal\0" +#endif "Child process status\0" "Continued\0" "Stopped (signal)\0" diff --git a/src/string/strstr.c b/src/string/strstr.c index 55ba1c7b4..96657bc23 100644 --- a/src/string/strstr.c +++ b/src/string/strstr.c @@ -10,16 +10,16 @@ static char *twobyte_strstr(const unsigned char *h, const unsigned char *n) static char *threebyte_strstr(const unsigned char *h, const unsigned char *n) { - uint32_t nw = n[0]<<24 | n[1]<<16 | n[2]<<8; - uint32_t hw = h[0]<<24 | h[1]<<16 | h[2]<<8; + uint32_t nw = (uint32_t)n[0]<<24 | n[1]<<16 | n[2]<<8; + uint32_t hw = (uint32_t)h[0]<<24 | h[1]<<16 | h[2]<<8; for (h+=2; *h && hw != nw; hw = (hw|*++h)<<8); return *h ? (char *)h-2 : 0; } static char *fourbyte_strstr(const unsigned char *h, const unsigned char *n) { - uint32_t nw = n[0]<<24 | n[1]<<16 | n[2]<<8 | n[3]; - uint32_t hw = h[0]<<24 | h[1]<<16 | h[2]<<8 | h[3]; + uint32_t nw = (uint32_t)n[0]<<24 | n[1]<<16 | n[2]<<8 | n[3]; + uint32_t hw = (uint32_t)h[0]<<24 | h[1]<<16 | h[2]<<8 | h[3]; for (h+=3; *h && hw != nw; hw = hw<<8 | *++h); return *h ? (char *)h-3 : 0; } @@ -96,7 +96,7 @@ static char *twoway_strstr(const unsigned char *h, const unsigned char *n) for (;;) { /* Update incremental end-of-haystack pointer */ if (z-h < l) { - /* Fast estimate for MIN(l,63) */ + /* Fast estimate for MAX(l,63) */ size_t grow = l | 63; const unsigned char *z2 = memchr(z, 0, grow); if (z2) { diff --git a/src/termios/tcgetwinsize.c b/src/termios/tcgetwinsize.c new file mode 100644 index 000000000..9b3a65a40 --- /dev/null +++ b/src/termios/tcgetwinsize.c @@ -0,0 +1,8 @@ +#include +#include +#include "syscall.h" + +int tcgetwinsize(int fd, struct winsize *wsz) +{ + return syscall(SYS_ioctl, fd, TIOCGWINSZ, wsz); +} diff --git a/src/termios/tcsetwinsize.c b/src/termios/tcsetwinsize.c new file mode 100644 index 000000000..e01d0e254 --- /dev/null +++ b/src/termios/tcsetwinsize.c @@ -0,0 +1,8 @@ +#include +#include +#include "syscall.h" + +int tcsetwinsize(int fd, const struct winsize *wsz) +{ + return syscall(SYS_ioctl, fd, TIOCSWINSZ, wsz); +} diff --git a/src/thread/__lock.c b/src/thread/__lock.c index 45557c888..60eece49a 100644 --- a/src/thread/__lock.c +++ b/src/thread/__lock.c @@ -18,9 +18,11 @@ void __lock(volatile int *l) { - if (!libc.threads_minus_1) return; + int need_locks = libc.need_locks; + if (!need_locks) return; /* fast path: INT_MIN for the lock, +1 for the congestion */ int current = a_cas(l, 0, INT_MIN + 1); + if (need_locks < 0) libc.need_locks = 0; if (!current) return; /* A first spin loop, for medium congestion. */ for (unsigned i = 0; i < 10; ++i) { diff --git a/src/thread/i386/__set_thread_area.s b/src/thread/i386/__set_thread_area.s index c2c21dd5d..aa6852beb 100644 --- a/src/thread/i386/__set_thread_area.s +++ b/src/thread/i386/__set_thread_area.s @@ -28,6 +28,7 @@ __set_thread_area: ret 2: mov %ebx,%ecx + xor %eax,%eax xor %ebx,%ebx xor %edx,%edx mov %ebx,(%esp) diff --git a/src/thread/pthread_attr_get.c b/src/thread/pthread_attr_get.c index 4aa5afdb2..f12ff4425 100644 --- a/src/thread/pthread_attr_get.c +++ b/src/thread/pthread_attr_get.c @@ -70,7 +70,7 @@ int pthread_condattr_getpshared(const pthread_condattr_t *restrict a, int *restr int pthread_mutexattr_getprotocol(const pthread_mutexattr_t *restrict a, int *restrict protocol) { - *protocol = PTHREAD_PRIO_NONE; + *protocol = a->__attr / 8U % 2; return 0; } int pthread_mutexattr_getpshared(const pthread_mutexattr_t *restrict a, int *restrict pshared) diff --git a/src/thread/pthread_cond_timedwait.c b/src/thread/pthread_cond_timedwait.c index d15012406..6b761455c 100644 --- a/src/thread/pthread_cond_timedwait.c +++ b/src/thread/pthread_cond_timedwait.c @@ -146,14 +146,18 @@ int __pthread_cond_timedwait(pthread_cond_t *restrict c, pthread_mutex_t *restri if (oldstate == WAITING) goto done; - if (!node.next) a_inc(&m->_m_waiters); + if (!node.next && !(m->_m_type & 8)) + a_inc(&m->_m_waiters); /* Unlock the barrier that's holding back the next waiter, and * either wake it or requeue it to the mutex. */ - if (node.prev) - unlock_requeue(&node.prev->barrier, &m->_m_lock, m->_m_type & 128); - else - a_dec(&m->_m_waiters); + if (node.prev) { + int val = m->_m_lock; + if (val>0) a_cas(&m->_m_lock, val, val|0x80000000); + unlock_requeue(&node.prev->barrier, &m->_m_lock, m->_m_type & (8|128)); + } else if (!(m->_m_type & 8)) { + a_dec(&m->_m_waiters); + } /* Since a signal was consumed, cancellation is not permitted. */ if (e == ECANCELED) e = 0; diff --git a/src/thread/pthread_create.c b/src/thread/pthread_create.c index 5f4910925..6f187ee89 100644 --- a/src/thread/pthread_create.c +++ b/src/thread/pthread_create.c @@ -69,15 +69,29 @@ _Noreturn void __pthread_exit(void *result) __pthread_tsd_run_dtors(); + __block_app_sigs(&set); + + /* This atomic potentially competes with a concurrent pthread_detach + * call; the loser is responsible for freeing thread resources. */ + int state = a_cas(&self->detach_state, DT_JOINABLE, DT_EXITING); + + if (state==DT_DETACHED && self->map_base) { + /* Since __unmapself bypasses the normal munmap code path, + * explicitly wait for vmlock holders first. This must be + * done before any locks are taken, to avoid lock ordering + * issues that could lead to deadlock. */ + __vm_wait(); + } + /* Access to target the exiting thread with syscalls that use * its kernel tid is controlled by killlock. For detached threads, * any use past this point would have undefined behavior, but for - * joinable threads it's a valid usage that must be handled. */ + * joinable threads it's a valid usage that must be handled. + * Signals must be blocked since pthread_kill must be AS-safe. */ LOCK(self->killlock); - /* The thread list lock must be AS-safe, and thus requires - * application signals to be blocked before it can be taken. */ - __block_app_sigs(&set); + /* The thread list lock must be AS-safe, and thus depends on + * application signals being blocked above. */ __tl_lock(); /* If this is the only thread in the list, don't proceed with @@ -85,19 +99,13 @@ _Noreturn void __pthread_exit(void *result) * signal state to prepare for exit to call atexit handlers. */ if (self->next == self) { __tl_unlock(); - __restore_sigs(&set); UNLOCK(self->killlock); + self->detach_state = state; + __restore_sigs(&set); exit(0); } - /* At this point we are committed to thread termination. Unlink - * the thread from the list. This change will not be visible - * until the lock is released, which only happens after SYS_exit - * has been called, via the exit futex address pointing at the lock. */ - libc.threads_minus_1--; - self->next->prev = self->prev; - self->prev->next = self->next; - self->prev = self->next = self; + /* At this point we are committed to thread termination. */ /* Process robust list in userspace to handle non-pshared mutexes * and the detached thread case where the robust list head will @@ -121,9 +129,15 @@ _Noreturn void __pthread_exit(void *result) __do_orphaned_stdio_locks(); __dl_thread_cleanup(); - /* This atomic potentially competes with a concurrent pthread_detach - * call; the loser is responsible for freeing thread resources. */ - int state = a_cas(&self->detach_state, DT_JOINABLE, DT_EXITING); + /* Last, unlink thread from the list. This change will not be visible + * until the lock is released, which only happens after SYS_exit + * has been called, via the exit futex address pointing at the lock. + * This needs to happen after any possible calls to LOCK() that might + * skip locking if process appears single-threaded. */ + if (!--libc.threads_minus_1) libc.need_locks = -1; + self->next->prev = self->prev; + self->prev->next = self->next; + self->prev = self->next = self; if (state==DT_DETACHED && self->map_base) { /* Detached threads must block even implementation-internal @@ -136,16 +150,13 @@ _Noreturn void __pthread_exit(void *result) if (self->robust_list.off) __syscall(SYS_set_robust_list, 0, 3*sizeof(long)); - /* Since __unmapself bypasses the normal munmap code path, - * explicitly wait for vmlock holders first. */ - __vm_wait(); - /* The following call unmaps the thread's stack mapping * and then exits without touching the stack. */ __unmapself(self->map_base, self->map_size); } /* Wake any joiner. */ + a_store(&self->detach_state, DT_EXITED); __wake(&self->detach_state, 1, 1); /* After the kernel thread exits, its tid may be reused. Clear it @@ -310,7 +321,7 @@ int __pthread_create(pthread_t *restrict res, const pthread_attr_t *restrict att new->detach_state = DT_JOINABLE; } new->robust_list.head = &new->robust_list.head; - new->CANARY = self->CANARY; + new->canary = self->canary; new->sysinfo = self->sysinfo; /* Setup argument structure for the new thread on its stack. @@ -336,7 +347,7 @@ int __pthread_create(pthread_t *restrict res, const pthread_attr_t *restrict att ~(1UL<<((SIGCANCEL-1)%(8*sizeof(long)))); __tl_lock(); - libc.threads_minus_1++; + if (!libc.threads_minus_1++) libc.need_locks = 1; ret = __clone((c11 ? start_c11 : start), stack, flags, args, &new->tid, TP_ADJ(new), &__thread_list_lock); /* All clone failures translate to EAGAIN. If explicit scheduling @@ -360,7 +371,7 @@ int __pthread_create(pthread_t *restrict res, const pthread_attr_t *restrict att new->next->prev = new; new->prev->next = new; } else { - libc.threads_minus_1--; + if (!--libc.threads_minus_1) libc.need_locks = 0; } __tl_unlock(); __restore_sigs(&set); diff --git a/src/thread/pthread_getschedparam.c b/src/thread/pthread_getschedparam.c index 1cba073d0..c098befb1 100644 --- a/src/thread/pthread_getschedparam.c +++ b/src/thread/pthread_getschedparam.c @@ -4,6 +4,8 @@ int pthread_getschedparam(pthread_t t, int *restrict policy, struct sched_param *restrict param) { int r; + sigset_t set; + __block_app_sigs(&set); LOCK(t->killlock); if (!t->tid) { r = ESRCH; @@ -14,5 +16,6 @@ int pthread_getschedparam(pthread_t t, int *restrict policy, struct sched_param } } UNLOCK(t->killlock); + __restore_sigs(&set); return r; } diff --git a/src/thread/pthread_kill.c b/src/thread/pthread_kill.c index 3d9395cb7..79ddb2097 100644 --- a/src/thread/pthread_kill.c +++ b/src/thread/pthread_kill.c @@ -4,9 +4,15 @@ int pthread_kill(pthread_t t, int sig) { int r; + sigset_t set; + /* Block not just app signals, but internal ones too, since + * pthread_kill is used to implement pthread_cancel, which + * must be async-cancel-safe. */ + __block_all_sigs(&set); LOCK(t->killlock); r = t->tid ? -__syscall(SYS_tkill, t->tid, sig) : (sig+0U >= _NSIG ? EINVAL : 0); UNLOCK(t->killlock); + __restore_sigs(&set); return r; } diff --git a/src/thread/pthread_mutex_destroy.c b/src/thread/pthread_mutex_destroy.c index 6d49e6898..8d1bf77b8 100644 --- a/src/thread/pthread_mutex_destroy.c +++ b/src/thread/pthread_mutex_destroy.c @@ -1,6 +1,10 @@ -#include +#include "pthread_impl.h" int pthread_mutex_destroy(pthread_mutex_t *mutex) { + /* If the mutex being destroyed is process-shared and has nontrivial + * type (tracking ownership), it might be in the pending slot of a + * robust_list; wait for quiescence. */ + if (mutex->_m_type > 128) __vm_wait(); return 0; } diff --git a/src/thread/pthread_mutexattr_setprotocol.c b/src/thread/pthread_mutexattr_setprotocol.c index 511cc32d8..8b80c1ce9 100644 --- a/src/thread/pthread_mutexattr_setprotocol.c +++ b/src/thread/pthread_mutexattr_setprotocol.c @@ -1,24 +1,23 @@ #include "pthread_impl.h" #include "syscall.h" -static pthread_once_t check_pi_once; -static int check_pi_result; - -static void check_pi() -{ - volatile int lk = 0; - check_pi_result = -__syscall(SYS_futex, &lk, FUTEX_LOCK_PI, 0, 0); -} +static volatile int check_pi_result = -1; int pthread_mutexattr_setprotocol(pthread_mutexattr_t *a, int protocol) { + int r; switch (protocol) { case PTHREAD_PRIO_NONE: a->__attr &= ~8; return 0; case PTHREAD_PRIO_INHERIT: - pthread_once(&check_pi_once, check_pi); - if (check_pi_result) return check_pi_result; + r = check_pi_result; + if (r < 0) { + volatile int lk = 0; + r = -__syscall(SYS_futex, &lk, FUTEX_LOCK_PI, 0, 0); + a_store(&check_pi_result, r); + } + if (r) return r; a->__attr |= 8; return 0; case PTHREAD_PRIO_PROTECT: diff --git a/src/thread/pthread_mutexattr_setrobust.c b/src/thread/pthread_mutexattr_setrobust.c index 04db92a62..30a9ac3be 100644 --- a/src/thread/pthread_mutexattr_setrobust.c +++ b/src/thread/pthread_mutexattr_setrobust.c @@ -1,22 +1,20 @@ #include "pthread_impl.h" #include "syscall.h" -static pthread_once_t check_robust_once; -static int check_robust_result; - -static void check_robust() -{ - void *p; - size_t l; - check_robust_result = -__syscall(SYS_get_robust_list, 0, &p, &l); -} +static volatile int check_robust_result = -1; int pthread_mutexattr_setrobust(pthread_mutexattr_t *a, int robust) { if (robust > 1U) return EINVAL; if (robust) { - pthread_once(&check_robust_once, check_robust); - if (check_robust_result) return check_robust_result; + int r = check_robust_result; + if (r < 0) { + void *p; + size_t l; + r = -__syscall(SYS_get_robust_list, 0, &p, &l); + a_store(&check_robust_result, r); + } + if (r) return r; a->__attr |= 4; return 0; } diff --git a/src/thread/pthread_setschedparam.c b/src/thread/pthread_setschedparam.c index 038d13d8a..76d4d45a3 100644 --- a/src/thread/pthread_setschedparam.c +++ b/src/thread/pthread_setschedparam.c @@ -4,8 +4,11 @@ int pthread_setschedparam(pthread_t t, int policy, const struct sched_param *param) { int r; + sigset_t set; + __block_app_sigs(&set); LOCK(t->killlock); r = !t->tid ? ESRCH : -__syscall(SYS_sched_setscheduler, t->tid, policy, param); UNLOCK(t->killlock); + __restore_sigs(&set); return r; } diff --git a/src/thread/pthread_setschedprio.c b/src/thread/pthread_setschedprio.c index 5bf4a0197..fc2e13ddb 100644 --- a/src/thread/pthread_setschedprio.c +++ b/src/thread/pthread_setschedprio.c @@ -4,8 +4,11 @@ int pthread_setschedprio(pthread_t t, int prio) { int r; + sigset_t set; + __block_app_sigs(&set); LOCK(t->killlock); r = !t->tid ? ESRCH : -__syscall(SYS_sched_setparam, t->tid, &prio); UNLOCK(t->killlock); + __restore_sigs(&set); return r; } diff --git a/src/thread/s390x/clone.s b/src/thread/s390x/clone.s index 577748eab..2125f20b8 100644 --- a/src/thread/s390x/clone.s +++ b/src/thread/s390x/clone.s @@ -17,6 +17,9 @@ __clone: # if (!tid) syscall(SYS_exit, a(d)); # return tid; + # preserve call-saved register used as syscall arg + stg %r6, 48(%r15) + # create initial stack frame for new thread nill %r3, 0xfff8 aghi %r3, -160 @@ -35,6 +38,9 @@ __clone: lg %r6, 160(%r15) svc 120 + # restore call-saved register + lg %r6, 48(%r15) + # if error or if we're the parent, return ltgr %r2, %r2 bnzr %r14 diff --git a/src/thread/s390x/syscall_cp.s b/src/thread/s390x/syscall_cp.s index c1da40de8..d094cbf5a 100644 --- a/src/thread/s390x/syscall_cp.s +++ b/src/thread/s390x/syscall_cp.s @@ -14,6 +14,7 @@ __cp_begin: icm %r2, 15, 0(%r2) jne __cp_cancel + stg %r6, 48(%r15) stg %r7, 56(%r15) lgr %r1, %r3 lgr %r2, %r4 @@ -26,6 +27,7 @@ __cp_begin: __cp_end: lg %r7, 56(%r15) + lg %r6, 48(%r15) br %r14 __cp_cancel: diff --git a/src/thread/sem_open.c b/src/thread/sem_open.c index de8555c5a..0ad29de96 100644 --- a/src/thread/sem_open.c +++ b/src/thread/sem_open.c @@ -12,6 +12,12 @@ #include #include #include "lock.h" +#include "fork_impl.h" + +#define malloc __libc_malloc +#define calloc __libc_calloc +#define realloc undef +#define free undef static struct { ino_t ino; @@ -19,6 +25,7 @@ static struct { int refcnt; } *semtab; static volatile int lock[1]; +volatile int *const __sem_open_lockptr = lock; #define FLAGS (O_RDWR|O_NOFOLLOW|O_CLOEXEC|O_NONBLOCK) @@ -163,10 +170,12 @@ int sem_close(sem_t *sem) int i; LOCK(lock); for (i=0; itid) + goto single_threaded; callback = func; context = ctx; diff --git a/src/thread/vmlock.c b/src/thread/vmlock.c index 75f3cb761..fa0a8e3c2 100644 --- a/src/thread/vmlock.c +++ b/src/thread/vmlock.c @@ -1,6 +1,8 @@ #include "pthread_impl.h" +#include "fork_impl.h" static volatile int vmlock[2]; +volatile int *const __vmlock_lockptr = vmlock; void __vm_wait() { diff --git a/src/time/__tz.c b/src/time/__tz.c index 49a7371eb..09a6317e6 100644 --- a/src/time/__tz.c +++ b/src/time/__tz.c @@ -6,6 +6,12 @@ #include #include "libc.h" #include "lock.h" +#include "fork_impl.h" + +#define malloc __libc_malloc +#define calloc undef +#define realloc undef +#define free undef long __timezone = 0; int __daylight = 0; @@ -30,6 +36,7 @@ static char *old_tz = old_tz_buf; static size_t old_tz_size = sizeof old_tz_buf; static volatile int lock[1]; +volatile int *const __timezone_lockptr = lock; static int getint(const char **p) { @@ -178,7 +185,7 @@ static void do_tzset() zi = map; if (map) { int scale = 2; - if (sizeof(time_t) > 4 && map[4]=='2') { + if (map[4]!='1') { size_t skip = zi_dotprod(zi+20, VEC(1,1,8,5,6,1), 6); trans = zi+skip+44+44; scale++; diff --git a/src/time/timer_create.c b/src/time/timer_create.c index 455d49fc5..4bef23905 100644 --- a/src/time/timer_create.c +++ b/src/time/timer_create.c @@ -2,6 +2,7 @@ #include #include #include "pthread_impl.h" +#include "atomic.h" struct ksigevent { union sigval sigev_value; @@ -32,19 +33,6 @@ static void cleanup_fromsig(void *p) longjmp(p, 1); } -static void timer_handler(int sig, siginfo_t *si, void *ctx) -{ -} - -static void install_handler() -{ - struct sigaction sa = { - .sa_sigaction = timer_handler, - .sa_flags = SA_SIGINFO | SA_RESTART - }; - __libc_sigaction(SIGTIMER, &sa, 0); -} - static void *start(void *arg) { pthread_t self = __pthread_self(); @@ -71,7 +59,7 @@ static void *start(void *arg) int timer_create(clockid_t clk, struct sigevent *restrict evp, timer_t *restrict res) { - static pthread_once_t once = PTHREAD_ONCE_INIT; + volatile static int init = 0; pthread_t td; pthread_attr_t attr; int r; @@ -83,11 +71,15 @@ int timer_create(clockid_t clk, struct sigevent *restrict evp, timer_t *restrict switch (evp ? evp->sigev_notify : SIGEV_SIGNAL) { case SIGEV_NONE: case SIGEV_SIGNAL: + case SIGEV_THREAD_ID: if (evp) { ksev.sigev_value = evp->sigev_value; ksev.sigev_signo = evp->sigev_signo; ksev.sigev_notify = evp->sigev_notify; - ksev.sigev_tid = 0; + if (evp->sigev_notify == SIGEV_THREAD_ID) + ksev.sigev_tid = evp->sigev_notify_thread_id; + else + ksev.sigev_tid = 0; ksevp = &ksev; } if (syscall(SYS_timer_create, clk, ksevp, &timerid) < 0) @@ -95,7 +87,11 @@ int timer_create(clockid_t clk, struct sigevent *restrict evp, timer_t *restrict *res = (void *)(intptr_t)timerid; break; case SIGEV_THREAD: - pthread_once(&once, install_handler); + if (!init) { + struct sigaction sa = { .sa_handler = SIG_DFL }; + __libc_sigaction(SIGTIMER, &sa, 0); + a_store(&init, 1); + } if (evp->sigev_notify_attributes) attr = *evp->sigev_notify_attributes; else @@ -115,7 +111,7 @@ int timer_create(clockid_t clk, struct sigevent *restrict evp, timer_t *restrict ksev.sigev_value.sival_ptr = 0; ksev.sigev_signo = SIGTIMER; - ksev.sigev_notify = 4; /* SIGEV_THREAD_ID */ + ksev.sigev_notify = SIGEV_THREAD_ID; ksev.sigev_tid = td->tid; if (syscall(SYS_timer_create, clk, &ksev, &timerid) < 0) timerid = -1; diff --git a/src/unistd/close.c b/src/unistd/close.c index 5b38e0194..a2105f506 100644 --- a/src/unistd/close.c +++ b/src/unistd/close.c @@ -1,5 +1,6 @@ #include #include +#include "aio_impl.h" #include "syscall.h" static int dummy(int fd) diff --git a/src/unistd/faccessat.c b/src/unistd/faccessat.c index 76bbd4c72..557503eb6 100644 --- a/src/unistd/faccessat.c +++ b/src/unistd/faccessat.c @@ -25,12 +25,17 @@ static int checker(void *p) int faccessat(int fd, const char *filename, int amode, int flag) { - if (!flag || (flag==AT_EACCESS && getuid()==geteuid() && getgid()==getegid())) - return syscall(SYS_faccessat, fd, filename, amode, flag); + if (flag) { + int ret = __syscall(SYS_faccessat2, fd, filename, amode, flag); + if (ret != -ENOSYS) return __syscall_ret(ret); + } - if (flag != AT_EACCESS) + if (flag & ~AT_EACCESS) return __syscall_ret(-EINVAL); + if (!flag || (getuid()==geteuid() && getgid()==getegid())) + return syscall(SYS_faccessat, fd, filename, amode); + char stack[1024]; sigset_t set; pid_t pid; diff --git a/src/unistd/readlink.c b/src/unistd/readlink.c index a152d5249..32f4537f9 100644 --- a/src/unistd/readlink.c +++ b/src/unistd/readlink.c @@ -4,9 +4,16 @@ ssize_t readlink(const char *restrict path, char *restrict buf, size_t bufsize) { + char dummy[1]; + if (!bufsize) { + buf = dummy; + bufsize = 1; + } #ifdef SYS_readlink - return syscall(SYS_readlink, path, buf, bufsize); + int r = __syscall(SYS_readlink, path, buf, bufsize); #else - return syscall(SYS_readlinkat, AT_FDCWD, path, buf, bufsize); + int r = __syscall(SYS_readlinkat, AT_FDCWD, path, buf, bufsize); #endif + if (buf == dummy && r > 0) r = 0; + return __syscall_ret(r); } diff --git a/src/unistd/readlinkat.c b/src/unistd/readlinkat.c index 9af45cd5a..f79d3d142 100644 --- a/src/unistd/readlinkat.c +++ b/src/unistd/readlinkat.c @@ -3,5 +3,12 @@ ssize_t readlinkat(int fd, const char *restrict path, char *restrict buf, size_t bufsize) { - return syscall(SYS_readlinkat, fd, path, buf, bufsize); + char dummy[1]; + if (!bufsize) { + buf = dummy; + bufsize = 1; + } + int r = __syscall(SYS_readlinkat, fd, path, buf, bufsize); + if (buf == dummy && r > 0) r = 0; + return __syscall_ret(r); } diff --git a/src/unistd/setxid.c b/src/unistd/setxid.c index 0239f8afa..487c1a160 100644 --- a/src/unistd/setxid.c +++ b/src/unistd/setxid.c @@ -1,20 +1,19 @@ #include -#include +#include #include "syscall.h" #include "libc.h" -#include "pthread_impl.h" struct ctx { int id, eid, sid; - int nr, err; + int nr, ret; }; static void do_setxid(void *p) { struct ctx *c = p; - if (c->err>0) return; - int ret = -__syscall(c->nr, c->id, c->eid, c->sid); - if (ret && !c->err) { + if (c->ret<0) return; + int ret = __syscall(c->nr, c->id, c->eid, c->sid); + if (ret && !c->ret) { /* If one thread fails to set ids after another has already * succeeded, forcibly killing the process is the only safe * thing to do. State is inconsistent and dangerous. Use @@ -22,18 +21,14 @@ static void do_setxid(void *p) __block_all_sigs(0); __syscall(SYS_kill, __syscall(SYS_getpid), SIGKILL); } - c->err = ret; + c->ret = ret; } int __setxid(int nr, int id, int eid, int sid) { - /* err is initially nonzero so that failure of the first thread does not + /* ret is initially nonzero so that failure of the first thread does not * trigger the safety kill above. */ - struct ctx c = { .nr = nr, .id = id, .eid = eid, .sid = sid, .err = -1 }; + struct ctx c = { .nr = nr, .id = id, .eid = eid, .sid = sid, .ret = 1 }; __synccall(do_setxid, &c); - if (c.err) { - if (c.err>0) errno = c.err; - return -1; - } - return 0; + return __syscall_ret(c.ret); }