Merge branch 'main' into mg/mingw

.github/workflows/ci.yml

@@ -6,7 +6,7 @@ on:
       - 'main'
 
 env:
-  BUILDER_VERSION: v0.9.43
+  BUILDER_VERSION: v0.9.49
   BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net
   BUILDER_SOURCE: releases
   PACKAGE_NAME: aws-c-common
@@ -197,10 +197,9 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Build ${{ env.PACKAGE_NAME }} + consumers
-        uses: cross-platform-actions/action@v0.10.0
+        uses: cross-platform-actions/action@v0.20.0
         with:
           operating_system: openbsd
-          architecture: x86-64
           version: '7.2'
           shell: bash
           run: |
@@ -209,6 +208,23 @@ jobs:
             chmod a+x builder
             ./builder build -p ${{ env.PACKAGE_NAME }}
 
+  freebsd:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: true
+      - name: Build ${{ env.PACKAGE_NAME }} + consumers
+        uses: cross-platform-actions/[email protected]
+        with:
+          operating_system: freebsd
+          version: '13.2'
+          run: |
+            sudo pkg install -y python3 py39-urllib3 py39-pip cmake
+            python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder')"
+            chmod a+x builder
+            ./builder build -p ${{ env.PACKAGE_NAME }}
+
   cross_compile:
     name: Cross Compile ${{matrix.arch}}
     runs-on: ubuntu-20.04 # latest

.github/workflows/stale_issue.yml

@@ -33,8 +33,8 @@ jobs:
         closed-for-staleness-label: closed-for-staleness
 
         # Issue timing
-        days-before-stale: 2
-        days-before-close: 5
+        days-before-stale: 10
+        days-before-close: 4
         days-before-ancient: 36500
 
         # If you don't want to mark a issue as being ancient based on a

CMakeLists.txt

@@ -24,7 +24,6 @@ include(AwsFeatureTests)
 include(AwsSanitizers)
 include(AwsThreadAffinity)
 include(AwsThreadName)
-include(AwsSIMD)
 include(CTest)
 
 set(GENERATED_ROOT_DIR "${CMAKE_CURRENT_BINARY_DIR}/generated")
@@ -70,6 +69,7 @@ if (WIN32)
 
     file(GLOB AWS_COMMON_OS_SRC
         "source/windows/*.c"
+        "source/platform_fallback_stubs/system_info.c"
         )
 
     if (MSVC)
@@ -81,7 +81,10 @@ if (WIN32)
     endif ()
 
     list(APPEND PLATFORM_DEFINES WINDOWS_KERNEL_LIB=${WINDOWS_KERNEL_LIB})
-    list(APPEND PLATFORM_LIBS bcrypt ${WINDOWS_KERNEL_LIB} ws2_32 shlwapi)
+    # PSAPI_VERSION=1 is needed to support GetProcessMemoryInfo on both pre and
+    # post Win7 OS's. 
+    list(APPEND PLATFORM_DEFINES PSAPI_VERSION=1)
+    list(APPEND PLATFORM_LIBS bcrypt ${WINDOWS_KERNEL_LIB} ws2_32 shlwapi psapi)
 else ()
     file(GLOB AWS_COMMON_OS_HEADERS
         "include/aws/common/posix/*"
@@ -108,19 +111,26 @@ else ()
         # Don't add the exact path to CoreFoundation as this would hardcode the SDK version
         list(APPEND PLATFORM_LIBS dl Threads::Threads "-framework CoreFoundation")
         list (APPEND AWS_COMMON_OS_SRC "source/darwin/*.c") # OS specific includes
+        list (APPEND AWS_COMMON_OS_SRC "source/platform_fallback_stubs/system_info.c")
     elseif (${CMAKE_SYSTEM_NAME} STREQUAL "Linux") # Android does not link to libpthread nor librt, so this is fine
         list(APPEND PLATFORM_LIBS dl m Threads::Threads rt)
         list (APPEND AWS_COMMON_OS_SRC "source/linux/*.c") # OS specific includes
     elseif(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD")
         list(APPEND PLATFORM_LIBS dl m thr execinfo)
+        list (APPEND AWS_COMMON_OS_SRC "source/platform_fallback_stubs/system_info.c")
     elseif(CMAKE_SYSTEM_NAME STREQUAL "NetBSD")
         list(APPEND PLATFORM_LIBS dl m Threads::Threads execinfo)
+        list (APPEND AWS_COMMON_OS_SRC "source/platform_fallback_stubs/system_info.c")
     elseif(CMAKE_SYSTEM_NAME STREQUAL "OpenBSD")
         list(APPEND PLATFORM_LIBS m Threads::Threads execinfo)
+        list (APPEND AWS_COMMON_OS_SRC "source/platform_fallback_stubs/system_info.c")
     elseif(CMAKE_SYSTEM_NAME STREQUAL "Android")
         list(APPEND PLATFORM_LIBS log)
         file(GLOB ANDROID_SRC "source/android/*.c")
         list(APPEND AWS_COMMON_OS_SRC "${ANDROID_SRC}")
+        list (APPEND AWS_COMMON_OS_SRC "source/platform_fallback_stubs/system_info.c")
+    else()
+        list (APPEND AWS_COMMON_OS_SRC "source/platform_fallback_stubs/system_info.c")
     endif()
 
 endif()
@@ -213,12 +223,9 @@ target_include_directories(${PROJECT_NAME} PUBLIC
 
 target_compile_definitions(${PROJECT_NAME} PRIVATE -DCJSON_HIDE_SYMBOLS)
 
-# Enable SIMD encoder if the compiler supports the right features
-simd_add_definitions(${PROJECT_NAME})
-
-if (HAVE_AVX2_INTRINSICS)
+if (AWS_HAVE_AVX2_INTRINSICS)
     target_compile_definitions(${PROJECT_NAME} PRIVATE -DUSE_SIMD_ENCODING)
-    simd_add_source_avx2(${PROJECT_NAME} "source/arch/intel/encoding_avx2.c")
+    simd_add_source_avx(${PROJECT_NAME} "source/arch/intel/encoding_avx2.c")
     message(STATUS "Building SIMD base64 decoder")
 endif()
 
@@ -298,6 +305,7 @@ configure_file(${CONFIG_HEADER_TEMPLATE}
 if (ALLOW_CROSS_COMPILED_TESTS OR NOT CMAKE_CROSSCOMPILING)
     if (BUILD_TESTING)
         add_subdirectory(tests)
+        add_subdirectory(bin/system_info)
     endif()
 endif()
 

README.md

@@ -158,7 +158,7 @@ Example:
 * Avoid C99 features in header files. For some types such as bool, uint32_t etc..., these are defined if not available for the language
 standard being used in `aws/common/common.h`, so feel free to use them.
 * For C++ compatibility, don't put const members in structs.
-* Avoid C++ style comments e.g. `//`.
+* Avoid C++ style comments e.g. `//` in header files and prefer block style  (`/* */`) for long blocks of text. C++ style comments are fine in C files.
 * All public API functions need C++ guards and Windows dll semantics.
 * Use Unix line endings.
 * Where implementation hiding is desired for either ABI or runtime polymorphism reasons, use the `void *impl` pattern. v-tables

THIRD-PARTY-LICENSES.txt

@@ -1,12 +1,9 @@
 ** ittapi ittnotify.h; version v3.24.2 -- https://github.com/intel/ittapi/blob/master/include/ittnotify.h
-/*
-  Copyright (C) 2005-2019 Intel Corporation
+Copyright (C) 2005-2019 Intel Corporation
+SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause
 
-  SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause
-*/
-/*
-  Amazon.com has chosen to use this file under the terms of the BSD-3-Clause license.
-*/
+Amazon.com has chosen to use this file under the terms of the BSD-3-Clause
+license.
 
 Copyright (c) 2019 Intel Corporation. All rights reserved.
 
@@ -34,3 +31,27 @@ LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
 ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+------
+
+** cJSON; version 1.7.16 -- https://github.com/DaveGamble/cJSON
+Copyright (c) 2009-2017 Dave Gamble and cJSON contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+

bin/system_info/CMakeLists.txt

@@ -0,0 +1,18 @@
+project(print-sys-info C)
+
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_INSTALL_PREFIX}/lib/cmake")
+
+file(GLOB SI_SRC
+        "*.c"
+        )
+
+set(SI_PROJECT_NAME print-sys-info)
+add_executable(${SI_PROJECT_NAME} ${SI_SRC})
+aws_set_common_properties(${SI_PROJECT_NAME})
+
+
+target_include_directories(${SI_PROJECT_NAME} PUBLIC
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+        $<INSTALL_INTERFACE:include>)
+
+target_link_libraries(${SI_PROJECT_NAME} PRIVATE aws-c-common)

bin/system_info/print_system_info.c

@@ -0,0 +1,48 @@
+
+
+#include <aws/common/byte_buf.h>
+#include <aws/common/logging.h>
+#include <aws/common/system_info.h>
+
+int main(void) {
+    struct aws_allocator *allocator = aws_default_allocator();
+    aws_common_library_init(allocator);
+    struct aws_logger_standard_options options = {
+        .file = stderr,
+        .level = AWS_LOG_LEVEL_TRACE,
+    };
+
+    struct aws_logger logger;
+    aws_logger_init_standard(&logger, allocator, &options);
+    aws_logger_set(&logger);
+
+    struct aws_system_environment *env = aws_system_environment_load(allocator);
+
+    fprintf(stdout, "crt-detected env: {\n");
+
+    struct aws_byte_cursor virtualization_vendor = aws_system_environment_get_virtualization_vendor(env);
+    fprintf(
+        stdout,
+        "  'virtualization vendor': '" PRInSTR "',\n",
+        (int)virtualization_vendor.len,
+        virtualization_vendor.ptr);
+    struct aws_byte_cursor product_name = aws_system_environment_get_virtualization_product_name(env);
+    fprintf(stdout, "  'product name': '" PRInSTR "',\n", (int)product_name.len, product_name.ptr);
+    fprintf(
+        stdout, "  'number of processors': '%lu',\n", (unsigned long)aws_system_environment_get_processor_count(env));
+    size_t numa_nodes = aws_system_environment_get_cpu_group_count(env);
+
+    if (numa_nodes > 1) {
+        fprintf(stdout, "  'numa architecture': 'true',\n");
+        fprintf(stdout, "  'number of numa nodes': '%lu'\n", (unsigned long)numa_nodes);
+    } else {
+        fprintf(stdout, "  'numa architecture': 'false'\n");
+    }
+
+    fprintf(stdout, "}\n");
+    aws_system_environment_release(env);
+    aws_logger_clean_up(&logger);
+
+    aws_common_library_clean_up();
+    return 0;
+}

cmake/AwsFeatureTests.cmake

@@ -117,3 +117,7 @@ if(MSVC)
         return 0;
     }" AWS_HAVE_MSVC_INTRINSICS_X64)
 endif()
+
+# This does a lot to detect when intrinsics are available and has to set cflags to do so.
+# leave it in its own file for ease of managing it.
+include(AwsSIMD)

cmake/AwsSIMD.cmake

@@ -8,18 +8,31 @@ if (USE_CPU_EXTENSIONS)
     if (MSVC)
         check_c_compiler_flag("/arch:AVX2" HAVE_M_AVX2_FLAG)
         if (HAVE_M_AVX2_FLAG)
-            set(AVX2_CFLAGS "/arch:AVX2")
+            set(AVX_CFLAGS "/arch:AVX2")
         endif()
     else()
         check_c_compiler_flag(-mavx2 HAVE_M_AVX2_FLAG)
         if (HAVE_M_AVX2_FLAG)
-            set(AVX2_CFLAGS "-mavx -mavx2")
+            set(AVX_CFLAGS "-mavx -mavx2")
         endif()
     endif()
 
+    if (MSVC)
+        check_c_compiler_flag("/arch:AVX512" HAVE_M_AVX512_FLAG)
+        if (HAVE_M_AVX512_FLAG)
+            # docs imply AVX512 brings in AVX2. And it will compile, but it will break at runtime on
+            # instructions such as _mm256_load_si256(). Leave it on.
+            set(AVX_CFLAGS "/arch:AVX512 /arch:AVX2")
+        endif()
+    else()
+        check_c_compiler_flag("-mavx512f -mvpclmulqdq" HAVE_M_AVX512_FLAG)
+        if (HAVE_M_AVX512_FLAG)
+            set(AVX_CFLAGS "-mavx512f -mvpclmulqdq -mpclmul -mavx -mavx2 -msse4.2")
+        endif()
+    endif()
 
     set(old_flags "${CMAKE_REQUIRED_FLAGS}")
-    set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} ${AVX2_CFLAGS}")
+    set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} ${AVX_CFLAGS}")
 
     check_c_source_compiles("
         #include <immintrin.h>
@@ -35,7 +48,15 @@ if (USE_CPU_EXTENSIONS)
             _mm256_permutevar8x32_epi32(vec, vec);
 
             return 0;
-        }"  HAVE_AVX2_INTRINSICS)
+        }"  AWS_HAVE_AVX2_INTRINSICS)          
+
+    check_c_source_compiles("
+        #include <immintrin.h>
+
+        int main() {
+            __m512 a = _mm512_setzero_ps();
+            return 0;
+        }" AWS_HAVE_AVX512_INTRINSICS)
 
     check_c_source_compiles("
         #include <immintrin.h>
@@ -45,30 +66,20 @@ if (USE_CPU_EXTENSIONS)
             __m256i vec;
             memset(&vec, 0, sizeof(vec));
             return (int)_mm256_extract_epi64(vec, 2);
-        }" HAVE_MM256_EXTRACT_EPI64)
+        }" AWS_HAVE_MM256_EXTRACT_EPI64)
 
     set(CMAKE_REQUIRED_FLAGS "${old_flags}")
 endif() # USE_CPU_EXTENSIONS
 
-macro(simd_add_definition_if target definition)
-    if(${definition})
-        target_compile_definitions(${target} PRIVATE -D${definition})
-    endif(${definition})
-endmacro(simd_add_definition_if)
-
-# Configure private preprocessor definitions for SIMD-related features
-# Does not set any processor feature codegen flags
-function(simd_add_definitions target)
-    simd_add_definition_if(${target} HAVE_AVX2_INTRINSICS)
-    simd_add_definition_if(${target} HAVE_MM256_EXTRACT_EPI64)
-endfunction(simd_add_definitions)
+# The part where the definition is added to the compiler flags has been moved to config.h.in
+# see git history for more details.
 
-# Adds source files only if AVX2 is supported. These files will be built with
-# avx2 intrinsics enabled.
-# Usage: simd_add_source_avx2(target file1.c file2.c ...)
-function(simd_add_source_avx2 target)
+# Adds AVX flags, if any, that are supported. These files will be built with
+# available avx intrinsics enabled.
+# Usage: simd_add_source_avx(target file1.c file2.c ...)
+function(simd_add_source_avx target)
     foreach(file ${ARGN})
         target_sources(${target} PRIVATE ${file})
-        set_source_files_properties(${file} PROPERTIES COMPILE_FLAGS "${AVX2_CFLAGS}")
+        set_source_files_properties(${file} PROPERTIES COMPILE_FLAGS "${AVX_CFLAGS}")
     endforeach()
-endfunction(simd_add_source_avx2)
+endfunction(simd_add_source_avx)

include/aws/common/allocator.h

@@ -12,6 +12,22 @@
 AWS_PUSH_SANE_WARNING_LEVEL
 AWS_EXTERN_C_BEGIN
 
+/*
+ * Quick guide to allocators:
+ *  CRT offers several flavours of allocators:
+ * - default: basic allocator that invokes system one directly.
+ * - aligned: basic allocator that aligns small allocations on 8 byte
+ *   boundary and big buffers on 32/64 byte (system dependent) boundary.
+ *   Aligned mem can improve perf on some operations, like memcpy or hashes.
+ *   Depending on a system, can result in higher peak memory count in heavy
+ *   acquire/free scenarios (ex. s3), due to memory fragmentation related to how
+ *   aligned allocators work (over allocate, find aligned offset, release extra memory)
+ * - wrapped_cf: wraps MacOS's Security Framework allocator.
+ * - mem_tracer: wraps any allocator and provides tracing functionality to allocations
+ * - small_block_allocator: pools smaller allocations into preallocated buckets.
+ *   Not actively maintained. Avoid if possible.
+ */
+
 /* Allocator structure. An instance of this will be passed around for anything needing memory allocation */
 struct aws_allocator {
     void *(*mem_acquire)(struct aws_allocator *allocator, size_t size);
@@ -32,6 +48,13 @@ bool aws_allocator_is_valid(const struct aws_allocator *alloc);
 AWS_COMMON_API
 struct aws_allocator *aws_default_allocator(void);
 
+/*
+ * Allocator that align small allocations on 8 byte boundary and big allocations
+ * on 32/64 byte boundary.
+ */
+AWS_COMMON_API
+struct aws_allocator *aws_aligned_allocator(void);
+
 #ifdef __MACH__
 /* Avoid pulling in CoreFoundation headers in a header file. */
 struct __CFAllocator; /* NOLINT(bugprone-reserved-identifier) */