Add support for out-of-tree builds

.github/workflows/build-rpm.yml

@@ -23,7 +23,7 @@ jobs:
       run: |
         # symlink source directory for ndpi.spec
         ln -sr . $HOME/nDPI
-        ./autogen.sh
+        ./autogen.sh && ./configure
         cd packages/rpm
         ./configure
 

.github/workflows/build.yml

@@ -29,7 +29,7 @@ jobs:
           sudo apt-get install autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev
       - name: Build nDPI library
         run: |
-          ./autogen.sh
+          ./autogen.sh && ./configure
           make -j
           sudo make install
       - name: Generate Python bindings
@@ -182,7 +182,7 @@ jobs:
           brew install libmaxminddb
       - name: Configure nDPI
         run: |
-          ./autogen.sh --enable-option-checking=fatal --enable-debug-messages ${{ matrix.msan }} ${{ matrix.pcre }} ${{ matrix.maxminddb }} --enable-tls-sigs ${{ matrix.global_context}}
+          ./autogen.sh && ./configure --enable-option-checking=fatal --enable-debug-messages ${{ matrix.msan }} ${{ matrix.pcre }} ${{ matrix.maxminddb }} --enable-tls-sigs ${{ matrix.global_context}}
       - name: Build nDPI
         run: |
           make -j all
@@ -230,7 +230,7 @@ jobs:
         if: startsWith(matrix.os, 'ubuntu') && !endsWith(matrix.os, 'arm') && !startsWith(matrix.msan, '--with-') && !startsWith(matrix.nBPF, 'nBPF') && !startsWith(matrix.global_context, '--without') # Only on a few "standard" builds
         run: |
           make distclean
-          ./autogen.sh --enable-option-checking=fatal --enable-debug-messages --enable-tls-sigs --host=x86_64-w64-mingw32
+          ./autogen.sh && ./configure --enable-option-checking=fatal --enable-debug-messages --enable-tls-sigs --host=x86_64-w64-mingw32
           make -j $(nproc) all
         env:
           CC:
@@ -263,7 +263,7 @@ jobs:
           sudo apt-get install autoconf automake debhelper libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev rrdtool librrd-dev parallel
       - name: Configure nDPI
         run: |
-          AR=${{ matrix.ar }} RANLIB=${{ matrix.ranlib }} ./autogen.sh --enable-option-checking=fatal --enable-debug-messages --with-sanitizer --with-lto-and-gold-linker
+          ./autogen.sh && AR=${{ matrix.ar }} RANLIB=${{ matrix.ranlib }} ./configure --enable-option-checking=fatal --enable-debug-messages --with-sanitizer --with-lto-and-gold-linker
       - name: Build nDPI
         run: |
           make -j $(nproc) all
@@ -294,7 +294,7 @@ jobs:
           install: git mingw-w64-x86_64-toolchain automake1.16 automake-wrapper autoconf libtool make mingw-w64-x86_64-json-c mingw-w64-x86_64-crt-git mingw-w64-x86_64-pcre mingw-w64-x86_64-libpcap parallel
       - name: Configure nDPI on Windows msys2
         run: |
-          msys2 -c './autogen.sh --enable-option-checking=fatal --enable-debug-messages --enable-tls-sigs --disable-npcap'
+          msys2 -c './autogen.sh && ./configure --enable-option-checking=fatal --enable-debug-messages --enable-tls-sigs --disable-npcap'
       - name: Build nDPI on Windows msys2
         run: |
           msys2 -c 'make -j all'
@@ -306,3 +306,54 @@ jobs:
           msys2 -c './tests/do.sh'
           msys2 -c './tests/do-unit.sh'
           msys2 -c './tests/do-dga.sh'
+
+  test-out-of-tree-builds:
+    name: Out-of-tree builds
+    runs-on: ubuntu-latest
+    env:
+      CFLAGS: -Wextra -Werror -DNDPI_EXTENDED_SANITY_CHECKS
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis
+      - name: Install Ubuntu Prerequisites
+        run: |
+          sudo apt-get update
+          sudo apt-get install autoconf automake debhelper libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev rrdtool librrd-dev parallel
+      - name: Autogen nDPI
+        run: |
+          ./autogen.sh
+      - name: Configure and build nDPI (standard)
+        run: |
+          mkdir -p build-standard
+          cd build-standard
+          ../configure
+          make -j $(nproc) all
+          make -j $(nproc) -C example ndpiSimpleIntegration
+          make -j $(nproc) -C rrdtool
+          cd -
+      - name: Configure and build nDPI (fuzzing)
+        run: |
+          mkdir -p build-fuzzing
+          cd build-fuzzing
+          CC=clang CXX=clang++ ../configure --with-sanitizer --enable-fuzztargets
+          make -j $(nproc) all
+          make -j $(nproc) -C example ndpiSimpleIntegration
+          make -j $(nproc) -C rrdtool
+          cd -
+      - name: Print nDPI long help
+        run: |
+          cd ./build-standard/example && ./ndpiReader -H && cd -
+          cd ./build-fuzzing/example && ./ndpiReader -H && cd -
+      - name: Tests #We are only intersting into tests scripts being valid --> less tests than usual
+        run: |
+          cd build-standard
+          NDPI_FORCE_PARALLEL_UTESTS=1 NDPI_SKIP_PARALLEL_BAR=1 NDPI_TEST_ONLY_RECENTLY_UPDATED_PCAPS=1 ./tests/do.sh
+          ./tests/do-unit.sh
+          ./tests/do-dga.sh
+          cd -
+          cd build-fuzzing
+          MAX_TOTAL_TIME=10 NDPI_FORCE_PARALLEL_UTESTS=1 NDPI_SKIP_PARALLEL_BAR=1 NDPI_TEST_ONLY_RECENTLY_UPDATED_PCAPS=1 ./tests/do.sh
+          ./tests/do-unit.sh
+          ./tests/do-dga.sh
+          cd -

.github/workflows/build_docker.yml

@@ -50,7 +50,7 @@ jobs:
             "apt-get -y update &&
             apt-get -y install git wdiff colordiff autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev rrdtool librrd-dev &&
             git config --global --add safe.directory $(realpath .) &&
-            env CC=gcc ./autogen.sh --enable-option-checking=fatal --enable-debug-messages &&
+            env CC=gcc ./autogen.sh && ./configure --enable-option-checking=fatal --enable-debug-messages &&
             make -j $(nproc) all &&
             make -C example ndpiSimpleIntegration &&
             make -C rrdtool &&
@@ -74,7 +74,7 @@ jobs:
             "apt-get -y update &&
             apt-get -y install git wdiff colordiff autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev rrdtool librrd-dev &&
             git config --global --add safe.directory $(realpath .) &&
-            env CC=gcc ./autogen.sh --enable-option-checking=fatal --enable-debug-messages &&
+            env CC=gcc ./autogen.sh && ./configure --enable-option-checking=fatal --enable-debug-messages &&
             make -j $(nproc) all &&
             make -C example ndpiSimpleIntegration &&
             make -C rrdtool &&

.github/workflows/build_masan.yml

@@ -33,7 +33,7 @@ jobs:
           sudo apt-get install autoconf automake libtool pkg-config gettext flex bison libjson-c-dev libpcap-dev rrdtool librrd-dev parallel libpcre3-dev libmaxminddb-dev
       - name: Configure nDPI
         run: |
-          ./autogen.sh --enable-option-checking=fatal --with-pcre2 --with-maxminddb --with-memory-sanitizer --disable-memory-track-origins
+          ./autogen.sh && ./configure --enable-option-checking=fatal --with-pcre2 --with-maxminddb --with-memory-sanitizer --disable-memory-track-origins
       - name: Build nDPI
         run: |
           make -j $(nproc) all

.github/workflows/build_scheduled.yml

@@ -23,7 +23,7 @@ jobs:
           sudo apt-get install libpcre3-dev libmaxminddb-dev lcov
           sudo apt-get install wdiff colordiff
       - name: Configure
-        run: ./autogen.sh --enable-option-checking=fatal --enable-debug-messages --enable-code-coverage --with-pcre2 --with-maxminddb --enable-tls-sigs
+        run: ./autogen.sh && ./configure --enable-option-checking=fatal --enable-debug-messages --enable-code-coverage --with-pcre2 --with-maxminddb --enable-tls-sigs
       - name: Build
         run: make all
       - name: Test
@@ -55,7 +55,7 @@ jobs:
           sudo apt-get install autoconf automake libtool pkg-config gettext flex bison doxygen
       - name: Configure nDPI library
         run: |
-          ./autogen.sh --with-only-libndpi --enable-option-checking=fatal --enable-debug-build
+          ./autogen.sh && ./configure --with-only-libndpi --enable-option-checking=fatal --enable-debug-build
       - name: Generate Documentation
         run: |
           pip install --upgrade pip
@@ -89,7 +89,7 @@ jobs:
           pprof -h
       - name: Configure nDPI library
         run: |
-          ./autogen.sh --enable-gprof --enable-option-checking=fatal --with-pcre2 --with-maxminddb --enable-tls-sigs
+          ./autogen.sh && ./configure --enable-gprof --enable-option-checking=fatal --with-pcre2 --with-maxminddb --enable-tls-sigs
       - name: Build nDPI library
         run: |
           make -j
@@ -123,7 +123,7 @@ jobs:
           sudo apt-get install autoconf automake libtool pkg-config gettext flex bison libjson-c-dev libpcap-dev rrdtool librrd-dev parallel
       - name: Configure nDPI
         run: |
-          ./autogen.sh --enable-option-checking=fatal --with-thread-sanitizer
+          ./autogen.sh && ./configure --enable-option-checking=fatal --with-thread-sanitizer
       - name: Build nDPI
         run: |
           make -j $(nproc) all
@@ -159,7 +159,7 @@ jobs:
           brew install coreutils wdiff colordiff autoconf automake libtool gettext json-c rrdtool parallel libgcrypt
       - name: Configure nDPI
         run: |
-          ./autogen.sh --enable-option-checking=fatal --with-local-libgcrypt
+          ./autogen.sh && ./configure --enable-option-checking=fatal --with-local-libgcrypt
       - name: Build nDPI
         run: |
           make -j $(nproc) all
@@ -192,7 +192,7 @@ jobs:
           install: git mingw-w64-x86_64-toolchain automake1.16 automake-wrapper autoconf libtool make mingw-w64-x86_64-json-c mingw-w64-x86_64-crt-git mingw-w64-x86_64-pcre mingw-w64-x86_64-libpcap mingw-w64-x86_64-libgcrypt parallel
       - name: Configure nDPI on Windows msys2
         run: |
-          msys2 -c './autogen.sh --enable-option-checking=fatal --enable-debug-messages --enable-tls-sigs --disable-npcap --with-local-libgcrypt'
+          msys2 -c './autogen.sh && ./configure --enable-option-checking=fatal --enable-debug-messages --enable-tls-sigs --disable-npcap --with-local-libgcrypt'
       - name: Build nDPI on Windows msys2
         run: |
           msys2 -c 'make -j all'

.github/workflows/codeql.yml

@@ -40,7 +40,7 @@ jobs:
           pprof -h
       - name: Configure nDPI library
         run: |
-          ./autogen.sh --enable-gprof --enable-option-checking=fatal --with-pcre2 --with-maxminddb --enable-tls-sigs --enable-debug-messages
+          ./autogen.sh && ./configure --enable-gprof --enable-option-checking=fatal --with-pcre2 --with-maxminddb --enable-tls-sigs --enable-debug-messages
 
       - name: Initialize CodeQL
         uses: github/codeql-action/init@v3

Makefile.am

@@ -53,10 +53,10 @@ clean-coverage:
 	find . -type f -name "*.gcno" -exec rm "{}" +
 
 pull:
-	git pull --rebase
-	if test -d ../nDPI-custom; then echo "Pulling ../nDPI-custom"; cd ../nDPI-custom; git pull --rebase; cd ../nDPI; fi
+	cd $(top_srcdir) && git pull --rebase
+	if test -d $(top_srcdir)/../nDPI-custom; then echo "Pulling ../nDPI-custom"; cd $(top_srcdir)/../nDPI-custom && git pull --rebase; fi
 
 changelog:
-	git log --since={`curl -s https://github.com/ntop/ndpi/releases | grep datetime | head -n1 | egrep -o "[0-9]+\-[0-9]+\-[0-9]+"`} --name-only --pretty=format:" - %s" | grep  "^ " > CHANGELOG.latest
+	cd $(top_srcdir) && git log --since={`curl -s https://github.com/ntop/ndpi/releases | grep datetime | head -n1 | egrep -o "[0-9]+\-[0-9]+\-[0-9]+"`} --name-only --pretty=format:" - %s" | grep  "^ " > CHANGELOG.latest
 
 .PHONY: doc doc-view coverage clean-coverage changelog

README.fuzzer.md

@@ -7,7 +7,7 @@
   - export CXX=/usr/bin/clang++-18
   - export LD=/usr/bin/ld.lld-18
 - Run autogen.sh
-  - ./autogen.sh --with-sanitizer --enable-fuzztargets 
+  - ./autogen.sh && ./configure --with-sanitizer --enable-fuzztargets 
 
 
 ## Testing nDPI with ClusterFuzz Artifacts

README.md

@@ -13,19 +13,25 @@ A generic FAQ about nDPI is available [here](https://github.com/ntop/nDPI/blob/d
 
 In order to compile this project do
 
-- ./autogen.sh
+- ./autogen.sh && ./configure
 - make
 
 If you get some errors while compiling `croaring.c`, try:
 
-- ./autogen.sh --enable-old-croaring
+- ./autogen.sh && ./configure --enable-old-croaring
 - make
 
 To compile the library w/o any tools or tests:
 
-- ./autogen.sh --with-only-libndpi
+- ./autogen.sh && ./configure --with-only-libndpi
 - make
 
+Out-of-tree builds are supported:
+- ./autogen.sh
+- mkdir build
+- cd build
+- ../configure && make
+
 To run tests do additionally:
 
 - ./tests/do.sh # Generate and check for diff's in PCAP files
@@ -85,7 +91,7 @@ The entire procedure of adding new protocols in detail:
 5. Choose (do not change anything) a selection bitmask from: `src/include/ndpi_define.h`
 6. Set protocol default ports in `ndpi_init_protocol_defaults` in: `src/lib/ndpi_main.c`
 7. Be sure to have nBPF support, cloning `PF_RING` in the same directory where you cloned `nDPI`: `git clone https://github.com/ntop/PF_RING/ && cd PF_RING/userland/nbpf && ./configure && make`. You can ignore the `/bin/sh: 1: ../lib/pfring_config: not found` error
-8. From the `nDPI` root directory, `./autogen.sh --with-pcre2` (nBPF and PCRE2 are usually optional, but they are needed to run/update *all* the unit tests)
+8. From the `nDPI` root directory, `./autogen.sh && ./configure --with-pcre2` (nBPF and PCRE2 are usually optional, but they are needed to run/update *all* the unit tests)
 9. `make`
 10. `make check`
 11. Update the documentation, adding this new protocol to `doc/protocols.rst`

autogen.sh

@@ -42,6 +42,6 @@ fi
 
 autoreconf -ivf
 
-echo "./configure $*"
-chmod +x configure
-./configure "$@"
+#####
+# Don't call `configure` here!!!! It breaks out-of-tree builds
+#####

configure.ac

@@ -13,8 +13,6 @@ AS_IF([test "${with_only_libndpi+set}" = set],[
   EXTRA_TARGETS=""
 ])
 
-PWD=`pwd`
-
 
 AC_ARG_WITH(sanitizer, AS_HELP_STRING([--with-sanitizer], [Build with support for address, undefined and leak sanitizer]))
 AC_ARG_WITH(thread-sanitizer, AS_HELP_STRING([--with-thread-sanitizer], [Build with support for thread sanitizer]))
@@ -26,7 +24,7 @@ AC_ARG_ENABLE(code-coverage, AS_HELP_STRING([--enable-code-coverage], [Generate
 AC_ARG_WITH(local-libgcrypt, AS_HELP_STRING([--with-local-libgcrypt], [Build with libgcrypt (if present) instead of the enclosed gcrypt light]))
 AC_ARG_ENABLE(tls-sigs, AS_HELP_STRING([--enable-tls-sigs], [Enable TLS Client signature algorithm dissection. Rarely used, but requires significantly more memory.]))
 AC_ARG_ENABLE(npcap, AS_HELP_STRING([--disable-npcap], [msys2 only: Disable linkage against the wpcap/npcap import library in windows/WpdPack/Lib.]))
-AC_ARG_WITH(nbpf-path, AS_HELP_STRING([--with-nbpf-path], [nBPF library custom path; default: ${PWD}/../PF_RING/userland/nbpf]),[NBPF_HOME=$withval],[NBPF_HOME=${PWD}/../PF_RING/userland/nbpf])
+AC_ARG_WITH(nbpf-path, AS_HELP_STRING([--with-nbpf-path], [nBPF library custom path; default: ${srcdir}/../PF_RING/userland/nbpf]),[NBPF_HOME=$withval],[NBPF_HOME="`cd ${srcdir}/../PF_RING/userland/nbpf 2>/dev/null && pwd || echo ${srcdir}/../PF_RING/userland/nbpf`"])
 AC_ARG_WITH(lto-and-gold-linker, AS_HELP_STRING([--with-lto-and-gold-linker], [Build with LTO and Gold linker]))
 AC_ARG_ENABLE(debug-build, AS_HELP_STRING([--enable-debug-build], [Enable debug build (`-g` flag)]),[enable_debugbuild=$enableval],[enable_debugbuild=no])
 AC_ARG_ENABLE(global-context-support, AS_HELP_STRING([--disable-global-context-support], [Disable support for global context. No external dependency on libpthread]))
@@ -170,23 +168,23 @@ AC_DEFINE_UNQUOTED(NDPI_PATCH_LEVEL,   "${NDPI_PATCH}", [nDPI patch level])
 
 # .git as directory in a cloned repo
 # .git as file in submodule based integration
-if test -d ".git" || test -r ".git"; then
-     GIT_TAG=`git log -1 --abbrev=7 --format=%h`
-     GIT_DATE=`git log -1 --abbrev=7 --format=%cd`
+if test -d "${srcdir}/.git" || test -r "${srcdir}/.git"; then
+     GIT_TAG=`cd ${srcdir} && git log -1 --abbrev=7 --format=%h`
+     GIT_DATE=`cd ${srcdir} && git log -1 --abbrev=7 --format=%cd`
      #
      # On CentOS 6 `git rev-list HEAD --count` does not work
      #
      #
-     GIT_NUM=`git log --pretty=oneline | wc -l | tr -d '[[:space:]]'`
+     GIT_NUM=`cd ${srcdir} && git log --pretty=oneline | wc -l | tr -d '[[:space:]]'`
      GIT_RELEASE="${PACKAGE_VERSION}-${GIT_NUM}-${GIT_TAG}"
 
-     A=`git log src/include/ndpi_typedefs.h|wc -l`
-     B=`git log src/include/ndpi_protocol_ids.h|wc -l`
-     C=`git log src/include/ndpi_api.h|wc -l`
+     A=`cd ${srcdir} && git log src/include/ndpi_typedefs.h|wc -l`
+     B=`cd ${srcdir} && git log src/include/ndpi_protocol_ids.h|wc -l`
+     C=`cd ${srcdir} && git log src/include/ndpi_api.h|wc -l`
      NDPI_API_VERSION=$((A+B+C))
 else
      GIT_RELEASE="${PACKAGE_VERSION}"
-     GIT_DATE=`date -u -r CHANGELOG.md`
+     GIT_DATE=`date -u -r ${srcdir}/CHANGELOG.md`
      if test -z "$SOURCE_DATE_EPOCH" ; then :
          SOURCE_DATE_EPOCH=`date +%s`
      fi
@@ -293,7 +291,7 @@ else
 fi
 
 if ! test -d $PCAP_HOME; then :
-     PCAP_HOME=`pwd`/../../PF_RING/userland
+     PCAP_HOME=${srcdir}/../../PF_RING/userland
 fi
 SHORT_MACHINE=`uname -m | cut -b1-3`
 if test $SHORT_MACHINE = "arm"; then
@@ -318,7 +316,7 @@ else
 fi
 
 CUSTOM_NDPI=
-if test -d ../nDPI-custom; then :
+if test -d ${srcdir}/../nDPI-custom; then :
    CUSTOM_NDPI="-DCUSTOM_NDPI_PROTOCOLS"
    AC_MSG_RESULT([Compiling with custom nDPI protocols])
 fi
@@ -458,7 +456,10 @@ if test "${with_maxminddb+set}" = set; then :
   fi
 fi
 
-AC_DEFINE_UNQUOTED(NDPI_BASE_DIR, "${PWD}", [nDPI base directory])
+dnl> Use absolute source directory for NDPI_BASE_DIR
+NDPI_ABS_SRCDIR="`cd ${srcdir} && pwd`"
+AC_DEFINE_UNQUOTED(NDPI_BASE_DIR, "${NDPI_ABS_SRCDIR}", [nDPI base directory])
+AC_SUBST(NDPI_BASE_DIR, "${NDPI_ABS_SRCDIR}")
 
 dnl> Curl
 dnl> AC_CHECK_LIB([curl], [curl_easy_perform])
@@ -469,8 +470,17 @@ dnl>   ADDITIONAL_LIBS="${ADDITIONAL_LIBS} -lcurl"
 dnl>   AC_DEFINE_UNQUOTED(HAVE_CURL, 1, [curl is present])
 dnl> fi
 
+dnl> ndpiReader has some hard-coded unit tests: in the out-of-tree builds we need some configuration files in the expected path
+if test "${srcdir}" != . ; then :
+  mkdir -p example
+  cp ${srcdir}/example/categories.txt example/
+  cp -R ${srcdir}/lists lists/
+fi
+
 AC_CONFIG_FILES([Makefile example/Makefile example/Makefile.dpdk tests/Makefile tests/unit/Makefile tests/performance/Makefile tests/dga/Makefile rrdtool/Makefile influxdb/Makefile libndpi.pc src/include/ndpi_define.h src/lib/Makefile fuzz/Makefile doc/Doxyfile.cfg utils/Makefile])
 AC_CONFIG_FILES([tests/do.sh], [chmod +x tests/do.sh])
+AC_CONFIG_FILES([tests/do-dga.sh], [chmod +x tests/do-dga.sh])
+AC_CONFIG_FILES([tests/do-unit.sh], [chmod +x tests/do-unit.sh])
 AC_CONFIG_HEADERS(src/include/ndpi_config.h)
 AC_SUBST(GIT_RELEASE)
 AC_SUBST(NDPI_MAJOR)