Merge pull request #458 from anarkiwi/plat

matrix buildx by tool.

Author: cglewis

.github/workflows/docker.yml

@@ -9,19 +9,31 @@ on:
 
 jobs:
   buildx:
-
     runs-on: ubuntu-latest
-
+    strategy:
+      matrix:
+        include:
+          # tools are build in order, for dependencies.
+          - name: "rbqwrapped tools (p0f only)"
+            tools: "rbqwrapper p0f"
+          - name: pcap_to_node_pcap
+            tools: "pcap_to_node_pcap"
+          - name: tcprewrite_dot1q
+            tools: "tcprewrite_dot1q"
+          - name: ncapture
+            tools: "network_tap/ncapture"
+          - name: network_tap
+            tools: "network_tap"
+          - name: mercury
+            tools: "mercury"
+          - name: pcap_stats
+            tools: "pcap_stats"
+          - name: snort
+            tools: "snort"
     steps:
     - uses: actions/checkout@v2
       with:
         fetch-depth: 0
-    - name: Define list of images to build
-      id: get_images
-      run: |
-        echo ::set-output name=NAMES::${NAMES}
-      env:
-        NAMES: rbqwrapper pcap_to_node_pcap tcprewrite_dot1q network_tap/ncapture network_tap p0f mercury pcap_stats snort
     - name: Get version number
       id: get_version
       run: |
@@ -76,11 +88,9 @@ jobs:
         VERSION: ${{ steps.get_version.outputs.VERSION }}
     - name: Test building only
       run: |
-        for N in ${{ steps.get_images.outputs.NAMES }}
-        do
-          TAGGED_IMAGE="${{ secrets.DOCKER_NAMESPACE }}/$(basename ${N}):${{ steps.publish_tag.outputs.TAG }}"
-          docker build \
-            -t ${TAGGED_IMAGE} -f ${N}/Dockerfile .
+        for tool in ${{ matrix.tools }} ; do
+          TAGGED_IMAGE="${{ secrets.DOCKER_NAMESPACE }}/$(basename ${tool}):${{ steps.publish_tag.outputs.TAG }}"
+          docker build -t ${TAGGED_IMAGE} -f "${tool}/Dockerfile" .
         done
       if: steps.docker.outputs.PUSH_TO_DOCKER == 'false'
     - name: Set up qemu
@@ -103,22 +113,20 @@ jobs:
       env:
         DOCKER_CLI_EXPERIMENTAL: enabled
       run: |
-        for N in ${{ steps.get_images.outputs.NAMES }}
-        do
-          TAGGED_IMAGE="${{ secrets.DOCKER_NAMESPACE }}/$(basename ${N}):${{ steps.publish_tag.outputs.TAG }}"
+        for tool in ${{ matrix.tools }} ; do
+          TAGGED_IMAGE="${{ secrets.DOCKER_NAMESPACE }}/$(basename ${tool}):${{ steps.publish_tag.outputs.TAG }}"
           docker buildx build \
             --platform linux/amd64,linux/arm/v7,linux/arm64 \
             --push \
-            -t ${TAGGED_IMAGE} -f ${N}/Dockerfile .
+            -t ${TAGGED_IMAGE} -f "${tool}/Dockerfile" .
         done
       if: success() && (steps.docker.outputs.PUSH_TO_DOCKER == 'true')
     - name: List available tags for images
       env:
         DOCKER_CLI_EXPERIMENTAL: enabled
       run: |
-        for N in ${{ steps.get_images.outputs.NAMES }}
-        do
-          image="${{ secrets.DOCKER_NAMESPACE }}/$(basename ${N})"
+        for tool in ${{ matrix.tools }} ; do
+          image="${{ secrets.DOCKER_NAMESPACE }}/$(basename ${tool})"
           echo "${image}:" $(
             wget -q https://registry.hub.docker.com/v1/repositories/${image}/tags -O - |
               tr -d '[]" ' |

snort/Dockerfile

@@ -28,12 +28,14 @@ RUN apt-get update && \
         unzip && \
         apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 COPY snort/downloadhelper.sh /opt/downloadhelper.sh
+# hadolint ignore=DL3010
+COPY snort/*.tar.gz /opt/
 WORKDIR /opt
-RUN /opt/downloadhelper.sh https://www.snort.org/downloads/snort/daq-${DAQ_VERSION}.tar.gz daq.tgz && tar zxvf daq.tgz && mv daq-${DAQ_VERSION} daq && rm daq.tgz
+RUN /opt/downloadhelper.sh https://www.snort.org/downloads/snort/daq-${DAQ_VERSION}.tar.gz daq-${DAQ_VERSION}.tar.gz && tar zxvf daq-${DAQ_VERSION}.tar.gz && mv daq-${DAQ_VERSION} daq && rm daq-${DAQ_VERSION}.tar.gz
 WORKDIR /opt/daq
 RUN autoreconf -f -i && ./configure && make && make install
 WORKDIR /opt
-RUN /opt/downloadhelper.sh https://www.snort.org/downloads/snort/snort-${SNORT_VERSION}.tar.gz snort.tgz && tar zxvf snort.tgz && mv snort-${SNORT_VERSION} snort && rm snort.tgz
+RUN /opt/downloadhelper.sh https://www.snort.org/downloads/snort/snort-${SNORT_VERSION}.tar.gz snort-${SNORT_VERSION}.tar.gz && tar zxvf snort-${SNORT_VERSION}.tar.gz && mv snort-${SNORT_VERSION} snort && rm snort-${SNORT_VERSION}.tar.gz
 WORKDIR /opt/snort
 RUN ./configure --disable-open-appid && make && make install
 

snort/daq-2.0.7.tar.gz

@@ -9,14 +9,23 @@ tarfile=$2
 retries=5
 if [[ "$url" == "" || "$tarfile" == "" ]] ; then
 	echo need URL and tarfile
+	exit 1
+fi
+
+if [[ -f "$tarfile" ]] ; then
+	echo $tarfile exists, skipping download
+	exit 0
 fi
 
 i=0
 while [ $i -lt $retries ]; do
 	i=$((i+1))
 	rm -f $outfile
+	# TODO: workaround curl segfault in getaddrinfo() handling redirect under qemu
+	finalurl=$(curl -Ls -w %{url_effective} -o /dev/null $url)
+	echo final URL: $finalurl
 	# TODO: snort binary serving does not work with TLS 1.3
-	curl -Lv $url --tlsv1.2 --tls-max 1.2 --output $tarfile --trace -
+	curl -v "$finalurl" --tlsv1.2 --tls-max 1.2 --output $tarfile --trace -
 	tar ztvf $tarfile
 	tarstatus=$?
 	if [[ -f "$tarfile" && $tarstatus -eq 0 ]] ; then
@@ -25,7 +34,7 @@ while [ $i -lt $retries ]; do
 		break
 	fi
 	echo retrying....
-	sleep 5
+	sleep 60
 done
 
 echo failed to download $tarfile.