QATAPP-32770: Dockerfile support with QAT Crypto & Compression acceleration

rshanm8x · krithikx · Yogaraj-Alamenda · commit c37d7ef11d34 · 2024-11-08T12:21:25.000+05:30
Co-authored-by: Krithika Kumaravelu &lt;krithikax.kumaravelu@intel.com&gt;
Signed-off-by: Rajesh Shanmugam &lt;rajesh1x.shanmugam@intel.com&gt;
diff --git a/README.md b/README.md
@@ -99,6 +99,8 @@ and [QATzip](https://github.com/intel/QATzip#installation-instructions) installa
 * Refer QAT Settings [here](https://intel.github.io/quickassist/GSG/2.X/installation.html#running-applications-as-non-root-user) for running Nginx
 under non-root user.
 
+Also there is dockerfile available for Async mode nginx with QATlib which can be built into docker images. Please refer [here](dockerfiles/README.md) for more details.
+
 ## Testing
 
 ### Official Unit tests
diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile
@@ -0,0 +1,191 @@
+#==========================================================================
+#                                                                         \
+#                                                                         \
+#   BSD LICENSE                                                           \
+#                                                                         \
+#   Copyright(c) 2024 Intel Corporation.                                  \
+#   All rights reserved.                                                  \
+#                                                                         \
+#   Redistribution and use in source and binary forms, with or without    \
+#   modification, are permitted provided that the following conditions    \
+#   are met:                                                              \
+#                                                                         \
+#     * Redistributions of source code must retain the above copyright    \
+#       notice, this list of conditions and the following disclaimer.     \
+#     * Redistributions in binary form must reproduce the above copyright \
+#       notice, this list of conditions and the following disclaimer in   \
+#       the documentation and/or other materials provided with the        \
+#       distribution.                                                     \
+#     * Neither the name of Intel Corporation nor the names of its        \
+#       contributors may be used to endorse or promote products derived   \
+#       from this software without specific prior written permission.     \
+#                                                                         \
+#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS   \
+#   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT     \
+#   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR \
+#   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT  \
+#   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, \
+#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT      \
+#   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, \
+#   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY \
+#   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT   \
+#   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE \
+#   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.  \
+#                                                                         \
+#                                                                         \
+#==========================================================================
+
+ARG UBUNTU_BASE=ubuntu:22.04
+FROM ${UBUNTU_BASE} AS builder
+
+ARG OPENSSL_VERSION="openssl-3.0.15"
+ARG QATLIB_VERSION="24.02.0"
+ARG QAT_ENGINE_VERSION="v1.6.2"
+ARG IPSEC_MB_VERSION="v1.5"
+ARG IPP_CRYPTO_VERSION="ippcp_2021.12.1"
+ARG QATZIP_VERSION="v1.2.0"
+ARG ASYNC_NGINX_VERSION="v0.5.2"
+ARG GID
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install required packages
+RUN apt-get update && \
+    apt-get install -y apt-utils
+
+# Upgrade all other packages
+RUN apt-get upgrade -y && \
+    apt-get install -y \
+    libudev-dev \
+    make \
+    gcc \
+    g++ \
+    nasm \
+    pkg-config \
+    libssl-dev \
+    libpcre3-dev \
+    zlib1g-dev \
+    libreadline-dev \
+    lua5.4 \
+    liblua5.4-dev \
+    nasm \
+    autoconf \
+    automake \
+    cmake \
+    git \
+    ca-certificates \
+    liblz4-dev \
+    libtool && \
+    git clone --depth 1 -b $OPENSSL_VERSION https://github.com/openssl/openssl.git && \
+    git clone --depth 1 -b $QAT_ENGINE_VERSION https://github.com/intel/QAT_Engine && \
+    git clone --depth 1 -b $IPP_CRYPTO_VERSION https://github.com/intel/ipp-crypto && \
+    git clone --depth 1 -b $IPSEC_MB_VERSION https://github.com/intel/intel-ipsec-mb && \
+    git clone --depth 1 -b $QATLIB_VERSION https://github.com/intel/qatlib && \
+    git clone --depth 1 -b $QATZIP_VERSION https://github.com/intel/QATzip.git && \
+    git clone --depth 1 -b $ASYNC_NGINX_VERSION https://github.com/intel/asynch_mode_nginx
+
+# Create a non-root user and group
+RUN groupadd -r appuser && useradd -r -g appuser -s /bin/bash appuser
+
+# Build and Install OpenSSL
+WORKDIR /openssl
+RUN ./config && \
+    make -j && \
+    make install -j
+
+# Build and Install QATLib
+WORKDIR /qatlib
+RUN ./autogen.sh && \
+    ./configure --enable-systemd=no && \
+    make -j && \
+    make install samples-install && \
+    groupadd qat -g ${GID} && \
+    usermod -a -G qat appuser
+
+# Build and Install Crypto-MB
+WORKDIR /ipp-crypto/sources/ippcp/crypto_mb
+RUN cmake . -B"../build" \
+    -DOPENSSL_INCLUDE_DIR=/usr/local/include \
+    -DOPENSSL_LIBRARIES=/usr/local/lib64 \
+    -DOPENSSL_ROOT_DIR=/openssl
+
+WORKDIR /ipp-crypto/sources/ippcp/build
+RUN make crypto_mb -j && make install -j
+
+# Building the Ipsec-MB
+WORKDIR /intel-ipsec-mb
+RUN make -j && make install LIB_INSTALL_DIR=/usr/local/lib
+
+# Build & Install QATEngine
+WORKDIR /QAT_Engine
+RUN ./autogen.sh && \
+    ./configure \
+    --with-openssl_install_dir=/usr/local/ \
+    --with-qat-hw-dir=/usr/local/ \
+    --enable-qat_sw && \
+    make -j && make install -j
+
+# Build & Install QATzip
+WORKDIR /QATzip
+RUN ./autogen.sh && \
+    ./configure && \
+    make -j && make install -j
+
+# Build & Install Asynch_mode_nginx
+WORKDIR /asynch_mode_nginx
+RUN ./configure \
+    --prefix=/var/www \
+    --conf-path=/usr/share/nginx/conf/nginx.conf \
+    --sbin-path=/usr/bin/ \
+    --with-http_ssl_module \
+    --with-http_stub_status_module \
+    --add-dynamic-module=modules/nginx_qatzip_module \
+    --add-dynamic-module=modules/nginx_qat_module/ \
+    --with-cc-opt="-DNGX_SECURE_MEM -I/usr/local/include -I/usr/local/include/qat -Wno-error=deprecated-declarations" \
+    --with-ld-opt="-Wl,-rpath=/usr/local/lib64 -L/usr/local/lib64 -L/QATzip/src -lqatzip -lz" && \
+    make -j && \
+    make install -j
+
+#Added to remove libc library for vulnerability issue
+RUN apt-get purge -y linux-libc-dev
+
+FROM ${UBUNTU_BASE}
+
+COPY --from=builder /usr/local/lib/libqat.so.4.2.0 /usr/lib/
+COPY --from=builder /usr/local/lib/libusdm.so.0.1.0 /usr/lib/
+COPY --from=builder /usr/local/lib/libIPSec_MB.so.1.5.0 /usr/lib/x86_64-linux-gnu/
+COPY --from=builder /usr/local/lib64/libcrypto.so.3 /usr/lib/x86_64-linux-gnu/
+COPY --from=builder /usr/local/lib/libcrypto_mb.so.11.15 /usr/lib/x86_64-linux-gnu/
+COPY --from=builder /usr/local/bin/openssl /usr/bin/
+COPY --from=builder /usr/local/lib64/engines-3/qatengine.so /usr/lib/x86_64-linux-gnu/engines-3/qatengine.so
+COPY --from=builder /var/www/ /var/www/
+COPY --from=builder /usr/bin/nginx /usr/bin/nginx
+COPY --from=builder /usr/share/nginx/conf/* /usr/share/nginx/conf/
+COPY --from=builder /usr/share/nginx/conf/nginx.conf /usr/share/nginx/conf/nginx.conf
+COPY --from=builder /usr/local/lib/libqatzip.so.3.0.3 /usr/lib/x86_64-linux-gnu/
+COPY --from=builder /usr/local/bin/qzip /usr/bin/qzip
+COPY --from=builder /usr/local/bin/qatzip-test /usr/bin/qatzip-test
+COPY --from=builder /etc/group /etc/group
+COPY --from=builder /etc/passwd /etc/passwd
+RUN touch /var/www/logs/error.log /var/www/logs/nginx.pid /var/www/logs/access.log && chown appuser /var/www/logs/error.log /var/www/logs/nginx.pid /var/www/logs/access.log
+
+RUN ldconfig
+
+# Create necessary directories and set permissions
+RUN chown -R appuser:appuser /usr/share/nginx /var/www/ /usr/bin/nginx /usr/bin/qzip && \
+    chmod -R 777 /usr/share/nginx /var/www/ /usr/bin/nginx /usr/bin/qzip
+
+RUN mkdir -p /var/www/client_body_temp /var/www/proxy_temp /var/www/scgi_temp /var/www/uwsgi_temp /var/www/fastcgi_temp /var/www/html/basic_status && \
+    chown -R appuser:appuser /var/www/client_body_temp /var/www/proxy_temp /var/www/scgi_temp /var/www/uwsgi_temp /var/www/fastcgi_temp /var/www/html/basic_status && \
+    chmod 755 /var/www/client_body_temp /var/www/proxy_temp /var/www/scgi_temp /var/www/uwsgi_temp /var/www/fastcgi_temp /var/www/html/basic_status
+
+#Switch to non-root user
+USER appuser
+
+ENV OPENSSL_ENGINES="/usr/lib/x86_64-linux-gnu/engines-3/"
+ENV LD_LIBRARY_PATH="/usr/lib/x86_64-linux-gnu/"
+ENV QAT_POLICY=1
+
+#Expose ports
+EXPOSE 8080 
+
+CMD ["/usr/bin/nginx", "-g", "daemon off;"]
diff --git a/dockerfiles/README.md b/dockerfiles/README.md
@@ -0,0 +1,78 @@
+# Intel® QuickAssist Technology(QAT) Async Mode Nginx\* Container support
+
+Async Mode Nginx Dockerfile contains Crypto and Compression acceleration with both QAT_HW and QAT_SW which can be built into docker images on the platforms with [Intel® QuickAssist 4xxx Series](https://www.intel.com/content/www/us/en/products/details/processors/xeon/scalable.html) QAT device.
+
+This Dockerfile(qat_crypto_base+compression/Dockerfile) with qatengine is built and validated on top of OpenSSL-3.0.15, QAT_HW(qatlib intree driver) and QAT_SW with software versions mentioned in [software_requirements](../README.md#software_requirements) section.
+
+## Docker setup and testing
+
+Refer [here](https://intel.github.io/quickassist/AppNotes/Containers/setup.html)
+for setting up the host for QAT_HW (qatlib intree) if the platform has QAT 4xxx Hardware
+device. Stop QAT service if any running on the host.
+
+### QAT_HW settings
+
+Follow the below steps to enable required service. The service can be asym only, sym only or both
+in step 2 depending on the particular use case. Configure the required service only to get best performance.
+
+1. Bring down the QAT devices
+```
+    for i in `lspci -D -d :4940| awk '{print $1}'`; do echo down > /sys/bus/pci/devices/$i/qat/state;done
+```
+
+2. Set up the required crypto or compression service(s)
+   To enable crypto service use "sym;asym"
+```
+    for i in `lspci -D -d :4940| awk '{print $1}'`; do echo "sym;asym" > /sys/bus/pci/devices/$i/qat/cfg_services;done
+```
+   To enable compression service use "dc" or both means "dc;sym" / "dc;asym" update accordingly in above command.
+
+3. Bring up the QAT devices
+```
+    for i in `lspci -D -d :4940| awk '{print $1}'`; do echo up> /sys/bus/pci/devices/$i/qat/state;done
+```
+
+4. Check the status of the QAT devices
+```
+    for i in `lspci -D -d :4940| awk '{print $1}'`; do cat /sys/bus/pci/devices/$i/qat/state;done
+```
+
+5. Enable VF for the PF in the host
+```
+    for i in `lspci -D -d :4940| awk '{print $1}'`; do echo 16|sudo tee /sys/bus/pci/devices/$i/sriov_numvfs; done
+```
+
+6. Add QAT group and Permission to the VF devices in the host
+```
+    chown root.qat /dev/vfio/*
+    chmod 660 /dev/vfio/*
+```
+
+### Generate certificates
+
+Create the TLS key and certificate for enabling encryption
+
+```
+openssl genrsa -out rsa1k.key.pem 1024
+openssl req -new -x509 -key rsa1k.key.pem -out rsa1k.cert.pem -days 360 -subj "/C=US/ST=State/L=Locality/O=Company/OU=Section/CN=(1024 bit RSA)/emailAddress=xxx@company.com"
+```
+Note: Replace <path> for the absolute path where you want to save the file(/etc/ssl/certs/).
+
+### Image creation
+
+Docker images can be build using the below command with appropiate image name.
+
+```
+docker build --build-arg GID=$(getent group qat | cut -d ':' -f 3) -t <docker_image_name> <path-to-dockerfile> --no-cache
+```
+Note: GID is the group id of qat group in the host.
+
+### Test using Async Nginx\* crypto utility
+
+```
+Server command: docker run --rm -it --cpuset-cpus <2-n+1> --cap-add=IPC_LOCK --security-opt seccomp=unconfined --security-opt apparmor=unconfined $(for i in `ls /dev/vfio/*`; do echo --device $i; done) --env QAT_POLICY=1 --ulimit memlock=524288000:524288000 -v /usr/share/nginx/:/usr/share/nginx/ -v /etc/ssl/certs/:/etc/ssl/certs/ -v /var/www/html/:/var/www/html/ -v /var/www/logs/:/var/www/logs/ -d -p 8080:8080 <docker_image_name>
+
+Client command: openssl  s_time -connect <server_ip>:8080 -new -cipher AES128-GCM-SHA256 -www /10mb_file.html -time 5
+```
+Note: n is number of process or thread. 8080 port to be used for starting the async nginx container using -v /usr/share/nginx/, /etc/ssl/certs/, /var/www/html/ and /var/www/logs/.
+
diff --git a/dockerfiles/nginx.conf_qat b/dockerfiles/nginx.conf_qat
@@ -0,0 +1,81 @@
+worker_processes  1;
+master_process on;
+load_module /var/www/modules/ngx_http_qatzip_filter_module.so;
+load_module /var/www/modules/ngx_ssl_engine_qat_module.so;
+
+ssl_engine {
+    use_engine qatengine;
+    default_algorithms ALL;
+    #default_algorithms RSA,EC,DH,DSA,PKEY; // For Assymmetric PKE only offload
+    #default algorithm CIPHERS; // For symmetric Ciphers only Offload
+    qat_engine {
+        qat_sw_fallback off;
+        qat_offload_mode async;
+        qat_notify_mode poll;
+        qat_poll_mode heuristic;
+    }
+}
+
+worker_rlimit_nofile 1000000;
+
+events {
+    use epoll;
+    worker_connections  204800;
+    multi_accept on;
+    accept_mutex on;
+}
+
+http {
+    include       mime.types;
+    default_type  application/octet-stream;
+    sendfile        on;
+    keepalive_timeout  0;
+    keepalive_requests 0;
+    ssl_buffer_size 65536;
+    gzip_http_version   1.0;
+
+    gzip_proxied any;
+    qatzip_sw no;
+    qatzip_min_length 128;
+    qatzip_comp_level     4;
+    qatzip_buffers 16 8k;
+    qatzip_types text/css text/javascript text/xml text/plain text/x-component application/javascript application/json application/xml application/rss+xml font/truetype font/opentype application/vnd.ms-fontobject image/svg+xml application/octet-stream image/jpeg;
+    qatzip_chunk_size   64k;
+    qatzip_stream_size     256k;
+    qatzip_sw_threshold 256;
+
+    server {
+        listen    8080 backlog=131072 reuseport so_keepalive=off ssl; // For crypto or crypto + compression
+        #listen    8080; // For compression alone
+        server_name  localhost;
+
+        sendfile on;
+        keepalive_timeout 0s;
+        tcp_nopush on;
+        tcp_nodelay on;
+        ssl_verify_client off;
+        ssl_session_tickets on;
+        access_log  off;
+        lingering_close off;
+        lingering_time 1;
+
+        ssl_certificate      /etc/ssl/certs/TestServer.cert.pem;
+        ssl_certificate_key  /etc/ssl/certs/TestServer.key.pem;
+        ssl_session_cache    off;
+        ssl_dhparam          /etc/ssl/certs/dhparam8k.pem;
+        ssl_asynch on;
+        ssl_buffer_size 64k;
+        ssl_session_timeout  300s;
+        ssl_protocols TLSv1.3;
+        ssl_ciphers            ALL;
+        ssl_prefer_server_ciphers   on;
+
+        location / {
+            index  index.html index.htm;
+        }
+
+        error_page   500 502 503 504  /50x.html;
+        location = /50x.html {
+        }
+    }
+}
