cstrans-df-run: introduce the --shell-form option

.github/workflows/macos.yml

@@ -31,4 +31,4 @@ jobs:
           brew install boost boost-python3 help2man
 
       - name: Build and check
-        run: make distcheck CMAKE='cmake -DPython3_EXECUTABLE=/opt/homebrew/bin/python3.12'
+        run: make distcheck CMAKE="cmake -DPython3_EXECUTABLE=/opt/homebrew/bin/$(brew info boost-python3 | grep -oE 'python@3.\d+' | tr -d @)"

src/cstrans-df-run.cc

@@ -33,11 +33,31 @@ typedef std::vector<std::string> TStringList;
 
 const char *prog_name;
 
+/// read non-empty list from value map "vm" with key "key" and store to "dst"
+template<class TDst, class TMap>
+bool readListFromValMap(TDst *pDst, const TMap &vm, const char *key)
+{
+    if (!vm.count(key))
+        // "key" not found in "vm"
+        return false;
+
+    // store the list to "*pDst" and return "true" if the list is not empty
+    TDst &dst = *pDst;
+    dst = vm[key].template as<TDst>();
+    return !dst.empty();
+}
+
+/// transformation properties set on cmd-line
+struct TransformerProps {
+    TStringList prefixCmd;  ///< cmd-line operands
+    bool shellForm;         ///< if true, write shell form of RUN lines
+    bool verbose;           ///< if true, print in/out of each transformation
+};
+
 class DockerFileTransformer {
     public:
-        DockerFileTransformer(const TStringList &prefixCmd, const bool verbose):
-            prefixCmd_(prefixCmd),
-            verbose_(verbose),
+        DockerFileTransformer(const TransformerProps &tp):
+            tp_(tp),
             lineNum_(0)
         {
         }
@@ -46,9 +66,8 @@ class DockerFileTransformer {
         bool transform(std::istream &in, std::ostream &out);
 
     private:
-        const TStringList   prefixCmd_;         ///< cmd-line operands
-        const bool          verbose_;           ///< --verbose on cmd-line
-        int                 lineNum_;           ///< line number being read
+        const TransformerProps  tp_;            ///< props set on cmd-line
+        int                     lineNum_;       ///< line number being read
 
         void transformRunLine(std::string *);
 
@@ -66,6 +85,9 @@ class DockerFileTransformer {
 
         /// match in-line comments
         const RE reComment_     = RE("^\\s*#.*$");
+
+        /// construct transformed RUN command from execList
+        std::string runCmdFromExecList(const TStringList &execList);
 };
 
 /// parse serialized list in the form: "item1", "item2", ...
@@ -141,19 +163,51 @@ void appendShellExec(TStringList *pExecList, const std::string &str)
     pExecList->push_back(str);
 }
 
-/// precede each back-slash and each quote by back-slash
-std::string runQuoteArg(std::string arg)
+/// precede selected special chars by back-slash
+std::string runQuoteArg(std::string arg, const bool escapeTick=false)
 {
     boost::algorithm::replace_all(arg, "\\", "\\\\");
     boost::algorithm::replace_all(arg, "\"", "\\\"");
     boost::algorithm::replace_all(arg, "\n", "\\n");
     boost::algorithm::replace_all(arg, "\t", "\\t");
+
+    if (escapeTick)
+        boost::algorithm::replace_all(arg, "'", "\\'");
+
     return arg;
 }
 
-std::string runCmdFromExecList(const TStringList &execList)
+/// construct "'cmd' 'arg1' 'arg2' ..." from execList
+std::string runShellCmdFromExecList(const TStringList &execList)
+{
+    std::string cmd;
+
+    int i = 0;
+    for (const std::string &arg : execList) {
+        if (i++)
+            // space-separator
+            cmd += " ";
+
+        const std::string quoted = runQuoteArg(arg, /* escapeTick */ true);
+        if (quoted != arg)
+            // construct $'...'
+            cmd += "$";
+
+        // append the quoted arg
+        cmd += "'" + quoted + "'";
+    }
+
+    return cmd;
+}
+
+/// construct transformed RUN command from execList
+std::string DockerFileTransformer::runCmdFromExecList(
+        const TStringList &execList)
 {
-    // construct RUN ["cmd", "arg1", "arg2", ...] from execList
+    if (this->tp_.shellForm)
+        return runShellCmdFromExecList(execList);
+
+    // construct ["cmd", "arg1", "arg2", ...] from execList
     std::string runLine = "[";
     int i = 0;
     for (const std::string &arg : execList) {
@@ -178,7 +232,7 @@ void DockerFileTransformer::transformRunLine(std::string *pRunLine)
     const std::string cmd = sm[2];
 
     // start with the prefix specified on cmd-line
-    TStringList execList = prefixCmd_;
+    TStringList execList = tp_.prefixCmd;
 
     if (boost::regex_match(cmd, sm, reLineRunExec_))
         // ["cmd", "arg1", "arg2", ...]
@@ -187,8 +241,8 @@ void DockerFileTransformer::transformRunLine(std::string *pRunLine)
         // arbitrary shell code...
         appendShellExec(&execList, cmd);
 
-    newRunLine += runCmdFromExecList(execList);
-    if (verbose_) {
+    newRunLine += this->runCmdFromExecList(execList);
+    if (tp_.verbose) {
         // diagnostic output printed with --verbose
         std::cerr << prog_name << " <<< " << *pRunLine << std::endl;
         std::cerr << prog_name << " >>> " << newRunLine << std::endl;
@@ -314,6 +368,8 @@ int main(int argc, char *argv[])
         desc.add_options()
             ("in-place,i", po::value<std::string>(),
              "modify the specified file in-place")
+            ("shell-form", "write transformed RUN lines using the shell form"
+             " (rather than exec form, which is the default format)")
             ("verbose", "print transformations to standard error output");
 
         desc.add_options()
@@ -351,21 +407,19 @@ int main(int argc, char *argv[])
         return 0;
     }
 
-    const bool verbose = !!vm.count("verbose");
-
-    if (!vm.count("prefix-cmd")) {
-        desc.print(std::cerr);
-        return 1;
-    }
+    // read cmd-line flags
+    TransformerProps tp;
+    tp.shellForm    = !!vm.count("shell-form");
+    tp.verbose      = !!vm.count("verbose");
 
-    const TStringList &prefixCmd = vm["prefix-cmd"].as<TStringList>();
-    if (prefixCmd.empty()) {
+    // read the prefix command
+    if (!readListFromValMap(&tp.prefixCmd, vm, "prefix-cmd")) {
         desc.print(std::cerr);
         return 1;
     }
 
-    // pass cmd-line args to DockerFileTransformer
-    DockerFileTransformer dft(prefixCmd, verbose);
+    // create the transformer object
+    DockerFileTransformer dft(tp);
 
     if (vm.count("in-place"))
         // transform Dockerfile in-place

tests/cstrans-df-run/0001-sf-stdout.txt

@@ -0,0 +1,38 @@
+FROM openshift/golang-builder:1.11 AS builder
+ENV SOURCE_GIT_COMMIT=39c2c111ea818cd16dbd11e31cc682cf2b4042d3 SOURCE_GIT_TAG=golang-github-prometheus-alertmanager-4.0.0-0.100.0-115-g39c2c11
+
+ARG ALERTMANAGER_GOPATH=/go/src/github.com/prometheus/alertmanager
+ARG BUILD_PROMU=false
+COPY . ${ALERTMANAGER_GOPATH}
+RUN '/opt/cov-sa-2019.09/bin/cov-build' '--dir=/cov' '--append-log' 'sh' '-c' 'cd ${ALERTMANAGER_GOPATH} &&     yum install -y prometheus-promu &&     make build &&     yum clean all'
+
+FROM ubi7:7.6-159
+ENV SOURCE_GIT_COMMIT=39c2c111ea818cd16dbd11e31cc682cf2b4042d3 SOURCE_GIT_TAG=golang-github-prometheus-alertmanager-4.0.0-0.100.0-115-g39c2c11
+
+ARG ALERTMANAGER_GOPATH=/go/src/github.com/prometheus/alertmanager
+COPY --from=builder ${ALERTMANAGER_GOPATH}/amtool                       /bin/amtool
+COPY --from=builder ${ALERTMANAGER_GOPATH}/alertmanager                 /bin/alertmanager
+COPY --from=builder ${ALERTMANAGER_GOPATH}/examples/ha/alertmanager.yml /etc/alertmanager/alertmanager.yml
+
+EXPOSE     9093
+RUN '/opt/cov-sa-2019.09/bin/cov-build' '--dir=/cov' '--append-log' 'sh' '-c' 'mkdir -p /alertmanager'
+VOLUME     [ "/alertmanager" ]
+WORKDIR    /etc/alertmanager
+ENTRYPOINT [ "/bin/alertmanager" ]
+CMD        [ "--storage.path=/alertmanager" ]
+
+LABEL \
+        io.k8s.description="This is the Prometheus Alert Manager image." \
+        com.redhat.component="golang-github-prometheus-alertmanager-container" \
+        maintainer="OpenShift Development <[email protected]>" \
+        name="openshift/ose-prometheus-alertmanager" \
+        License="ASL 2.0" \
+        io.k8s.display-name="Prometheus Alert Manager" \
+        io.openshift.build.source-location="https://github.com/openshift/prometheus-alertmanager" \
+        io.openshift.build.commit.url="https://github.com/openshift/prometheus-alertmanager/commit/39c2c111ea818cd16dbd11e31cc682cf2b4042d3" \
+        version="v4.1.0" \
+        io.openshift.build.commit.id="39c2c111ea818cd16dbd11e31cc682cf2b4042d3" \
+        release="201905191700" \
+        vendor="Red Hat" \
+        io.openshift.tags="openshift,prometheus"
+

tests/cstrans-df-run/0002-sf-stdout.txt

@@ -0,0 +1,13 @@
+FROM openshift/golang-builder:1.11 AS builder
+ENV SOURCE_GIT_COMMIT=39c2c111ea818cd16dbd11e31cc682cf2b4042d3 SOURCE_GIT_TAG=golang-github-prometheus-alertmanager-4.0.0-0.100.0-115-g39c2c11
+COPY . ${ALERTMANAGER_GOPATH}
+RUN '/opt/cov-sa-2019.09/bin/cov-build' '--dir=/cov' '--append-log' 'sh' '-c' 'cd ${ALERTMANAGER_GOPATH} &&  yum install -y prometheus-promu &&  make build &&  yum clean all'
+FROM ubi7:7.6-159
+ENV SOURCE_GIT_COMMIT=39c2c111ea818cd16dbd11e31cc682cf2b4042d3 SOURCE_GIT_TAG=golang-github-prometheus-alertmanager-4.0.0-0.100.0-115-g39c2c11
+
+CMD        [ "--storage.path=/alertmanager" ]
+
+LABEL \
+        io.k8s.description="This is the Prometheus Alert Manager image." \
+        com.redhat.component="golang-github-prometheus-alertmanager-container" \
+        maintainer="OpenShift Development <[email protected]>"

tests/cstrans-df-run/0003-sf-stdout.txt

@@ -0,0 +1,13 @@
+FROM openshift/golang-builder:1.11 AS builder
+ENV SOURCE_GIT_COMMIT=39c2c111ea818cd16dbd11e31cc682cf2b4042d3 SOURCE_GIT_TAG=golang-github-prometheus-alertmanager-4.0.0-0.100.0-115-g39c2c11
+COPY . ${ALERTMANAGER_GOPATH}
+RUN '/opt/cov-sa-2019.09/bin/cov-build' '--dir=/cov' '--append-log' 'make'
+FROM ubi7:7.6-159
+ENV SOURCE_GIT_COMMIT=39c2c111ea818cd16dbd11e31cc682cf2b4042d3 SOURCE_GIT_TAG=golang-github-prometheus-alertmanager-4.0.0-0.100.0-115-g39c2c11
+
+CMD        [ "--storage.path=/alertmanager" ]
+
+LABEL \
+        io.k8s.description="This is the Prometheus Alert Manager image." \
+        com.redhat.component="golang-github-prometheus-alertmanager-container" \
+        maintainer="OpenShift Development <[email protected]>"

tests/cstrans-df-run/0004-sf-stdout.txt

@@ -0,0 +1,13 @@
+FROM openshift/golang-builder:1.11 AS builder
+ENV SOURCE_GIT_COMMIT=39c2c111ea818cd16dbd11e31cc682cf2b4042d3 SOURCE_GIT_TAG=golang-github-prometheus-alertmanager-4.0.0-0.100.0-115-g39c2c11
+COPY . ${ALERTMANAGER_GOPATH}
+RUN '/opt/cov-sa-2019.09/bin/cov-build' '--dir=/cov' '--append-log' 'go' 'build' '--verbose'
+FROM ubi7:7.6-159
+ENV SOURCE_GIT_COMMIT=39c2c111ea818cd16dbd11e31cc682cf2b4042d3 SOURCE_GIT_TAG=golang-github-prometheus-alertmanager-4.0.0-0.100.0-115-g39c2c11
+
+CMD        [ "--storage.path=/alertmanager" ]
+
+LABEL \
+        io.k8s.description="This is the Prometheus Alert Manager image." \
+        com.redhat.component="golang-github-prometheus-alertmanager-container" \
+        maintainer="OpenShift Development <[email protected]>"

tests/cstrans-df-run/0005-sf-stdout.txt

@@ -0,0 +1,13 @@
+FROM openshift/golang-builder:1.11 AS builder
+ENV SOURCE_GIT_COMMIT=39c2c111ea818cd16dbd11e31cc682cf2b4042d3 SOURCE_GIT_TAG=golang-github-prometheus-alertmanager-4.0.0-0.100.0-115-g39c2c11
+COPY . ${ALERTMANAGER_GOPATH}
+RUN '/opt/cov-sa-2019.09/bin/cov-build' '--dir=/cov' '--append-log' 'sh' '-c' $'echo \"OK\"'
+FROM ubi7:7.6-159
+ENV SOURCE_GIT_COMMIT=39c2c111ea818cd16dbd11e31cc682cf2b4042d3 SOURCE_GIT_TAG=golang-github-prometheus-alertmanager-4.0.0-0.100.0-115-g39c2c11
+
+CMD        [ "--storage.path=/alertmanager" ]
+
+LABEL \
+        io.k8s.description="This is the Prometheus Alert Manager image." \
+        com.redhat.component="golang-github-prometheus-alertmanager-container" \
+        maintainer="OpenShift Development <[email protected]>"

tests/cstrans-df-run/0006-sf-stdout.txt

@@ -0,0 +1,13 @@
+FROM openshift/golang-builder:1.11 AS builder
+ENV SOURCE_GIT_COMMIT=39c2c111ea818cd16dbd11e31cc682cf2b4042d3 SOURCE_GIT_TAG=golang-github-prometheus-alertmanager-4.0.0-0.100.0-115-g39c2c11
+COPY . ${ALERTMANAGER_GOPATH}
+RUN '/opt/cov-sa-2019.09/bin/cov-build' '--dir=/cov' '--append-log' 'sh' '-c' $'echo \"OK\"'
+FROM ubi7:7.6-159
+ENV SOURCE_GIT_COMMIT=39c2c111ea818cd16dbd11e31cc682cf2b4042d3 SOURCE_GIT_TAG=golang-github-prometheus-alertmanager-4.0.0-0.100.0-115-g39c2c11
+
+CMD        [ "--storage.path=/alertmanager" ]
+
+LABEL \
+        io.k8s.description="This is the Prometheus Alert Manager image." \
+        com.redhat.component="golang-github-prometheus-alertmanager-container" \
+        maintainer="OpenShift Development <[email protected]>"

tests/cstrans-df-run/0007-sf-stdout.txt

@@ -0,0 +1,79 @@
+#
+# This is the OpenShift ovn overlay network image.
+# it provides an overlay network using ovs/ovn/ovn-kube
+#
+# The standard name for this image is ovn-kube
+
+# Notes:
+# This is for a build where the ovn-kubernetes utilities
+# are built in this Dockerfile and included in the image (instead of the rpm)
+#
+
+FROM openshift/golang-builder:1.10 AS builder
+ENV SOURCE_GIT_COMMIT=674977b1fdf0f966970179b9fae338f8347b2dfe SOURCE_GIT_TAG=674977b
+
+WORKDIR /go-controller
+COPY go-controller/ .
+
+# build the binaries
+RUN '/opt/cov-sa-2019.09/bin/cov-build' '--dir=/cov' '--append-log' 'sh' '-c' 'make'
+
+FROM openshift/ose-cli:v4.1.0-201905191700 AS cli
+ENV SOURCE_GIT_COMMIT=674977b1fdf0f966970179b9fae338f8347b2dfe SOURCE_GIT_TAG=674977b
+
+FROM openshift/ose-base:v4.1.0-201905191700
+ENV SOURCE_GIT_COMMIT=674977b1fdf0f966970179b9fae338f8347b2dfe SOURCE_GIT_TAG=674977b
+
+USER root
+
+ENV PYTHONDONTWRITEBYTECODE yes
+
+# install needed rpms - openvswitch must be 2.9.2 or higher
+# install selinux-policy first to avoid a race
+RUN '/opt/cov-sa-2019.09/bin/cov-build' '--dir=/cov' '--append-log' 'sh' '-c' $'yum install -y  \tselinux-policy && \tyum clean all'
+
+RUN '/opt/cov-sa-2019.09/bin/cov-build' '--dir=/cov' '--append-log' 'sh' '-c' $'yum install -y  \tPyYAML bind-utils \topenssl \tnumactl-libs \tfirewalld-filesystem \tlibpcap \thostname \t\"openvswitch2.11\" \t\"openvswitch2.11-ovn-common\" \t\"openvswitch2.11-ovn-central\" \t\"openvswitch2.11-ovn-host\" \t\"openvswitch2.11-ovn-vtep\" \t\"openvswitch2.11-devel\" \tcontainernetworking-plugins \tiproute strace socat && \tyum clean all'
+
+RUN '/opt/cov-sa-2019.09/bin/cov-build' '--dir=/cov' '--append-log' 'sh' '-c' 'rm -rf /var/cache/yum'
+
+RUN '/opt/cov-sa-2019.09/bin/cov-build' '--dir=/cov' '--append-log' 'sh' '-c' 'mkdir -p /var/run/openvswitch &&     mkdir -p /etc/cni/net.d &&     mkdir -p /opt/cni/bin &&     mkdir -p /usr/libexec/cni/'
+
+COPY --from=builder /go-controller/_output/go/bin/ovnkube /usr/bin/
+COPY --from=builder /go-controller/_output/go/bin/ovn-kube-util /usr/bin/
+COPY --from=builder /go-controller/_output/go/bin/ovn-k8s-cni-overlay /usr/libexec/cni/ovn-k8s-cni-overlay
+
+COPY --from=cli /usr/bin/oc /usr/bin
+RUN '/opt/cov-sa-2019.09/bin/cov-build' '--dir=/cov' '--append-log' 'sh' '-c' 'ln -s /usr/bin/oc /usr/bin/kubectl'
+
+# copy git commit number into image
+COPY .git/HEAD /root/.git/HEAD
+COPY .git/refs/heads/ /root/.git/refs/heads/
+
+# ovnkube.sh is the entry point. This script examines environment
+# variables to direct operation and configure ovn
+COPY dist/images/ovnkube.sh /root/
+COPY dist/images/ovn-debug.sh /root/
+# override the rpm's ovn_k8s.conf with this local copy
+COPY dist/images/ovn_k8s.conf /etc/openvswitch/ovn_k8s.conf
+
+
+
+WORKDIR /root
+ENTRYPOINT /root/ovnkube.sh
+
+LABEL \
+        io.k8s.description="This is a component of OpenShift Container Platform that provides an overlay network using ovn." \
+        com.redhat.component="ose-ovn-kubernetes-container" \
+        maintainer="Phil Cameron <[email protected]>" \
+        name="openshift/ose-ovn-kubernetes" \
+        License="GPLv2+" \
+        io.k8s.display-name="ovn kubernetes" \
+        io.openshift.build.source-location="https://github.com/openshift/ovn-kubernetes" \
+        summary="This is a component of OpenShift Container Platform that provides an overlay network using ovn." \
+        io.openshift.build.commit.url="https://github.com/openshift/ovn-kubernetes/commit/674977b1fdf0f966970179b9fae338f8347b2dfe" \
+        version="v4.1.0" \
+        io.openshift.build.commit.id="674977b1fdf0f966970179b9fae338f8347b2dfe" \
+        release="201905231545" \
+        vendor="Red Hat" \
+        io.openshift.tags="openshift"
+

tests/cstrans-df-run/0008-sf-stdout.txt

@@ -0,0 +1,13 @@
+FROM openshift/golang-builder:1.11 AS builder
+ENV SOURCE_GIT_COMMIT=39c2c111ea818cd16dbd11e31cc682cf2b4042d3 SOURCE_GIT_TAG=golang-github-prometheus-alertmanager-4.0.0-0.100.0-115-g39c2c11
+COPY . ${ALERTMANAGER_GOPATH}
+RUN '/opt/cov-sa-2019.09/bin/cov-build' '--dir=/cov' '--append-log' 'sh' '-c' $'echo \tOK\n'
+FROM ubi7:7.6-159
+ENV SOURCE_GIT_COMMIT=39c2c111ea818cd16dbd11e31cc682cf2b4042d3 SOURCE_GIT_TAG=golang-github-prometheus-alertmanager-4.0.0-0.100.0-115-g39c2c11
+
+CMD        [ "--storage.path=/alertmanager" ]
+
+LABEL \
+        io.k8s.description="This is the Prometheus Alert Manager image." \
+        com.redhat.component="golang-github-prometheus-alertmanager-container" \
+        maintainer="OpenShift Development <[email protected]>"