Merge pull request #5 from okurz/feature/count-fail-ratio

Add count-fail-ratio with tests

Makefile

@@ -15,7 +15,7 @@ test-more-bash:
 
 checkbashisms:
 	@command -v wget >/dev/null 2>&1 || echo "Command 'wget' not found, can not download checkbashisms"
-	wget -q https://salsa.debian.org/debian/devscripts/-/raw/master/scripts/checkbashisms.pl -O checkbashisms
+	wget -q https://salsa.debian.org/debian/devscripts/-/raw/main/scripts/checkbashisms.pl -O checkbashisms
 	chmod +x checkbashisms
 	command -v checkbashisms >/dev/null || echo "Downloaded checkbashisms. You can check the file and add to PATH, then call make again"
 
@@ -35,3 +35,4 @@ test-checkbashisms:
 .PHONY: install
 install:
 	install -m 755 retry "$(DESTDIR)"/usr/bin/retry
+	install -m 755 count-fail-ratio "$(DESTDIR)"/usr/bin/count-fail-ratio

README.md

@@ -43,6 +43,30 @@ timeout per execution:
 retry -- timeout 10 $cmd
 ```
 
+### count-fail-ratio - simple statistics about retries
+
+Another tool provided is "count-fail-ratio" which can count fails, the fail
+ratio, the failure probability and timing information from repeated command
+calls. Simply call
+
+```
+count-fail-ratio $cmd
+```
+
+to execute `$cmd` automatically multiple times collecting the mentioned
+statistics.
+
+Further options to count-fail-ratio can be specified by runtime variables. For
+example to change the number of runs from the default of 20 set the variable
+"runs" while disabling computing timing information:
+
+```
+runs=100 timing=0 count-fail-ratio $cmd
+```
+
+For the complete list of variables take a look into the script file
+count-fail-ratio itself.
+
 ## Contribute
 
 This project lives in https://github.com/okurz/retry
@@ -79,6 +103,11 @@ make test
 make checkstyle
 ```
 
+## Alternatives
+
+* https://github.com/minfrin/retry - an older C implementation which is also
+  available in Debian and Ubuntu.
+
 ## License
 
 This project is licensed under the MIT license, see LICENSE file for details.

count-fail-ratio

@@ -0,0 +1,45 @@
+#!/bin/bash -e
+# shellcheck disable=SC2048
+[ "$1" = "-h" ] || [ "$1" = "--help" ] && echo "Run an arbitrary command multiple times and count failures and fail ratio" && exit
+
+fails="${fails:-0}"
+runs="${runs:-"20"}"
+start="${start:-1}"
+timing="${timing:-1}"
+if [ "$timing" = 1 ]; then t_start=$(date +%s%N); fi
+declare -a times=()
+for ((i=start; i <= runs; i++)); do
+    echo "## Run $i"
+    if [ "$timing" = 1 ]; then t_run_start=$(date +%s%N); fi
+    $* || fails=$((fails+1))
+    if [ "$timing" = 1 ]; then
+        t_run_end=$(date +%s%N)
+        runtime=$(( (t_run_end - t_run_start) / 1000000 ))
+        times+=("$runtime")
+    fi
+    p=$(bc <<< "scale=9;${fails}/${i}")
+    standard_error=$(bc <<< "scale=9;sqrt(${p}*(1 - ${p})/${i})")
+    # critical value (z_value) for a 95% confidence level. In this
+    # case, the critical value is approximately 1.96.
+    z_value=1.96
+    me=$(bc <<< "scale=9;${z_value}*${standard_error}")
+    echo -n "## $(basename "$0"): Run: $i. Fails: $fails. Fail ratio $(bc <<< "r=${p} * 100;scale=2;r/1")±$(bc <<< "r=${me}*100;scale=2;r/1")%"
+    [[ $fails = 0 ]] && echo -n ". No fails, computed failure probability < $(bc <<< "scale=2;3 * 100/${i}")%"
+    echo ""
+    if [ "$timing" = 1 ]; then
+        t_end=$(date +%s%N)
+        # Compute standard deviation
+        sum=0
+        for time in "${times[@]}"; do
+            sum=$((sum + time))
+        done
+        mean=$((sum / i))
+        variance=0
+        for time in "${times[@]}"; do
+            diff=$((time - mean))
+            variance=$((variance + diff*diff))
+        done
+        stddev=$(bc <<< "scale=2;sqrt(${variance}/${i})")
+        echo "## mean runtime: $(( (t_end - t_start) / i / 1000000 ))±$stddev ms"
+    fi
+done

dist/rpm/retry.spec

@@ -42,5 +42,6 @@ mkdir -p %{buildroot}%{_bindir}
 
 %files
 %{_bindir}/retry
+%{_bindir}/count-fail-ratio
 
 %changelog

test/01-count-fail-ratio.t

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+set -e
+dir=$(cd "$(dirname "${BASH_SOURCE[0]}")"; pwd)
+
+TEST_MORE_PATH=$dir/../test-more-bash
+BASHLIB="`
+    find $TEST_MORE_PATH -type d |
+    grep -E '/(bin|lib)$' |
+    xargs -n1 printf "%s:"`"
+PATH=$BASHLIB$PATH
+
+source bash+ :std
+use Test::More
+plan tests 9
+
+call_cmd() {
+    $dir/../count-fail-ratio $*
+}
+
+rc=0
+output=$(runs=3 call_cmd true 2>&1) || rc=$?
+is "$rc" 0 'successful run for no fails'
+like "$output" 'Run: 3. Fails: 0. Fail ratio 0.*%. No fails, computed failure probability < 100.00%' 'counted all successes'
+
+rc=0
+output=$(runs=30 call_cmd true 2>&1) || rc=$?
+is "$rc" 0 'successful run for many no fails'
+like "$output" 'Run: 30. Fails: 0. Fail ratio 0.*%.*< 10.00%' 'computed failure probability lowers to < 10% for enough runs'
+
+rc=0
+output=$(runs=3 call_cmd false 2>&1) || rc=$?
+is "$rc" 0 'successful run for all fails'
+like "$output" 'count-fail-ratio: Run: 3. Fails: 3. Fail ratio 100.00.*%' 'counted all fails'
+
+rc=0
+tmp="${tmp:-"/tmp/tmp.fail-once-every-third-call"}"
+echo 0 > $tmp
+output=$(runs=10 call_cmd $dir/fail-once-every-third-call 2>&1) || rc=$?
+is "$rc" 0 'successful run for sporadically failing script'
+like "$output" 'count-fail-ratio: Run: 10. Fails: 3. Fail ratio 30.00±28.40%' 'counted sporadic failure'
+
+output=$(runs=1 timing=1 call_cmd false 2>&1)
+like "$output" 'mean runtime' 'timing info shows up'

test/fail-once-every-third-call

@@ -0,0 +1,12 @@
+#!/bin/bash
+TMPDIR="${TMPDIR:-"/tmp"}"
+tmp="${tmp:-"$TMPDIR/tmp.$(basename "$0")"}"
+if [ -e "$tmp" ]; then
+    attempts="$(cat "$tmp")"
+fi
+if [[ -z "$attempts" ]]; then
+    attempts=0
+fi
+((attempts++))
+echo "$attempts" > "$tmp"
+((attempts%3))