perfcheck

#!/bin/bash

GO_PACKAGE=github.com/jumptrading/influx-spout
REFERENCE_REVISION=${REFERENCE_REVISION:-1f340bedb40f69fb9624ad600e0930141150ff2f}

max_attempts=3
iterations=3
orig_gopath=$GOPATH

echo "Comparing working tree against $REFERENCE_REVISION"

fatal() {
    echo $1
    exit 1
}

title() {
    echo ">>>> $1"
}

capture_benchmarks() {
    test_sizes=$1
    output=$2

    go test -tags="$test_sizes" -run='^$' -bench=. ./... &>> $output
    if [[ $? -ne 0 ]]; then
        cat $output
        exit 1
    fi
}

# Validate test sizes
test_sizes=$@
if [[ $test_sizes == "" ]]; then
    fatal "at least one size of tests must be selected ('small', 'medium' or 'large')"
fi
for test_size in $test_sizes; do
    case $test_size in
        small) ;; medium) ;; large) ;;
        *) fatal "invalid test size (value must be 'small', 'medium' or 'large')" ;;
    esac
done

current_bench_output="$PWD/current.bench"
reference_bench_output="$PWD/reference.bench"

title "Building benchcheck tool"
benchcheck_bin=cmd/benchcheck/benchcheck
go build -o $benchcheck_bin ./cmd/benchcheck || exit 1

title "Setting up reference branch"

# Create a temporary GOPATH which gets removed on exit.
ref_gopath=`mktemp -d`
function cleanup() {
    rm -rf $ref_gopath
}
trap cleanup EXIT

# Clone the repo into the temporary GOPATH
set -e
clone_dir=$ref_gopath/src/$GO_PACKAGE
git clone --quiet . $clone_dir
pushd $clone_dir > /dev/null

title "Fetching upstream reference branch"
git remote add upstream https://github.com/jumptrading/influx-spout.git
git fetch --depth=10 upstream perfcheck-reference

git checkout --quiet -b perfcheck $REFERENCE_REVISION > /dev/null
popd > /dev/null
set +e


# Run the tests for the current benchmarks and reference benchmarks
# $iterations times. The runs are interleaved to minimise the effects
# of other load on the host.
for a in `seq $max_attempts`; do
    title "Attempt $a ($max_attempts max)"

    # Remove output from previous runs
    rm -f $reference_bench_output $current_bench_output

    for i in `seq $iterations`; do
        title "Iteration $i/$iterations"

        title "Running current benchmarks"
        export GOPATH=$orig_gopath
        capture_benchmarks "$test_sizes" $current_bench_output

        title "Running reference benchmarks"
        export GOPATH=$ref_gopath
        pushd $clone_dir > /dev/null
        capture_benchmarks "$test_sizes" $reference_bench_output
        popd > /dev/null
    done

    title "Comparing benchmarks"
    $benchcheck_bin -print $reference_bench_output $current_bench_output && exit 0
done