|
| 1 | +"""Shared harness for the benchmark scripts in this directory. |
| 2 | +
|
| 3 | +Every benchmark describes its workload as a list of :class:`Vector` |
| 4 | +objects and hands them to :func:`main`, which takes care of running them, |
| 5 | +printing a table and deriving a verdict. A vector is a payload builder |
| 6 | +``n -> sql`` plus the entry point that consumes it, measured over a range |
| 7 | +of sizes:: |
| 8 | +
|
| 9 | + from _harness import Vector, main |
| 10 | +
|
| 11 | + def wide_select(n): |
| 12 | + return 'SELECT ' + ', '.join(f'col_{i}' for i in range(n)) + ' FROM t' |
| 13 | +
|
| 14 | + VECTORS = [Vector('wide select', wide_select, (500, 1000, 2000, 4000))] |
| 15 | +
|
| 16 | + if __name__ == '__main__': |
| 17 | + sys.exit(main('grouping', VECTORS)) |
| 18 | +
|
| 19 | +Absolute timings are host dependent, so the verdict is based on growth |
| 20 | +instead: a least-squares fit of log(time) over log(n) yields an exponent |
| 21 | +of about 1 for linear behaviour and about 2 for quadratic behaviour. |
| 22 | +Benchmarks exit non-zero as soon as one vector grows super-linearly, |
| 23 | +which makes them usable as a regression check for the DoS advisories |
| 24 | +without hard-coding a timing threshold. |
| 25 | +""" |
| 26 | + |
| 27 | +import argparse |
| 28 | +import math |
| 29 | +import signal |
| 30 | +import time |
| 31 | +from collections.abc import Callable |
| 32 | +from dataclasses import dataclass |
| 33 | + |
| 34 | +import sqlparse |
| 35 | + |
| 36 | +DEFAULT_TIMEOUT = 30 |
| 37 | + |
| 38 | +#: Midpoint between linear (1.0) and quadratic (2.0) growth. At or above |
| 39 | +#: this the measured code path is reported as super-linear. |
| 40 | +SUPERLINEAR_EXPONENT = 1.5 |
| 41 | + |
| 42 | +#: Measurements below this are dominated by interpreter noise and are left |
| 43 | +#: out of the fit; a vector without enough usable points is inconclusive. |
| 44 | +MIN_SIGNIFICANT_MS = 1.0 |
| 45 | + |
| 46 | +_LINEAR = 'linear' |
| 47 | +_SUPERLINEAR = 'super-linear' |
| 48 | +_INCONCLUSIVE = 'inconclusive' |
| 49 | + |
| 50 | + |
| 51 | +@dataclass |
| 52 | +class Vector: |
| 53 | + """One measured code path. |
| 54 | +
|
| 55 | + :param name: Shown as the table heading. |
| 56 | + :param build: Builds the payload for a given size, ``n -> sql``. |
| 57 | + :param sizes: Sizes to measure, ascending. |
| 58 | + :param run: Entry point under test, defaults to :func:`sqlparse.parse`. |
| 59 | + """ |
| 60 | + |
| 61 | + name: str |
| 62 | + build: Callable[[int], str] |
| 63 | + sizes: tuple |
| 64 | + run: Callable[[str], object] = sqlparse.parse |
| 65 | + |
| 66 | + |
| 67 | +class Timeout(Exception): |
| 68 | + """Raised when a single measurement exceeds the timeout.""" |
| 69 | + |
| 70 | + |
| 71 | +def _alarm_handler(signum, frame): |
| 72 | + raise Timeout |
| 73 | + |
| 74 | + |
| 75 | +# signal.alarm() is Unix-only; elsewhere a benchmark simply runs to |
| 76 | +# completion, which is acceptable for a developer tool. |
| 77 | +_CAN_ALARM = hasattr(signal, 'SIGALRM') |
| 78 | +if _CAN_ALARM: |
| 79 | + signal.signal(signal.SIGALRM, _alarm_handler) |
| 80 | + |
| 81 | + |
| 82 | +def measure(run, sql, timeout=DEFAULT_TIMEOUT): |
| 83 | + """Run ``run(sql)`` once, return ``(status, elapsed milliseconds)``. |
| 84 | +
|
| 85 | + The status is ``ok``, ``cap`` when a grouping limit rejected the input |
| 86 | + before the measured path was reached, or ``timeout``. |
| 87 | + """ |
| 88 | + if _CAN_ALARM: |
| 89 | + signal.alarm(timeout) |
| 90 | + status = 'ok' |
| 91 | + t0 = time.perf_counter() |
| 92 | + try: |
| 93 | + run(sql) |
| 94 | + except sqlparse.exceptions.SQLParseError: |
| 95 | + status = 'cap' |
| 96 | + except Timeout: |
| 97 | + status = 'timeout' |
| 98 | + finally: |
| 99 | + if _CAN_ALARM: |
| 100 | + signal.alarm(0) |
| 101 | + return status, (time.perf_counter() - t0) * 1000 |
| 102 | + |
| 103 | + |
| 104 | +def _format_size(n_bytes): |
| 105 | + if n_bytes < 10_000: |
| 106 | + return f'{n_bytes} B' |
| 107 | + return f'{n_bytes / 1000:.1f} kB' |
| 108 | + |
| 109 | + |
| 110 | +def _exponent(points): |
| 111 | + """Least-squares slope of log(time) over log(n) for ``(n, ms)`` pairs. |
| 112 | +
|
| 113 | + Only the largest sizes are fitted. At small n the linear part of the |
| 114 | + work (lexing the payload) dominates and pulls the slope of a |
| 115 | + quadratic path down towards 1, which is exactly where a regression |
| 116 | + would slip through; the asymptotic end of the range is what the |
| 117 | + verdict should be based on. |
| 118 | + """ |
| 119 | + if len(points) < 2: |
| 120 | + return None |
| 121 | + points = points[-max(3, (len(points) + 1) // 2):] |
| 122 | + xs = [math.log(n) for n, _ in points] |
| 123 | + ys = [math.log(ms) for _, ms in points] |
| 124 | + mean_x = sum(xs) / len(xs) |
| 125 | + mean_y = sum(ys) / len(ys) |
| 126 | + variance = sum((x - mean_x) ** 2 for x in xs) |
| 127 | + if variance == 0: |
| 128 | + return None |
| 129 | + covariance = sum((x - mean_x) * (y - mean_y) |
| 130 | + for x, y in zip(xs, ys, strict=True)) |
| 131 | + return covariance / variance |
| 132 | + |
| 133 | + |
| 134 | +def run_vector(vector, timeout=DEFAULT_TIMEOUT): |
| 135 | + """Measure one vector, print its table and return its verdict.""" |
| 136 | + print(f'{vector.name}:') |
| 137 | + print(f' {"n":>8} {"input":>9} {"time":>11} {"ratio":>7} status') |
| 138 | + |
| 139 | + points = [] |
| 140 | + previous = None |
| 141 | + for n in vector.sizes: |
| 142 | + sql = vector.build(n) |
| 143 | + status, elapsed = measure(vector.run, sql, timeout) |
| 144 | + ratio = f'{elapsed / previous:.2f}x' if previous else '-' |
| 145 | + print(f' {n:>8} {_format_size(len(sql)):>9} {elapsed:>8.1f} ms ' |
| 146 | + f'{ratio:>7} {status}') |
| 147 | + previous = elapsed |
| 148 | + if status == 'ok' and elapsed >= MIN_SIGNIFICANT_MS: |
| 149 | + points.append((n, elapsed)) |
| 150 | + |
| 151 | + exponent = _exponent(points) |
| 152 | + if exponent is None: |
| 153 | + print(' too few usable measurements => inconclusive\n') |
| 154 | + return _INCONCLUSIVE |
| 155 | + |
| 156 | + verdict = _SUPERLINEAR if exponent >= SUPERLINEAR_EXPONENT else _LINEAR |
| 157 | + print(f' scaling exponent {exponent:.2f} ' |
| 158 | + f'(1.0 linear, 2.0 quadratic) => {verdict}\n') |
| 159 | + return verdict |
| 160 | + |
| 161 | + |
| 162 | +def _parse_args(argv, title, vectors): |
| 163 | + parser = argparse.ArgumentParser( |
| 164 | + description=f'sqlparse benchmark: {title}', |
| 165 | + formatter_class=argparse.ArgumentDefaultsHelpFormatter) |
| 166 | + parser.add_argument( |
| 167 | + '--sizes', type=lambda s: tuple(int(p) for p in s.split(',')), |
| 168 | + help='comma-separated sizes, overriding the per-vector defaults') |
| 169 | + parser.add_argument( |
| 170 | + '--timeout', type=int, default=DEFAULT_TIMEOUT, |
| 171 | + help='abort a single measurement after this many seconds') |
| 172 | + parser.add_argument( |
| 173 | + '--vector', action='append', metavar='SUBSTRING', |
| 174 | + help='only run vectors whose name contains SUBSTRING ' |
| 175 | + f'(available: {", ".join(v.name for v in vectors)})') |
| 176 | + return parser.parse_args(argv) |
| 177 | + |
| 178 | + |
| 179 | +def main(title, vectors, note=None, argv=None): |
| 180 | + """Run `vectors` and return a process exit code. |
| 181 | +
|
| 182 | + Exit code 0 means no vector grew super-linearly, 1 means at least one |
| 183 | + did -- i.e. the quadratic behaviour a fix was meant to remove is back. |
| 184 | + """ |
| 185 | + args = _parse_args(argv, title, vectors) |
| 186 | + if args.vector: |
| 187 | + wanted = [s.lower() for s in args.vector] |
| 188 | + vectors = [v for v in vectors |
| 189 | + if any(s in v.name.lower() for s in wanted)] |
| 190 | + if not vectors: |
| 191 | + print(f'No vector matches {args.vector}.') |
| 192 | + return 2 |
| 193 | + |
| 194 | + print(f'sqlparse benchmark: {title}') |
| 195 | + if note: |
| 196 | + print(note) |
| 197 | + print() |
| 198 | + |
| 199 | + verdicts = [ |
| 200 | + run_vector( |
| 201 | + vector if args.sizes is None |
| 202 | + else Vector(vector.name, vector.build, args.sizes, vector.run), |
| 203 | + args.timeout) |
| 204 | + for vector in vectors |
| 205 | + ] |
| 206 | + |
| 207 | + superlinear = verdicts.count(_SUPERLINEAR) |
| 208 | + inconclusive = verdicts.count(_INCONCLUSIVE) |
| 209 | + if superlinear: |
| 210 | + overall = _SUPERLINEAR |
| 211 | + elif inconclusive: |
| 212 | + overall = f'{_LINEAR} where measurable' |
| 213 | + else: |
| 214 | + overall = _LINEAR |
| 215 | + plural = '' if len(verdicts) == 1 else 's' |
| 216 | + summary = f'{len(verdicts)} vector{plural}, {superlinear} super-linear' |
| 217 | + if inconclusive: |
| 218 | + summary += f', {inconclusive} inconclusive' |
| 219 | + print(f'verdict: {overall} ({summary})') |
| 220 | + return 1 if superlinear else 0 |
0 commit comments