51 lines
2.1 KiB
Python
51 lines
2.1 KiB
Python
|
#!/usr/bin/env python3
|
||
|
|
"""Verify benchmark gates without depending on noisy real-machine timings."""
|
||
|
|
import json
|
||
|
|
from pathlib import Path
|
||
|
|
import subprocess
|
||
|
|
import sys
|
||
|
|
import tempfile
|
||
|
|
|
||
|
|
with tempfile.TemporaryDirectory(prefix='xtc-benchmark-') as directory:
|
||
|
|
root = Path(directory)
|
||
|
|
binary = root / 'measurement'
|
||
|
|
binary.write_text('''#!/usr/bin/env python3
|
||
|
|
import json, sys
|
||
|
|
nodes, channels, rate, frames, blocks = map(int, sys.argv[1:])
|
||
|
|
print(json.dumps(dict(nodes=nodes, channels=channels, rate=rate, frames=frames,
|
||
|
|
blocks=blocks, mean_us=10, p99_us=12, deadline_misses=0,
|
||
|
|
process_cpu_ms=1, peak_rss_kib=1024)))
|
||
|
|
''')
|
||
|
|
binary.chmod(0o700)
|
||
|
|
report, baseline = root / 'report.json', root / 'baseline.json'
|
||
|
|
command = [sys.executable, 'tools/benchmark.py', '--binary', str(binary),
|
||
|
|
'--output', str(report)]
|
||
|
|
|
||
|
|
def run(arguments, success, message=None):
|
||
|
|
result = subprocess.run(arguments, capture_output=True, text=True, timeout=30)
|
||
|
|
assert (result.returncode == 0) == success, result.stdout + result.stderr
|
||
|
|
if message:
|
||
|
|
assert message in result.stdout + result.stderr
|
||
|
|
|
||
|
|
run(command, True)
|
||
|
|
original = json.loads(report.read_text())
|
||
|
|
baseline.write_text(json.dumps(original))
|
||
|
|
compare = command + ['--baseline', str(baseline)]
|
||
|
|
run(compare, True)
|
||
|
|
changed = json.loads(json.dumps(original))
|
||
|
|
changed['cases'][0]['mean_us'] = 1
|
||
|
|
baseline.write_text(json.dumps(changed))
|
||
|
|
run(compare, False, 'exceeds')
|
||
|
|
changed = json.loads(json.dumps(original))
|
||
|
|
changed['cases'].pop()
|
||
|
|
baseline.write_text(json.dumps(changed))
|
||
|
|
run(compare, False, 'missing baseline case')
|
||
|
|
changed = json.loads(json.dumps(original))
|
||
|
|
changed['context']['flags'] = 'incompatible'
|
||
|
|
baseline.write_text(json.dumps(changed))
|
||
|
|
run(compare, False, 'mismatch')
|
||
|
|
changed = json.loads(json.dumps(original))
|
||
|
|
changed['context']['workload'] = 'project'
|
||
|
|
baseline.write_text(json.dumps(changed))
|
||
|
|
run(compare, False, 'workload mismatch')
|
||
|
|
print('Benchmark matching, regression, missing-case and context gates passed.')
|