armin/xtc
1
0
Fork 0
forked from hanez/xtc
Code Pull requests Activity

Phase 29: add project benchmarks, endurance stress and callback safety checks

Browse source
This commit is contained in:
Johannes Findeisen 2026-10-27 17:07:37 +01:00
commit afc049e2c8
14 changed files with 1032 additions and 43 deletions
Download patch file Download diff file Expand all files Collapse all files

29
tools/benchmark.py
View file

@ -11,6 +11,7 @@ import sys
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument('--workload', choices=('graph', 'project'), default='graph')
p.add_argument('--binary', type=Path, required=True)
p.add_argument('--output', type=Path, required=True)
p.add_argument('--baseline', type=Path)
@ -28,14 +29,21 @@ def main():
context = dict(cpu=cpu, architecture=platform.machine(), compiler=compiler, flags=a.flags,
kernel=platform.release(), scheduler='ordinary process; no realtime scheduling')
cases = []
for nodes in (2, 16, 64, 256):
for channels in (1, 2):
for rate, frames in ((44100, 31), (48000, 128), (96000, 1024)):
result = subprocess.check_output([str(a.binary.resolve()), *map(str, (nodes, channels, rate, frames, a.blocks))], text=True)
case = json.loads(result)
cases.append(case)
print(f'{nodes:3} nodes {channels}ch {rate}/{frames}: compile {case["compile_ms"]:.3f} ms, '
f'mean {case["mean_us"]:.3f} us, p99 {case["p99_us"]:.3f} us, load {case["load"]:.3f}')
if a.workload == 'graph':
matrix = [(nodes, channels, rate, frames, a.blocks)
for nodes in (2, 16, 64, 256) for channels in (1, 2)
for rate, frames in ((44100, 31), (48000, 128), (96000, 1024))]
else:
matrix = [(a.blocks, tracks, frames) for tracks in (1, 16, 64)
for frames in (31, 128, 1024)]
for arguments in matrix:
result = subprocess.check_output([str(a.binary.resolve()), *map(str, arguments)], text=True, timeout=120)
case = json.loads(result)
cases.append(case)
print(f'{a.workload} {arguments}: mean {case["mean_us"]:.3f} us, '
f'p99 {case["p99_us"]:.3f} us, misses {case["deadline_misses"]}, '
f'CPU {case["process_cpu_ms"]:.3f} ms, peak RSS {case["peak_rss_kib"]} KiB')
context['workload'] = a.workload
report = dict(schema=1, context=context, cases=cases)
a.output.parent.mkdir(parents=True, exist_ok=True)
a.output.write_text(json.dumps(report, indent=2) + '\n')
@ -43,7 +51,10 @@ def main():
baseline = json.loads(a.baseline.read_text())
if baseline.get('schema') != 1 or any(baseline['context'][k] != context[k] for k in ('cpu', 'architecture', 'compiler', 'flags')):
sys.exit('Baseline host/compiler/flags mismatch; record a baseline for this configuration.')
key = lambda c: (c['nodes'], c['channels'], c['rate'], c['frames'])
if baseline['context'].get('workload', 'graph') != a.workload:
sys.exit('Baseline workload mismatch.')
fields = ('nodes', 'channels', 'rate', 'frames') if a.workload == 'graph' else ('tracks', 'notes', 'points', 'rate', 'frames')
key = lambda c: tuple(c[field] for field in fields)
old = {key(c): c for c in baseline['cases']}
failed = []
for c in cases:

17
tools/stress.py
View file

@ -10,13 +10,26 @@ def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument('--build-dir', type=Path, required=True)
p.add_argument('--gui', action='store_true')
p.add_argument('--plugin', type=Path, help='optional original test LADSPA fixture')
p.add_argument('--rounds', type=int, default=1)
a = p.parse_args()
if not 1 <= a.rounds <= 1000:
p.error('rounds must be 1..1000')
workers = []
try:
for _ in range(2):
workers.append(subprocess.Popen([sys.executable, '-c', 'x=1\nwhile True: x=(x*1664525+1013904223)&0xffffffff']))
for name in ('test_realtime', 'test_reliability', 'test_scale'):
subprocess.run([str((a.build_dir / name).resolve())], check=True, timeout=120)
names = ['test_realtime', 'test_reliability', 'test_scale', 'test_portability',
'test_project_diagnostics', 'test_soak']
if (a.build_dir / 'test_backend').exists():
names.append('test_backend')
for iteration in range(a.rounds):
for name in names:
subprocess.run([str((a.build_dir / name).resolve())], check=True, timeout=120)
if a.plugin:
subprocess.run([str((a.build_dir / 'test_plugins').resolve()),
str(a.plugin.resolve()), 'fixture'], check=True, timeout=120)
print(f'Stress round {iteration + 1}/{a.rounds} passed.', flush=True)
if a.gui:
# The GUI suite pumps timers and forces redraw during engine/edit tests.
subprocess.run(['xvfb-run', '-a', '-s', '-screen 0 1280x800x24',