diff --git a/Makefile b/Makefile index cc6a5f8..4d4b8db 100644 --- a/Makefile +++ b/Makefile @@ -245,14 +245,14 @@ test-recovery: $(BUILD_DIR)/test_recovery # Linux/GNU-compatible linker test; no instrumentation in production objects. $(BUILD_DIR)/test_reliability: $(BUILD_DIR)/tests/integration/test_reliability.o $(BUILD_DIR)/libxtc.a - $(CC) $(LDFLAGS) -o $@ $^ -lm -ldl -Wl,--wrap=malloc,--wrap=calloc,--wrap=realloc,--wrap=free + $(CC) $(LDFLAGS) -o $@ $^ -lm -ldl -Wl,--wrap=malloc,--wrap=calloc,--wrap=realloc,--wrap=free,--wrap=fopen,--wrap=fread,--wrap=fwrite,--wrap=read,--wrap=write,--wrap=pthread_mutex_lock,--wrap=pthread_cond_wait -pthread .PHONY: test-reliability test-sanitize analyze test-reliability: test $(BUILD_DIR)/test_reliability ./$(BUILD_DIR)/test_reliability test-sanitize: - ASAN_OPTIONS=detect_leaks=1:halt_on_error=1 UBSAN_OPTIONS=halt_on_error=1 $(MAKE) BUILD_DIR="$(BUILD_DIR)/sanitize" CC=clang CFLAGS="-O1 -g -std=c99 -Wall -Wextra -Wpedantic -Wconversion -Wshadow -Iinclude -fsanitize=address,undefined -fno-omit-frame-pointer" LDFLAGS="-fsanitize=address,undefined" test-reliability test-recovery + ASAN_OPTIONS=detect_leaks=1:halt_on_error=1 UBSAN_OPTIONS=halt_on_error=1 $(MAKE) BUILD_DIR="$(BUILD_DIR)/sanitize" CC=clang CFLAGS="-O1 -g -std=c99 -Wall -Wextra -Wpedantic -Wconversion -Wshadow -Iinclude -fsanitize=address,undefined -fno-omit-frame-pointer" LDFLAGS="-fsanitize=address,undefined" test-reliability test-recovery test-scale test-soak analyze: $(PYTHON) tools/analyze.py $(CORE_SOURCES) @@ -350,7 +350,7 @@ $(BUILD_DIR)/benchmark_graph: $(BUILD_DIR)/tests/integration/benchmark_graph.o $ $(CC) $(LDFLAGS) -o $@ $^ -lm -ldl .PHONY: benchmark benchmark: $(BUILD_DIR)/benchmark_graph - $(PYTHON) tools/benchmark.py --binary "$(BUILD_DIR)/benchmark_graph" --output "$(BUILD_DIR)/performance.json" --compiler "$(CC)" --flags "$(CFLAGS)" + $(PYTHON) tools/benchmark.py --binary "$(BUILD_DIR)/benchmark_graph" --output "$(BUILD_DIR)/performance.json" --compiler "$(CC)" --flags "$(CFLAGS)" --blocks "$(BENCHMARK_BLOCKS)" -include $(BUILD_DIR)/tests/integration/benchmark_graph.d $(BUILD_DIR)/test_scale: $(BUILD_DIR)/tests/integration/test_scale.o $(BUILD_DIR)/libxtc.a @@ -359,14 +359,15 @@ $(BUILD_DIR)/test_scale: $(BUILD_DIR)/tests/integration/test_scale.o $(BUILD_DIR test-scale: $(BUILD_DIR)/test_scale ./$(BUILD_DIR)/test_scale test-stress: $(BUILD_DIR)/test_realtime $(BUILD_DIR)/test_reliability $(BUILD_DIR)/test_scale $(if $(GUI_AVAILABLE),$(BUILD_DIR)/test_gui) - $(PYTHON) tools/stress.py --build-dir "$(BUILD_DIR)" $(if $(GUI_AVAILABLE),--gui) + $(PYTHON) tools/stress.py --build-dir "$(BUILD_DIR)" --rounds "$(STRESS_ROUNDS)" $(if $(GUI_AVAILABLE),--gui) -include $(BUILD_DIR)/tests/integration/test_scale.d +BENCHMARK_BLOCKS ?= 300 BENCHMARK_BASELINE ?= docs/performance-baseline.json BENCHMARK_MAX_REGRESSION ?= 1.5 .PHONY: benchmark-compare benchmark-compare: $(BUILD_DIR)/benchmark_graph - $(PYTHON) tools/benchmark.py --binary "$(BUILD_DIR)/benchmark_graph" --output "$(BUILD_DIR)/performance.json" --compiler "$(CC)" --flags "$(CFLAGS)" --baseline "$(BENCHMARK_BASELINE)" --max-regression "$(BENCHMARK_MAX_REGRESSION)" + $(PYTHON) tools/benchmark.py --binary "$(BUILD_DIR)/benchmark_graph" --output "$(BUILD_DIR)/performance.json" --compiler "$(CC)" --flags "$(CFLAGS)" --blocks "$(BENCHMARK_BLOCKS)" --baseline "$(BENCHMARK_BASELINE)" --max-regression "$(BENCHMARK_MAX_REGRESSION)" PACKAGE_ARCH := $(shell uname -m) PACKAGE_FILE := $(BUILD_DIR)/xtc-$(VERSION)-linux-$(PACKAGE_ARCH).tar.gz @@ -430,3 +431,26 @@ test: test-project-diagnostics test-project-inspection: $(BUILD_DIR)/xtc-check $(PYTHON) tests/integration/test_project_inspection.py $(BUILD_DIR)/xtc-check test: test-project-inspection + +$(BUILD_DIR)/test_soak: $(BUILD_DIR)/tests/integration/test_soak.o $(BUILD_DIR)/libxtc.a + $(CC) $(LDFLAGS) -o $@ $^ -lm -ldl +.PHONY: test-soak benchmark-project benchmark-project-compare check-development +test-soak: $(BUILD_DIR)/test_soak + ./$(BUILD_DIR)/test_soak +PROJECT_BENCHMARK_BASELINE ?= docs/project-performance-baseline.json +STRESS_ROUNDS ?= 3 +test-stress: $(BUILD_DIR)/test_portability $(BUILD_DIR)/test_project_diagnostics $(BUILD_DIR)/test_soak $(BACKEND_TEST) +benchmark-project: $(BUILD_DIR)/test_performance + $(PYTHON) tools/benchmark.py --workload project --binary "$(BUILD_DIR)/test_performance" --output "$(BUILD_DIR)/project-performance.json" --compiler "$(CC)" --flags "$(CFLAGS)" --blocks "$(BENCHMARK_BLOCKS)" +benchmark-project-compare: $(BUILD_DIR)/test_performance + $(PYTHON) tools/benchmark.py --workload project --binary "$(BUILD_DIR)/test_performance" --output "$(BUILD_DIR)/project-performance.json" --compiler "$(CC)" --flags "$(CFLAGS)" --blocks "$(BENCHMARK_BLOCKS)" --baseline "$(PROJECT_BENCHMARK_BASELINE)" --max-regression "$(BENCHMARK_MAX_REGRESSION)" +check-development: all test-reliability test-scale test-soak analyze test-sanitize +-include $(BUILD_DIR)/tests/integration/test_soak.d +.PHONY: test-benchmark-tools +test-benchmark-tools: + $(PYTHON) tests/tools/test_benchmark.py +check-development: test-benchmark-tools + +.PHONY: test-stress-plugins +test-stress-plugins: $(BUILD_DIR)/test_realtime $(BUILD_DIR)/test_reliability $(BUILD_DIR)/test_scale $(BUILD_DIR)/test_portability $(BUILD_DIR)/test_project_diagnostics $(BUILD_DIR)/test_soak $(BACKEND_TEST) $(BUILD_DIR)/test_plugins $(BUILD_DIR)/plugin-fixture.so + $(PYTHON) tools/stress.py --build-dir "$(BUILD_DIR)" --rounds "$(STRESS_ROUNDS)" --plugin "$(BUILD_DIR)/plugin-fixture.so" diff --git a/docs/BUILD.md b/docs/BUILD.md index ff61481..47d6456 100644 --- a/docs/BUILD.md +++ b/docs/BUILD.md @@ -257,3 +257,9 @@ configuration explicitly, use a fresh BUILD_DIR with Link static consumers with `-lm -ldl` plus the backend dependencies when used. See LICENSES.md for header/plugin licensing and decision 0009 for rationale. Recording stress/recovery is available with `make test-recording`. + +Phase 29 developer qualification: `make check-development` runs the core build, +regressions, scale/virtual endurance, benchmark-gate checks, Clang analysis and +ASan/UBSan. Run `make benchmark benchmark-project` separately without contention +or sanitizers for timing reports. Optional `make test-stress-plugins` requires +installed LADSPA headers; `make test-stress` supports minimal builds too. diff --git a/docs/PERFORMANCE.md b/docs/PERFORMANCE.md index 75fd925..e772eeb 100644 --- a/docs/PERFORMANCE.md +++ b/docs/PERFORMANCE.md @@ -1,6 +1,6 @@ # Performance and realtime qualification -`make test-performance` runs a hardware-independent 48 kHz, 128-frame stereo +The original Phase 9 `test-performance` ran a hardware-independent 48 kHz, 128-frame stereo workload: 64 graph nodes, 4096 MIDI notes (32 simultaneous), 16 tracks and gain automation. Preparation and block timing use CLOCK_MONOTONIC outside the engine; callback code itself does not clock or log. The test checks finite output and @@ -89,3 +89,130 @@ and bounded per-sample lane searches and may be costly; they are excluded from the gain-chain baseline. Large WAV memory limits remain those in AUDIO_FILES.md. Physical-device endurance, CPU-overload sound quality and a full GUI latency benchmark remain unqualified. + +## Phase 29 qualification workloads + +The current `test-performance` emits one JSON record (the old prose output is +retired). Default: 2,000 stereo blocks at 48 kHz/128, 64 graph nodes, 16 tracks, +1,024 stored notes and 256 automation points, at most 32 active voices. This +changes the Phase 9 workload; its historical numbers above are not a baseline +for this revision. `benchmark-project` runs nine cases: 1/16/64 tracks, +64/1,024/4,096 stored notes, 16/256/1,024 points and 31/128/1,024-frame buffers. +Two long notes per first 16 tracks remain audible. Other notes are scheduled +later to remain within the 32-voice plan limit. The position wraps every 96,000 +samples to keep long runs active. Future notes exercise storage/preparation and +bounded scans rather than additional simultaneous synthesis. Each track has +16 gain points; graph processing uses the same 64-node chain in every case. +The graph matrix separately scales nodes and chain connections from 2 to 256. +These synthetic cases do not qualify every possible dense graph or plugin chain. + +```sh +make benchmark benchmark-project +make benchmark-compare +make benchmark-project-compare +# Baselines must match the current CPU/compiler/flags and workload. +make benchmark-project-compare PROJECT_BENCHMARK_BASELINE=/path/to/report.json +./build/test_performance 10000 64 128 +make test-benchmark-tools +``` + +Both matrices report wall-clock mean/p99/max processing duration and the count +of measured calls exceeding frames/sample-rate. These are offline deadline +exceedances, not actual JACK xruns: there is no server scheduling in the timing +loop. Graph compilation or project construction/preparation is timed separately. +`process_cpu_ms` measures CLOCK_PROCESS_CPUTIME_ID over the complete measured +loop, including timing and finite-output checks; it excludes preparation and +sorting. Project `cpu_audio_ratio` divides that CPU time by simulated audio time. +`peak_rss_kib` is Linux getrusage process high-water RSS, including model, plans, +measurement storage and executable/runtime overhead; it is not current heap size +or a leak test. Graph buffer bytes remain a separate exact buffer-size estimate. +Timing probes, memory queries, sorting and output occur outside engine calls. + +Checked-in `project-performance-baseline.json` provides initial project cases; +the original graph baseline is retained. `graph-performance-phase29.json` adds +a current graph report with CPU/RSS/deadline fields. Gates use the existing +1.5x mean-time budget and 1 us noise allowance, with p99/max/misses and memory +reported for investigation. There is no universal hardware-independent deadline +or RSS pass threshold. `test-benchmark-tools` verifies same-context success, +regression rejection, missing cases and host/workload mismatch without relying +on machine timing. Do not replace a failing baseline without investigating. + +## Endurance, faults and callback audit + +```sh +make test-soak test-stress test-stress-plugins +make test-stress STRESS_ROUNDS=10 +./build/test_soak 10000000 +make check-development +``` + +`test-soak` renders exactly 1,000,000 alternating 31/128/1,024-frame stereo +blocks: 2.282 hours of virtual 48 kHz audio. It checks every output sample, +processed block counts and final publication acknowledgement, with 9,999 +quiescent position/rebuild cycles. It runs faster than wall time and models +position wrap/seek and restart publication; it is not a real device start/stop +or hardware endurance test. The project benchmark separately wraps active MIDI. + +`test-stress` defaults to three rounds under two ordinary CPU workers. Each round +runs concurrent publication, allocation/lifetime guards, maximum model capacity, +project relocation/reload and missing-media retention, save-fault diagnostics, +and the soak test. With JACK headers present it also repeats the deterministic +backend fake's start/stop/seek, rate/buffer/server-loss and failure/retry cases. +Optional GUI tests run after those rounds under the same CPU load. +`test-stress-plugins` adds the original LADSPA fixture's identity/layout/range, +instantiation-error and invalid-metadata cases each round; it needs LADSPA headers. +No command changes a physical device, kills the shared server or loads unknown +plugins. Subprocess deadlines bound each test and workers are cleaned up on exit. + +The reliability executable now interposes fopen/fread/fwrite/read/write, +pthread_mutex_lock and pthread_cond_wait in addition to malloc/calloc/realloc/free. +Guarded host graph/realtime render calls fail immediately on these operations. +This complements the existing native lifetime and plugin callback guards and +control/callback ownership review in RELIABILITY.md. It does not intercept every +libc entry point, raw syscalls, hidden dynamic-module calls, or GUI code by magic; +third-party DSP remains trusted. No known host callback safety violation was found. + +`check-development` combines the ordinary build/tests, capacity/soak checks, +benchmark gate tests, static analysis and ASan/UBSan. The sanitizer target now +includes maximum capacity and virtual endurance. These are opt-in developer +checks requiring installed Clang/Python; no runtime dependency was added. +Sanitized/contended/profiled runs must not become timing baselines. + +## Profiling evidence and remaining costs + +Before considering DSP changes, Phase 29 sampled a 10,000-block, 64-track/128-frame +project with GCC `-O2 -g -pg` and GNU gprof on the documented baseline host. +Flat samples attributed about 30% to mixer automation `value`, 29% to MIDI +`segment`, and 11% to finite-value handling. Instrumentation and inlining affect +attribution; this identifies investigation candidates, not exact production +cost or evidence that checks should be removed. MIDI scans bounded stored-note +arrays and evaluates per-voice, per-sample gain/pan lanes. No DSP optimization +was introduced without a separate correctness/performance justification. + +Reproduce in an isolated build (GNU gprof must already be installed): + +```sh +make BUILD_DIR=build/profile CFLAGS='-O2 -g -pg -std=c99 -Iinclude' LDFLAGS=-pg build/profile/test_performance +GMON_OUT_PREFIX="$PWD/build/profile/gmon" ./build/profile/test_performance 10000 64 128 +gprof build/profile/test_performance build/profile/gmon.* > build/profile/profile.txt +``` + +Remaining limits include per-sample LADSPA automation run(1), stored-note/clip +scans, preloaded WAV memory and immutable-plan/history overlap. This phase does +not qualify physical-device changes, arbitrary plugin hangs/crashes, multi-hour +wall-clock sessions or GUI redraw latency. Virtual endurance and deterministic +faults provide repeatable software evidence, not a hard realtime guarantee. + +Phase 29 observed results (same Threadripper/GCC host, 300 blocks per case): +64-track/128-frame project mean 380.417 us, p99 720.271 us, prepare 11.841 ms, +CPU/audio .142 and peak process RSS 19,296 KiB. The 256-node stereo/128 graph +mean was 286.398 us, p99 484.712 us, compile .300 ms. These case runs observed +zero deadline exceedances. See JSON for all cases and host metadata. + +The project comparison passed. Repeated graph comparisons flagged variable +small cases despite unchanged graph processing code; a 3,000-block run against +the retained Phase 19 baseline still flagged 16-node mono/31-frame mean 5.089 us +above its 1.5x budget. The new graph report is evidence, not a replacement chosen +to make that comparison pass. This shared-host timing alert remains an +investigation item; no CPU/latency guarantee is inferred from the correctness +suite. Longer comparisons can use `BENCHMARK_BLOCKS=3000`. diff --git a/docs/RELIABILITY.md b/docs/RELIABILITY.md index 67a1a7d..c87066f 100644 --- a/docs/RELIABILITY.md +++ b/docs/RELIABILITY.md @@ -125,3 +125,13 @@ Collection acquires completion and deactivates the backend before resetting voices/flags. A completed callback continues silence/holds position until that deactivation. No callback waits or calls JACK control functions. Immediate stop/close handles faults or missing callbacks. See AUDIO_QUALITY.md. + +## Phase 29 repeatable development checks + +`make check-development` combines normal tests, capacity/endurance, deterministic +benchmark-gate tests, static analysis and ASan/UBSan (including scale and soak). +`make test-stress` repeats project/backend/publication faults under CPU contention; +`make test-stress-plugins` adds the original LADSPA failure fixture. The reliability +callback guard now also rejects common host file I/O and blocking mutex/condition +calls. It audits directly linked host code, not arbitrary dynamic-plugin internals. +See PERFORMANCE.md for exact workloads, metrics, profiles and qualification limits. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 09f65ce..c5c5539 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -808,6 +808,21 @@ Establish measurable performance expectations and verify that larger projects re - No known real-time safety violation remains in the engine's own callback path. - Performance regressions can be detected by repeatable tests or benchmark reports. +### Implementation status +Implemented with the existing graph/capacity/fault infrastructure plus Phase 29 +project scaling matrices, CPU/RSS/deadline metrics, checked-in reports, virtual +endurance, repeated reload/device/plugin fault suites, wider host callback guards +and a combined development/sanitizer/static-analysis target. See PERFORMANCE.md +for exact workloads, profiling evidence and qualification limits. + +Validation: normal core/GUI/JACK/plugin/recovery/recording suites, three-round +CPU-contention stress, 2.282-hour virtual playback, minimal build, ASan/UBSan, +static analysis and benchmark-gate regression tests passed. Project timing +comparison passed. Graph comparisons detect shared-host variation (including a +16-node mono/31-frame alert in a longer run); the prior baseline is retained and +this limitation is recorded rather than suppressing the gate. Hardware and +wall-clock endurance remain unqualified. + ## Phase 30 — Release Engineering, Packaging, and Maintenance ### Goal diff --git a/docs/graph-performance-phase29.json b/docs/graph-performance-phase29.json new file mode 100644 index 0000000..07a34dc --- /dev/null +++ b/docs/graph-performance-phase29.json @@ -0,0 +1,422 @@ +{ + "schema": 1, + "context": { + "cpu": "AMD Ryzen Threadripper 1920X 12-Core Processor", + "architecture": "x86_64", + "compiler": "cc (GCC) 16.2.1 20260810", + "flags": "-O2 -g -std=c99 -Wall -Wextra -Wpedantic -Wconversion -Wshadow -Iinclude", + "kernel": "7.2.11-artix1-1", + "scheduler": "ordinary process; no realtime scheduling", + "workload": "graph" + }, + "cases": [ + { + "nodes": 2, + "channels": 1, + "rate": 44100, + "frames": 31, + "blocks": 300, + "compile_ms": 0.003657, + "mean_us": 0.357413, + "p99_us": 0.421001, + "max_us": 0.721004, + "deadline_us": 702.947846, + "load": 0.000508, + "graph_buffer_bytes": 496, + "deadline_misses": 0, + "process_cpu_ms": 0.12127, + "peak_rss_kib": 19308 + }, + { + "nodes": 2, + "channels": 1, + "rate": 48000, + "frames": 128, + "blocks": 300, + "compile_ms": 0.005571, + "mean_us": 1.303103, + "p99_us": 1.363005, + "max_us": 1.512992, + "deadline_us": 2666.666667, + "load": 0.000489, + "graph_buffer_bytes": 2048, + "deadline_misses": 0, + "process_cpu_ms": 0.424738, + "peak_rss_kib": 19308 + }, + { + "nodes": 2, + "channels": 1, + "rate": 96000, + "frames": 1024, + "blocks": 300, + "compile_ms": 0.033223, + "mean_us": 23.244337, + "p99_us": 34.926998, + "max_us": 36.950005, + "deadline_us": 10666.666667, + "load": 0.002179, + "graph_buffer_bytes": 16384, + "deadline_misses": 0, + "process_cpu_ms": 7.388398, + "peak_rss_kib": 19308 + }, + { + "nodes": 2, + "channels": 2, + "rate": 44100, + "frames": 31, + "blocks": 300, + "compile_ms": 0.005841, + "mean_us": 1.71521, + "p99_us": 2.685993, + "max_us": 11.882992, + "deadline_us": 702.947846, + "load": 0.00244, + "graph_buffer_bytes": 496, + "deadline_misses": 0, + "process_cpu_ms": 0.54705, + "peak_rss_kib": 19308 + }, + { + "nodes": 2, + "channels": 2, + "rate": 48000, + "frames": 128, + "blocks": 300, + "compile_ms": 0.008015, + "mean_us": 4.814904, + "p99_us": 9.418, + "max_us": 14.648002, + "deadline_us": 2666.666667, + "load": 0.001806, + "graph_buffer_bytes": 2048, + "deadline_misses": 0, + "process_cpu_ms": 1.505184, + "peak_rss_kib": 19308 + }, + { + "nodes": 2, + "channels": 2, + "rate": 96000, + "frames": 1024, + "blocks": 300, + "compile_ms": 0.011121, + "mean_us": 22.629783, + "p99_us": 31.78, + "max_us": 39.103994, + "deadline_us": 10666.666667, + "load": 0.002122, + "graph_buffer_bytes": 16384, + "deadline_misses": 0, + "process_cpu_ms": 7.04282, + "peak_rss_kib": 19308 + }, + { + "nodes": 16, + "channels": 1, + "rate": 44100, + "frames": 31, + "blocks": 300, + "compile_ms": 0.024256, + "mean_us": 5.00722, + "p99_us": 9.327996, + "max_us": 16.952996, + "deadline_us": 702.947846, + "load": 0.007123, + "graph_buffer_bytes": 3968, + "deadline_misses": 0, + "process_cpu_ms": 1.512578, + "peak_rss_kib": 19308 + }, + { + "nodes": 16, + "channels": 1, + "rate": 48000, + "frames": 128, + "blocks": 300, + "compile_ms": 0.040657, + "mean_us": 8.499017, + "p99_us": 15.569996, + "max_us": 19.628002, + "deadline_us": 2666.666667, + "load": 0.003187, + "graph_buffer_bytes": 16384, + "deadline_misses": 0, + "process_cpu_ms": 2.57139, + "peak_rss_kib": 19308 + }, + { + "nodes": 16, + "channels": 1, + "rate": 96000, + "frames": 1024, + "blocks": 300, + "compile_ms": 0.017684, + "mean_us": 73.465867, + "p99_us": 110.88101, + "max_us": 123.875987, + "deadline_us": 10666.666667, + "load": 0.006887, + "graph_buffer_bytes": 131072, + "deadline_misses": 0, + "process_cpu_ms": 22.071262, + "peak_rss_kib": 19308 + }, + { + "nodes": 16, + "channels": 2, + "rate": 44100, + "frames": 31, + "blocks": 300, + "compile_ms": 0.024356, + "mean_us": 7.90933, + "p99_us": 16.942999, + "max_us": 19.336992, + "deadline_us": 702.947846, + "load": 0.011252, + "graph_buffer_bytes": 3968, + "deadline_misses": 0, + "process_cpu_ms": 2.387202, + "peak_rss_kib": 19308 + }, + { + "nodes": 16, + "channels": 2, + "rate": 48000, + "frames": 128, + "blocks": 300, + "compile_ms": 0.02095, + "mean_us": 18.307943, + "p99_us": 38.874001, + "max_us": 49.833994, + "deadline_us": 2666.666667, + "load": 0.006865, + "graph_buffer_bytes": 16384, + "deadline_misses": 0, + "process_cpu_ms": 5.482221, + "peak_rss_kib": 19308 + }, + { + "nodes": 16, + "channels": 2, + "rate": 96000, + "frames": 1024, + "blocks": 300, + "compile_ms": 0.018595, + "mean_us": 142.5079, + "p99_us": 200.170995, + "max_us": 206.372002, + "deadline_us": 10666.666667, + "load": 0.01336, + "graph_buffer_bytes": 131072, + "deadline_misses": 0, + "process_cpu_ms": 42.820019, + "peak_rss_kib": 19308 + }, + { + "nodes": 64, + "channels": 1, + "rate": 44100, + "frames": 31, + "blocks": 300, + "compile_ms": 0.077537, + "mean_us": 8.674007, + "p99_us": 13.575991, + "max_us": 25.628004, + "deadline_us": 702.947846, + "load": 0.012339, + "graph_buffer_bytes": 15872, + "deadline_misses": 0, + "process_cpu_ms": 2.598515, + "peak_rss_kib": 19308 + }, + { + "nodes": 64, + "channels": 1, + "rate": 48000, + "frames": 128, + "blocks": 300, + "compile_ms": 0.123505, + "mean_us": 67.732122, + "p99_us": 102.083999, + "max_us": 104.92999, + "deadline_us": 2666.666667, + "load": 0.0254, + "graph_buffer_bytes": 65536, + "deadline_misses": 0, + "process_cpu_ms": 20.251813, + "peak_rss_kib": 19308 + }, + { + "nodes": 64, + "channels": 1, + "rate": 96000, + "frames": 1024, + "blocks": 300, + "compile_ms": 0.120299, + "mean_us": 348.007127, + "p99_us": 606.123009, + "max_us": 754.726003, + "deadline_us": 10666.666667, + "load": 0.032626, + "graph_buffer_bytes": 524288, + "deadline_misses": 0, + "process_cpu_ms": 103.942768, + "peak_rss_kib": 19308 + }, + { + "nodes": 64, + "channels": 2, + "rate": 44100, + "frames": 31, + "blocks": 300, + "compile_ms": 0.06793, + "mean_us": 18.297367, + "p99_us": 27.101996, + "max_us": 35.546997, + "deadline_us": 702.947846, + "load": 0.026029, + "graph_buffer_bytes": 15872, + "deadline_misses": 0, + "process_cpu_ms": 5.478211, + "peak_rss_kib": 19308 + }, + { + "nodes": 64, + "channels": 2, + "rate": 48000, + "frames": 128, + "blocks": 300, + "compile_ms": 0.124016, + "mean_us": 90.302024, + "p99_us": 158.481998, + "max_us": 163.279998, + "deadline_us": 2666.666667, + "load": 0.033863, + "graph_buffer_bytes": 65536, + "deadline_misses": 0, + "process_cpu_ms": 27.01664, + "peak_rss_kib": 19308 + }, + { + "nodes": 64, + "channels": 2, + "rate": 96000, + "frames": 1024, + "blocks": 300, + "compile_ms": 0.103767, + "mean_us": 651.941917, + "p99_us": 939.527992, + "max_us": 1111.795005, + "deadline_us": 10666.666667, + "load": 0.06112, + "graph_buffer_bytes": 524288, + "deadline_misses": 0, + "process_cpu_ms": 194.780424, + "peak_rss_kib": 19308 + }, + { + "nodes": 256, + "channels": 1, + "rate": 44100, + "frames": 31, + "blocks": 300, + "compile_ms": 0.393599, + "mean_us": 36.277767, + "p99_us": 61.768005, + "max_us": 69.022004, + "deadline_us": 702.947846, + "load": 0.051608, + "graph_buffer_bytes": 63488, + "deadline_misses": 0, + "process_cpu_ms": 10.8528, + "peak_rss_kib": 19308 + }, + { + "nodes": 256, + "channels": 1, + "rate": 48000, + "frames": 128, + "blocks": 300, + "compile_ms": 0.316512, + "mean_us": 155.434897, + "p99_us": 279.041007, + "max_us": 334.977012, + "deadline_us": 2666.666667, + "load": 0.058288, + "graph_buffer_bytes": 262144, + "deadline_misses": 0, + "process_cpu_ms": 46.467151, + "peak_rss_kib": 19308 + }, + { + "nodes": 256, + "channels": 1, + "rate": 96000, + "frames": 1024, + "blocks": 300, + "compile_ms": 0.474884, + "mean_us": 1246.40406, + "p99_us": 2426.396997, + "max_us": 2940.154998, + "deadline_us": 10666.666667, + "load": 0.11685, + "graph_buffer_bytes": 2097152, + "deadline_misses": 0, + "process_cpu_ms": 371.849272, + "peak_rss_kib": 19308 + }, + { + "nodes": 256, + "channels": 2, + "rate": 44100, + "frames": 31, + "blocks": 300, + "compile_ms": 0.573331, + "mean_us": 86.74766, + "p99_us": 192.195992, + "max_us": 197.79701, + "deadline_us": 702.947846, + "load": 0.123406, + "graph_buffer_bytes": 63488, + "deadline_misses": 0, + "process_cpu_ms": 25.90803, + "peak_rss_kib": 19308 + }, + { + "nodes": 256, + "channels": 2, + "rate": 48000, + "frames": 128, + "blocks": 300, + "compile_ms": 0.29951, + "mean_us": 286.398197, + "p99_us": 484.712, + "max_us": 567.41001, + "deadline_us": 2666.666667, + "load": 0.107399, + "graph_buffer_bytes": 262144, + "deadline_misses": 0, + "process_cpu_ms": 85.605838, + "peak_rss_kib": 19308 + }, + { + "nodes": 256, + "channels": 2, + "rate": 96000, + "frames": 1024, + "blocks": 300, + "compile_ms": 0.301544, + "mean_us": 2379.64819, + "p99_us": 4935.191013, + "max_us": 5069.556006, + "deadline_us": 10666.666667, + "load": 0.223092, + "graph_buffer_bytes": 2097152, + "deadline_misses": 0, + "process_cpu_ms": 709.538479, + "peak_rss_kib": 19308 + } + ] +} diff --git a/docs/project-performance-baseline.json b/docs/project-performance-baseline.json new file mode 100644 index 0000000..da72e87 --- /dev/null +++ b/docs/project-performance-baseline.json @@ -0,0 +1,185 @@ +{ + "schema": 1, + "context": { + "cpu": "AMD Ryzen Threadripper 1920X 12-Core Processor", + "architecture": "x86_64", + "compiler": "cc (GCC) 16.2.1 20260810", + "flags": "-O2 -g -std=c99 -Wall -Wextra -Wpedantic -Wconversion -Wshadow -Iinclude", + "kernel": "7.2.11-artix1-1", + "scheduler": "ordinary process; no realtime scheduling", + "workload": "project" + }, + "cases": [ + { + "tracks": 1, + "notes": 64, + "points": 16, + "nodes": 64, + "rate": 48000, + "frames": 31, + "blocks": 300, + "wraps": 0, + "prepare_ms": 2.037527, + "mean_us": 22.432314, + "p99_us": 37.752005, + "max_us": 41.168008, + "deadline_us": 645.833333, + "deadline_misses": 0, + "process_cpu_ms": 6.710229, + "cpu_audio_ratio": 0.034633, + "peak_rss_kib": 19296 + }, + { + "tracks": 1, + "notes": 64, + "points": 16, + "nodes": 64, + "rate": 48000, + "frames": 128, + "blocks": 300, + "wraps": 0, + "prepare_ms": 3.156677, + "mean_us": 136.447411, + "p99_us": 266.878007, + "max_us": 339.576, + "deadline_us": 2666.666667, + "deadline_misses": 0, + "process_cpu_ms": 40.746385, + "cpu_audio_ratio": 0.050933, + "peak_rss_kib": 19296 + }, + { + "tracks": 1, + "notes": 64, + "points": 16, + "nodes": 64, + "rate": 48000, + "frames": 1024, + "blocks": 300, + "wraps": 3, + "prepare_ms": 3.142049, + "mean_us": 853.354273, + "p99_us": 1651.11199, + "max_us": 1777.513011, + "deadline_us": 21333.333333, + "deadline_misses": 0, + "process_cpu_ms": 254.818128, + "cpu_audio_ratio": 0.039815, + "peak_rss_kib": 19296 + }, + { + "tracks": 16, + "notes": 1024, + "points": 256, + "nodes": 64, + "rate": 48000, + "frames": 31, + "blocks": 300, + "wraps": 0, + "prepare_ms": 5.05565, + "mean_us": 94.302404, + "p99_us": 186.395002, + "max_us": 235.758998, + "deadline_us": 645.833333, + "deadline_misses": 0, + "process_cpu_ms": 28.104842, + "cpu_audio_ratio": 0.145057, + "peak_rss_kib": 19296 + }, + { + "tracks": 16, + "notes": 1024, + "points": 256, + "nodes": 64, + "rate": 48000, + "frames": 128, + "blocks": 300, + "wraps": 0, + "prepare_ms": 5.312168, + "mean_us": 386.378117, + "p99_us": 692.126996, + "max_us": 694.069997, + "deadline_us": 2666.666667, + "deadline_misses": 0, + "process_cpu_ms": 115.256236, + "cpu_audio_ratio": 0.14407, + "peak_rss_kib": 19296 + }, + { + "tracks": 16, + "notes": 1024, + "points": 256, + "nodes": 64, + "rate": 48000, + "frames": 1024, + "blocks": 300, + "wraps": 3, + "prepare_ms": 5.224572, + "mean_us": 2722.953837, + "p99_us": 4571.800004, + "max_us": 4690.454996, + "deadline_us": 21333.333333, + "deadline_misses": 0, + "process_cpu_ms": 812.60835, + "cpu_audio_ratio": 0.12697, + "peak_rss_kib": 19296 + }, + { + "tracks": 64, + "notes": 4096, + "points": 1024, + "nodes": 64, + "rate": 48000, + "frames": 31, + "blocks": 300, + "wraps": 0, + "prepare_ms": 11.86645, + "mean_us": 144.810444, + "p99_us": 352.410992, + "max_us": 373.500996, + "deadline_us": 645.833333, + "deadline_misses": 0, + "process_cpu_ms": 41.640344, + "cpu_audio_ratio": 0.214918, + "peak_rss_kib": 19296 + }, + { + "tracks": 64, + "notes": 4096, + "points": 1024, + "nodes": 64, + "rate": 48000, + "frames": 128, + "blocks": 300, + "wraps": 0, + "prepare_ms": 11.841152, + "mean_us": 380.41659, + "p99_us": 720.271011, + "max_us": 826.122006, + "deadline_us": 2666.666667, + "deadline_misses": 0, + "process_cpu_ms": 113.546659, + "cpu_audio_ratio": 0.141933, + "peak_rss_kib": 19296 + }, + { + "tracks": 64, + "notes": 4096, + "points": 1024, + "nodes": 64, + "rate": 48000, + "frames": 1024, + "blocks": 300, + "wraps": 3, + "prepare_ms": 12.183272, + "mean_us": 2766.193183, + "p99_us": 3645.336998, + "max_us": 4391.266004, + "deadline_us": 21333.333333, + "deadline_misses": 0, + "process_cpu_ms": 825.19931, + "cpu_audio_ratio": 0.128937, + "peak_rss_kib": 19296 + } + ] +} diff --git a/tests/integration/benchmark_graph.c b/tests/integration/benchmark_graph.c index 300497c..2abb427 100644 --- a/tests/integration/benchmark_graph.c +++ b/tests/integration/benchmark_graph.c @@ -5,8 +5,11 @@ #include #include #include +#include static double now(void) { struct timespec t; if (clock_gettime(CLOCK_MONOTONIC,&t)) abort(); return (double)t.tv_sec+(double)t.tv_nsec/1e9; } +static double cpu_now(void) +{ struct timespec t; if (clock_gettime(CLOCK_PROCESS_CPUTIME_ID,&t)) abort(); return (double)t.tv_sec+(double)t.tv_nsec/1e9; } static int compare(const void *a,const void *b) { double x = *(const double *)a,y = *(const double *)b; return x < y ? -1 : x > y; } static size_t number(const char *s,size_t maximum) @@ -21,7 +24,9 @@ int main(int argc,char **argv) size_t nodes,channels,rate,frames,blocks,i,j,id = 0,previous = 0; xtc_graph *g = NULL; xtc_plan *p = NULL; float *in = NULL,*left = NULL,*right = NULL; - double *times = NULL,begin,compile,total = 0; + double *times = NULL,begin,compile,total = 0,cpu_begin,cpu; + size_t misses = 0; + struct rusage usage; int status = 1; if (argc != 6) return 1; nodes = number(argv[1],256); channels = number(argv[2],2); rate = number(argv[3],768000); @@ -42,18 +47,22 @@ int main(int argc,char **argv) for (i = 0; i < frames; ++i) in[i] = (float)sin((double)i*.1)*.1f; /* Warm caches outside the sample, then measure only the process call. */ for (i = 0; i < 20; ++i) if (xtc_plan_process_stereo(p,in,NULL,NULL,NULL,NULL,left,channels == 2 ? right : NULL,frames) != XTC_OK) goto done; + cpu_begin = cpu_now(); for (i = 0; i < blocks; ++i) { begin = now(); if (xtc_plan_process_stereo(p,in,NULL,NULL,NULL,NULL,left,channels == 2 ? right : NULL,frames) != XTC_OK) goto done; times[i] = now()-begin; total += times[i]; + if (times[i] > (double)frames/(double)rate) ++misses; for (j = 0; j < frames; ++j) if (!isfinite(left[j]) || (channels == 2 && !isfinite(right[j]))) goto done; } + cpu = cpu_now()-cpu_begin; + if (getrusage(RUSAGE_SELF,&usage)) goto done; qsort(times,blocks,sizeof(double),compare); printf("{\"nodes\":%zu,\"channels\":%zu,\"rate\":%zu,\"frames\":%zu,\"blocks\":%zu," "\"compile_ms\":%.6f,\"mean_us\":%.6f,\"p99_us\":%.6f,\"max_us\":%.6f,\"deadline_us\":%.6f," - "\"load\":%.6f,\"graph_buffer_bytes\":%zu}\n",nodes,channels,rate,frames,blocks,compile*1e3,total/(double)blocks*1e6, + "\"load\":%.6f,\"graph_buffer_bytes\":%zu,\"deadline_misses\":%zu,\"process_cpu_ms\":%.6f,\"peak_rss_kib\":%ld}\n",nodes,channels,rate,frames,blocks,compile*1e3,total/(double)blocks*1e6, times[blocks*99/100]*1e6,times[blocks-1]*1e6,(double)frames/(double)rate*1e6, - total/(double)blocks/((double)frames/(double)rate),nodes*frames*2*sizeof(float)); + total/(double)blocks/((double)frames/(double)rate),nodes*frames*2*sizeof(float),misses,cpu*1e3,usage.ru_maxrss); status = 0; done: free(in); free(left); free(right); free(times); xtc_plan_destroy(p); xtc_graph_destroy(g); return status; diff --git a/tests/integration/test_performance.c b/tests/integration/test_performance.c index 3817735..cd4735a 100644 --- a/tests/integration/test_performance.c +++ b/tests/integration/test_performance.c @@ -6,6 +6,8 @@ #include #include #include +#include +#include #define REQUIRE(c) \ do { \ if (!(c)) { \ @@ -19,6 +21,12 @@ static double now(void) clock_gettime(CLOCK_MONOTONIC, &t); return (double)t.tv_sec + (double)t.tv_nsec / 1e9; } +static double cpu_now(void) +{ + struct timespec t; + if (clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &t)) abort(); + return (double)t.tv_sec + (double)t.tv_nsec / 1e9; +} static int compare(const void *left, const void *right) { double a = *(const double *)left, b = *(const double *)right; @@ -32,39 +40,52 @@ int main(int argc, char **argv) xtc_midi_plan *midi = NULL; long blocks = 2000; char *end; - size_t i, j, source, previous, node; + size_t i, j, source, previous, node, tracks = 16, frames = 128, misses = 0; + size_t point_count, note_count; + uint64_t position = 0, wraps = 0; + struct rusage usage; uint64_t track, clip, note, future; - float left[128], right[128], out_left[128], out_right[128]; - double *times = NULL, total = 0, begin, prepare_ms; + float left[1024], right[1024], out_left[1024], out_right[1024]; + double *times = NULL, total = 0, begin, prepare_ms, cpu_begin, cpu; int result = EXIT_FAILURE; - if (argc > 2) + if (argc != 1 && argc != 2 && argc != 4) return EXIT_FAILURE; - if (argc == 2) { + if (argc >= 2) { blocks = strtol(argv[1], &end, 10); - if (!*argv[1] || *end || blocks < 1 || blocks > 100000) + if (!*argv[1] || *end || blocks < 1 || blocks > 10000000) return EXIT_FAILURE; } + if (argc == 4) { + long value = strtol(argv[2], &end, 10); + if (!*argv[2] || *end || value < 1 || value > 64) return EXIT_FAILURE; + tracks = (size_t)value; + value = strtol(argv[3], &end, 10); + if (!*argv[3] || *end || value < 1 || value > 1024) return EXIT_FAILURE; + frames = (size_t)value; + } times = calloc((size_t)blocks, sizeof(*times)); REQUIRE(times); begin = now(); REQUIRE(xtc_project_create(&project) == XTC_OK); - for (i = 0; i < 16; ++i) { + for (i = 0; i < tracks; ++i) { xtc_automation_point point; REQUIRE(xtc_project_add_track(project, "Performance", &track) == XTC_OK); REQUIRE(xtc_project_add_midi_clip(project, track, 0, 384000, &clip) == XTC_OK); REQUIRE(xtc_project_set_mix(project, track, 0.1f, i % 2 ? 0.5f : -0.5f, 0, 0) == XTC_OK); for (j = 0; j < 2; ++j) - REQUIRE(xtc_project_add_note(project, clip, 0, 384000, 48 + (unsigned int)i, 100, 0, + REQUIRE(xtc_project_add_note(project, clip, i < 16 ? 0 : 380000 + i * 2, i < 16 ? 370000 : 1, 48 + (unsigned int)i, 100, 0, ¬e) == XTC_OK); point = (xtc_automation_point){track, 0, XTC_AUTOMATION_GAIN, 0.1f,0}; REQUIRE(xtc_project_set_automation(project, &point) == XTC_OK); - point.tick = 384000; - point.value = 0.2f; - REQUIRE(xtc_project_set_automation(project, &point) == XTC_OK); + for (j = 1; j < 16; ++j) { + point.tick = j * 25600; + point.value = j % 2 ? 0.2f : 0.1f; + REQUIRE(xtc_project_set_automation(project, &point) == XTC_OK); + } } REQUIRE(xtc_project_add_midi_clip(project, track, 1000000, 10000, &future) == XTC_OK); - for (i = 32; i < XTC_PROJECT_MAX_NOTES; ++i) - REQUIRE(xtc_project_add_note(project, future, (i - 32) * 2, 1, 60, 100, 0, ¬e) == + for (i = tracks * 2; i < tracks * 64; ++i) + REQUIRE(xtc_project_add_note(project, future, (i - tracks * 2) * 2, 1, 60, 100, 0, ¬e) == XTC_OK); REQUIRE(xtc_graph_create(64, &graph) == XTC_OK && xtc_graph_add_instrument(graph, &source) == XTC_OK); @@ -79,26 +100,40 @@ int main(int argc, char **argv) previous = node; REQUIRE(xtc_graph_add_output(graph, &node) == XTC_OK && xtc_graph_connect(graph, previous, node) == XTC_OK); - REQUIRE(xtc_graph_compile(graph, 48000, 128, &plan) == XTC_OK && + REQUIRE(xtc_graph_compile(graph, 48000, frames, &plan) == XTC_OK && xtc_midi_plan_create(project, 48000, &midi) == XTC_OK); prepare_ms = (now() - begin) * 1000; + REQUIRE(xtc_project_note_count(project, ¬e_count) == XTC_OK); + REQUIRE(xtc_project_automation_count(project, &point_count) == XTC_OK); + cpu_begin = cpu_now(); for (i = 0; i < (size_t)blocks; ++i) { begin = now(); - REQUIRE(xtc_midi_plan_render_stereo(midi, i * 128, NULL, 0, left, right, 128) == XTC_OK); + REQUIRE(xtc_midi_plan_render_stereo(midi, position, NULL, 0, left, right, frames) == XTC_OK); REQUIRE(xtc_plan_process_stereo(plan, NULL, NULL, NULL, left, right, out_left, out_right, - 128) == XTC_OK); + frames) == XTC_OK); times[i] = now() - begin; total += times[i]; - for (j = 0; j < 128; ++j) + if (times[i] > (double)frames / 48000) ++misses; + position += frames; + /* Revisit the active arrangement in long offline runs. Explicit sample + * positions exercise discontinuous seek/wrap handling in the MIDI plan. */ + if (position >= 96000) { position = 0; ++wraps; } + + for (j = 0; j < frames; ++j) REQUIRE(isfinite(out_left[j]) && isfinite(out_right[j])); } + cpu = cpu_now() - cpu_begin; + REQUIRE(getrusage(RUSAGE_SELF, &usage) == 0); qsort(times, (size_t)blocks, sizeof(*times), compare); - printf("Offline 48kHz/128, 64 nodes, 4096 notes (32 active), 16 automated tracks; %ld blocks\n", - blocks); - printf("Prepare %.3f ms; block mean %.3f us, p99 %.3f us, max %.3f us; deadline 2666.667 us; " - "CPU/audio %.4f\n", - prepare_ms, total / (double)blocks * 1e6, times[(size_t)blocks * 99 / 100] * 1e6, - times[blocks - 1] * 1e6, total / ((double)blocks * 128 / 48000)); + printf("{\"tracks\":%zu,\"notes\":%zu,\"points\":%zu,\"nodes\":64," + "\"rate\":48000,\"frames\":%zu,\"blocks\":%ld,\"wraps\":%" PRIu64 "," + "\"prepare_ms\":%.6f,\"mean_us\":%.6f,\"p99_us\":%.6f,\"max_us\":%.6f," + "\"deadline_us\":%.6f,\"deadline_misses\":%zu,\"process_cpu_ms\":%.6f," + "\"cpu_audio_ratio\":%.6f,\"peak_rss_kib\":%ld}\n", + tracks, note_count, point_count, frames, blocks, wraps, prepare_ms, + total / (double)blocks * 1e6, times[(size_t)blocks * 99 / 100] * 1e6, + times[blocks - 1] * 1e6, (double)frames / 48000 * 1e6, misses, cpu * 1e3, + cpu / ((double)blocks * (double)frames / 48000), usage.ru_maxrss); result = EXIT_SUCCESS; cleanup: free(times); diff --git a/tests/integration/test_reliability.c b/tests/integration/test_reliability.c index 1bf5a52..b42d0b7 100644 --- a/tests/integration/test_reliability.c +++ b/tests/integration/test_reliability.c @@ -6,12 +6,37 @@ #include #include #include +#include +#include void *__real_malloc(size_t size); void *__real_calloc(size_t count, size_t size); void *__real_realloc(void *pointer, size_t size); void __real_free(void *pointer); static int processing, fail_after = -1; +/* Catch direct host I/O and blocking synchronization in guarded callbacks. + * Dynamic dependencies have their own symbols and are outside this audit. */ +FILE *__real_fopen(const char *path, const char *mode); +size_t __real_fread(void *buffer, size_t size, size_t count, FILE *file); +size_t __real_fwrite(const void *buffer, size_t size, size_t count, FILE *file); +ssize_t __real_read(int fd, void *buffer, size_t size); +ssize_t __real_write(int fd, const void *buffer, size_t size); +int __real_pthread_mutex_lock(pthread_mutex_t *mutex); +int __real_pthread_cond_wait(pthread_cond_t *condition, pthread_mutex_t *mutex); +FILE *__wrap_fopen(const char *path, const char *mode) +{ assert(!processing); return __real_fopen(path, mode); } +size_t __wrap_fread(void *buffer, size_t size, size_t count, FILE *file) +{ assert(!processing); return __real_fread(buffer, size, count, file); } +size_t __wrap_fwrite(const void *buffer, size_t size, size_t count, FILE *file) +{ assert(!processing); return __real_fwrite(buffer, size, count, file); } +ssize_t __wrap_read(int fd, void *buffer, size_t size) +{ assert(!processing); return __real_read(fd, buffer, size); } +ssize_t __wrap_write(int fd, const void *buffer, size_t size) +{ assert(!processing); return __real_write(fd, buffer, size); } +int __wrap_pthread_mutex_lock(pthread_mutex_t *mutex) +{ assert(!processing); return __real_pthread_mutex_lock(mutex); } +int __wrap_pthread_cond_wait(pthread_cond_t *condition, pthread_mutex_t *mutex) +{ assert(!processing); return __real_pthread_cond_wait(condition, mutex); } static unsigned int prepared, destroyed; static int fail_allocation(void) { diff --git a/tests/integration/test_soak.c b/tests/integration/test_soak.c new file mode 100644 index 0000000..589db6c --- /dev/null +++ b/tests/integration/test_soak.c @@ -0,0 +1,56 @@ +#define _POSIX_C_SOURCE 200809L +/* Accelerated virtual playback: synchronous consumer, no hardware/server. */ +#include "xtc/xtc_realtime.h" +#include +#include +#include +#include +int main(int argc, char **argv) +{ + xtc_graph *graph = NULL; + xtc_realtime *engine = NULL; + xtc_realtime_status status; + size_t gain, output, i, j, frames; + unsigned long blocks = 1000000, iterations = 0; + uint64_t samples = 0; + float input[1024], left[1024], right[1024]; + if (argc > 2) return 1; + if (argc == 2) { + char *end; + blocks = strtoul(argv[1], &end, 10); + if (!*argv[1] || *end || blocks < 100 || blocks > 100000000) return 1; + } + assert(xtc_graph_create(2, &graph) == XTC_OK); + assert(xtc_graph_add_gain(graph, 0.5f, &gain) == XTC_OK); + assert(xtc_graph_add_output(graph, &output) == XTC_OK); + assert(xtc_graph_connect(graph, gain, output) == XTC_OK); + assert(xtc_realtime_create(48000, 1024, &engine) == XTC_OK); + assert(xtc_realtime_submit(engine, graph) == XTC_OK); + for (j = 0; j < 1024; ++j) input[j] = j % 2 ? -0.5f : 0.25f; + for (i = 0; i < blocks; ++i) { + static const size_t sizes[] = {31, 128, 1024}; + frames = sizes[i % 3]; + if (i && i % 100 == 0) { + /* Quiescent control work models stop, seek/wrap and graph restart. + * No hardware lifecycle claims are made by this test. */ + xtc_realtime_collect(engine); + assert(xtc_realtime_position(engine, (i % 200) ? 48000 : 0) == XTC_OK); + assert(xtc_realtime_submit(engine, graph) == XTC_OK); + ++iterations; + } + assert(xtc_realtime_process_stereo(engine, input, NULL, NULL, NULL, NULL, + left, right, frames) == XTC_OK); + for (j = 0; j < frames; ++j) + assert(isfinite(left[j]) && left[j] == input[j] * 0.5f && right[j] == left[j]); + samples += frames; + } + assert(xtc_realtime_get_status(engine, &status) == XTC_OK); + assert(status.last_result == XTC_OK && status.processed_blocks == blocks); + assert(status.active_revision == status.submitted_revision); + xtc_realtime_collect(engine); + xtc_realtime_destroy(engine); + xtc_graph_destroy(graph); + printf("Virtual playback: %lu blocks, %.3f audio hours, %lu quiescent seek/rebuild cycles passed.\n", + blocks, (double)samples / 48000 / 3600, iterations); + return 0; +} diff --git a/tests/tools/test_benchmark.py b/tests/tools/test_benchmark.py new file mode 100644 index 0000000..b43e190 --- /dev/null +++ b/tests/tools/test_benchmark.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""Verify benchmark gates without depending on noisy real-machine timings.""" +import json +from pathlib import Path +import subprocess +import sys +import tempfile + +with tempfile.TemporaryDirectory(prefix='xtc-benchmark-') as directory: + root = Path(directory) + binary = root / 'measurement' + binary.write_text('''#!/usr/bin/env python3 +import json, sys +nodes, channels, rate, frames, blocks = map(int, sys.argv[1:]) +print(json.dumps(dict(nodes=nodes, channels=channels, rate=rate, frames=frames, + blocks=blocks, mean_us=10, p99_us=12, deadline_misses=0, + process_cpu_ms=1, peak_rss_kib=1024))) +''') + binary.chmod(0o700) + report, baseline = root / 'report.json', root / 'baseline.json' + command = [sys.executable, 'tools/benchmark.py', '--binary', str(binary), + '--output', str(report)] + + def run(arguments, success, message=None): + result = subprocess.run(arguments, capture_output=True, text=True, timeout=30) + assert (result.returncode == 0) == success, result.stdout + result.stderr + if message: + assert message in result.stdout + result.stderr + + run(command, True) + original = json.loads(report.read_text()) + baseline.write_text(json.dumps(original)) + compare = command + ['--baseline', str(baseline)] + run(compare, True) + changed = json.loads(json.dumps(original)) + changed['cases'][0]['mean_us'] = 1 + baseline.write_text(json.dumps(changed)) + run(compare, False, 'exceeds') + changed = json.loads(json.dumps(original)) + changed['cases'].pop() + baseline.write_text(json.dumps(changed)) + run(compare, False, 'missing baseline case') + changed = json.loads(json.dumps(original)) + changed['context']['flags'] = 'incompatible' + baseline.write_text(json.dumps(changed)) + run(compare, False, 'mismatch') + changed = json.loads(json.dumps(original)) + changed['context']['workload'] = 'project' + baseline.write_text(json.dumps(changed)) + run(compare, False, 'workload mismatch') +print('Benchmark matching, regression, missing-case and context gates passed.') diff --git a/tools/benchmark.py b/tools/benchmark.py index 25bbad3..e825cad 100644 --- a/tools/benchmark.py +++ b/tools/benchmark.py @@ -11,6 +11,7 @@ import sys def main(): p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--workload', choices=('graph', 'project'), default='graph') p.add_argument('--binary', type=Path, required=True) p.add_argument('--output', type=Path, required=True) p.add_argument('--baseline', type=Path) @@ -28,14 +29,21 @@ def main(): context = dict(cpu=cpu, architecture=platform.machine(), compiler=compiler, flags=a.flags, kernel=platform.release(), scheduler='ordinary process; no realtime scheduling') cases = [] - for nodes in (2, 16, 64, 256): - for channels in (1, 2): - for rate, frames in ((44100, 31), (48000, 128), (96000, 1024)): - result = subprocess.check_output([str(a.binary.resolve()), *map(str, (nodes, channels, rate, frames, a.blocks))], text=True) - case = json.loads(result) - cases.append(case) - print(f'{nodes:3} nodes {channels}ch {rate}/{frames}: compile {case["compile_ms"]:.3f} ms, ' - f'mean {case["mean_us"]:.3f} us, p99 {case["p99_us"]:.3f} us, load {case["load"]:.3f}') + if a.workload == 'graph': + matrix = [(nodes, channels, rate, frames, a.blocks) + for nodes in (2, 16, 64, 256) for channels in (1, 2) + for rate, frames in ((44100, 31), (48000, 128), (96000, 1024))] + else: + matrix = [(a.blocks, tracks, frames) for tracks in (1, 16, 64) + for frames in (31, 128, 1024)] + for arguments in matrix: + result = subprocess.check_output([str(a.binary.resolve()), *map(str, arguments)], text=True, timeout=120) + case = json.loads(result) + cases.append(case) + print(f'{a.workload} {arguments}: mean {case["mean_us"]:.3f} us, ' + f'p99 {case["p99_us"]:.3f} us, misses {case["deadline_misses"]}, ' + f'CPU {case["process_cpu_ms"]:.3f} ms, peak RSS {case["peak_rss_kib"]} KiB') + context['workload'] = a.workload report = dict(schema=1, context=context, cases=cases) a.output.parent.mkdir(parents=True, exist_ok=True) a.output.write_text(json.dumps(report, indent=2) + '\n') @@ -43,7 +51,10 @@ def main(): baseline = json.loads(a.baseline.read_text()) if baseline.get('schema') != 1 or any(baseline['context'][k] != context[k] for k in ('cpu', 'architecture', 'compiler', 'flags')): sys.exit('Baseline host/compiler/flags mismatch; record a baseline for this configuration.') - key = lambda c: (c['nodes'], c['channels'], c['rate'], c['frames']) + if baseline['context'].get('workload', 'graph') != a.workload: + sys.exit('Baseline workload mismatch.') + fields = ('nodes', 'channels', 'rate', 'frames') if a.workload == 'graph' else ('tracks', 'notes', 'points', 'rate', 'frames') + key = lambda c: tuple(c[field] for field in fields) old = {key(c): c for c in baseline['cases']} failed = [] for c in cases: diff --git a/tools/stress.py b/tools/stress.py index 7d5ad5a..b2a8d4e 100644 --- a/tools/stress.py +++ b/tools/stress.py @@ -10,13 +10,26 @@ def main(): p = argparse.ArgumentParser(description=__doc__) p.add_argument('--build-dir', type=Path, required=True) p.add_argument('--gui', action='store_true') + p.add_argument('--plugin', type=Path, help='optional original test LADSPA fixture') + p.add_argument('--rounds', type=int, default=1) a = p.parse_args() + if not 1 <= a.rounds <= 1000: + p.error('rounds must be 1..1000') workers = [] try: for _ in range(2): workers.append(subprocess.Popen([sys.executable, '-c', 'x=1\nwhile True: x=(x*1664525+1013904223)&0xffffffff'])) - for name in ('test_realtime', 'test_reliability', 'test_scale'): - subprocess.run([str((a.build_dir / name).resolve())], check=True, timeout=120) + names = ['test_realtime', 'test_reliability', 'test_scale', 'test_portability', + 'test_project_diagnostics', 'test_soak'] + if (a.build_dir / 'test_backend').exists(): + names.append('test_backend') + for iteration in range(a.rounds): + for name in names: + subprocess.run([str((a.build_dir / name).resolve())], check=True, timeout=120) + if a.plugin: + subprocess.run([str((a.build_dir / 'test_plugins').resolve()), + str(a.plugin.resolve()), 'fixture'], check=True, timeout=120) + print(f'Stress round {iteration + 1}/{a.rounds} passed.', flush=True) if a.gui: # The GUI suite pumps timers and forces redraw during engine/edit tests. subprocess.run(['xvfb-run', '-a', '-s', '-screen 0 1280x800x24',