Phase 9: Add native processors and reproducible Linux release tooling

tests/integration/README.md

@@ -156,3 +156,29 @@ ASan/UBSan core/GUI suite passed without compiler warnings. GUI leak checking
 retains only the existing fontconfig suppression. ThreadSanitizer also passed
 the recorder concurrency and realtime publication tests. Physical devices and
 long-running stereo performance remain unverified.
+
+## Phase 9 release preparation
+
+```sh
+make all test gui debug test-gui test-gui-jack test-jack test-jack-audio test-jack-midi
+make test-install test-recovery test-performance
+make test-release
+```
+
+Core test now includes native ABI/lifecycle/retirement and the native-gain example.
+Optional release tests use Python 3.12+ standard library, Git for indexed source
+exports, GNU/LLVM deterministic ar and temporary directories. No live prefix is
+installed. test-install checks core/GUI files, a prefix containing spaces,
+headless CLI, pkg-config C/C++ consumers, exact/idempotent uninstall and preservation
+of unrelated files. test-release compares two clean exported optimized builds and
+repacked archives byte-for-byte. test-recovery kills private child writers and
+forces a child-local write limit, checking complete finalized destinations and
+explicit orphan temporaries. Performance timings are reported, not used as a
+universal realtime guarantee. See RELEASING.md, RECOVERY.md and PERFORMANCE.md.
+
+Phase 9 verification passed a fresh warning-enabled core/debug/GUI build, all
+software JACK targets, staged install/uninstall, recovery and performance tests.
+No-JACK core/GUI/install checks and Clang ASan/UBSan core/GUI checks also passed
+(the GUI retains only its existing fontconfig leak suppression). TSan recorder
+and realtime-handoff checks passed. Two independent optimized source exports
+and their repacked source archives were byte-identical on this toolchain.

tests/integration/test_install.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+"""Exercise core/GUI install and exact uninstall solely below a temporary DESTDIR."""
+import argparse
+import os
+from pathlib import Path
+import shlex
+import shutil
+import subprocess
+import tempfile
+
+
+def run(args, **kwargs):
+    return subprocess.run(args, check=True, **kwargs)
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--build-dir', type=Path, required=True)
+    parser.add_argument('--gui', action='store_true')
+    args = parser.parse_args()
+    source = Path.cwd()
+    with tempfile.TemporaryDirectory(prefix='xtc-install-') as directory:
+        stage = Path(directory)
+        prefix = '/opt/xtc test'
+        root = stage / prefix.lstrip('/')
+        sentinel = root / 'lib/keep-unrelated.txt'
+        sentinel.parent.mkdir(parents=True)
+        sentinel.write_text('keep\n')
+        make = shlex.split(os.environ.get('MAKE', 'make'))
+        # MAKEFLAGS jobserver descriptors do not cross Python subprocesses.
+        env = dict(os.environ, MAKEFLAGS='', MFLAGS='')
+        variables = [f'BUILD_DIR={args.build_dir}', f'PREFIX={prefix}', f'DESTDIR={stage}']
+        run(make + ['install'] + variables, env=env)
+        assert not (root / 'bin/xtc').exists()
+        assert (root / 'share/licenses/xtc/LICENSE').read_bytes() == (source / 'LICENSE').read_bytes()
+        assert (root / 'share/doc/xtc/PROJECT_FORMAT.md').is_file()
+        assert os.access(root / 'bin/xtc-check', os.X_OK)
+        run([str(root / 'bin/xtc-check')])
+        version = subprocess.check_output([str(root / 'bin/xtc-check'), '--version'], text=True)
+        assert version.startswith('xtc ')
+        valid = stage / 'valid.xtc'
+        valid.write_text('XTC_PROJECT 4\nAPP 0 1 0\nTEMPO 120000\nIDS 0 0\nGRAPH 1 0 0 0\nTRACKS 0\nCLIPS 0\nNOTES 0 0\nAUTOMATION 0\nEND\n')
+        run([str(root / 'bin/xtc-check'), '--validate', str(valid)])
+        valid.write_text('damaged\n')
+        assert subprocess.run([str(root / 'bin/xtc-check'), '--validate', str(valid)],
+                              stdout=subprocess.PIPE, stderr=subprocess.PIPE).returncode != 0
+        pkg_env = dict(env, PKG_CONFIG_PATH=str(root / 'lib/pkgconfig') + ':' + env.get('PKG_CONFIG_PATH', ''),
+                       PKG_CONFIG_SYSROOT_DIR=str(stage))
+        pkg = shlex.split(os.environ.get('PKG_CONFIG', 'pkg-config'))
+        # A no-JACK test can be invoked with PKG_CONFIG=false; package consumer
+        # discovery itself still uses the installed pkg-config tool.
+        if pkg == ['false']:
+            pkg = ['pkg-config']
+        flags = shlex.split(subprocess.check_output(pkg + ['--cflags', '--libs', '--static', 'xtc'], env=pkg_env, text=True))
+        includes = ''.join(f'#include "xtc/{p.name}"\n' for p in sorted((source / 'include/xtc').glob('*.h')))
+        consumer = stage / 'consumer.c'
+        consumer.write_text(includes + '\nint main(void) { xtc_project *p = 0; xtc_result r = xtc_project_create(&p); xtc_project_destroy(p); return r != XTC_OK; }\n')
+        for compiler, standard in [('CC', 'c99'), ('CXX', 'c++17')]:
+            output = stage / ('consumer-' + compiler)
+            command = shlex.split(env.get(compiler, 'cc' if compiler == 'CC' else 'c++'))
+            run(command + [f'-std={standard}', '-Wall', '-Wextra', '-Werror', str(consumer)] + flags + ['-o', str(output)])
+            run([str(output)])
+        if args.gui:
+            run(make + ['install-gui'] + variables, env=env)
+            gui_env = dict(env, DISPLAY='', WAYLAND_DISPLAY='')
+            assert subprocess.check_output([str(root / 'bin/xtc'), '--version'], env=gui_env, text=True) == version
+            run([str(root / 'bin/xtc'), '--help'], env=gui_env)
+            if shutil.which('desktop-file-validate'):
+                run(['desktop-file-validate', str(root / 'share/applications/xtc.desktop')])
+        run(make + ['uninstall'] + variables, env=env)
+        run(make + ['uninstall'] + variables, env=env)
+        remaining = sorted(p.relative_to(root).as_posix() for p in root.rglob('*') if p.is_file() or p.is_symlink())
+        assert remaining == ['lib/keep-unrelated.txt'], remaining
+        assert sentinel.read_text() == 'keep\n'
+    print('Staged install/uninstall, C/C++ pkg-config consumers and headless CLI passed.')
+
+
+if __name__ == '__main__':
+    main()

tests/integration/test_performance.c

@@ -0,0 +1,110 @@
+#define _POSIX_C_SOURCE 200809L
+/* Offline workload timing, not hardware or hard realtime qualification. */
+#include "xtc/xtc_midi.h"
+#include "xtc/xtc_mixer.h"
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <time.h>
+#define REQUIRE(c)                                                                                 \
+    do {                                                                                           \
+        if (!(c)) {                                                                                \
+            fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #c);                           \
+            goto cleanup;                                                                          \
+        }                                                                                          \
+    } while (0)
+static double now(void)
+{
+    struct timespec t;
+    clock_gettime(CLOCK_MONOTONIC, &t);
+    return (double)t.tv_sec + (double)t.tv_nsec / 1e9;
+}
+static int compare(const void *left, const void *right)
+{
+    double a = *(const double *)left, b = *(const double *)right;
+    return a < b ? -1 : a > b;
+}
+int main(int argc, char **argv)
+{
+    xtc_project *project = NULL;
+    xtc_graph *graph = NULL;
+    xtc_plan *plan = NULL;
+    xtc_midi_plan *midi = NULL;
+    long blocks = 2000;
+    char *end;
+    size_t i, j, source, previous, node;
+    uint64_t track, clip, note, future;
+    float left[128], right[128], out_left[128], out_right[128];
+    double *times = NULL, total = 0, begin, prepare_ms;
+    int result = EXIT_FAILURE;
+    if (argc > 2)
+        return EXIT_FAILURE;
+    if (argc == 2) {
+        blocks = strtol(argv[1], &end, 10);
+        if (!*argv[1] || *end || blocks < 1 || blocks > 100000)
+            return EXIT_FAILURE;
+    }
+    times = calloc((size_t)blocks, sizeof(*times));
+    REQUIRE(times);
+    begin = now();
+    REQUIRE(xtc_project_create(&project) == XTC_OK);
+    for (i = 0; i < 16; ++i) {
+        xtc_automation_point point;
+        REQUIRE(xtc_project_add_track(project, "Performance", &track) == XTC_OK);
+        REQUIRE(xtc_project_add_midi_clip(project, track, 0, 384000, &clip) == XTC_OK);
+        REQUIRE(xtc_project_set_mix(project, track, 0.1f, i % 2 ? 0.5f : -0.5f, 0, 0) == XTC_OK);
+        for (j = 0; j < 2; ++j)
+            REQUIRE(xtc_project_add_note(project, clip, 0, 384000, 48 + (unsigned int)i, 100, 0,
+                                         &note) == XTC_OK);
+        point = (xtc_automation_point){track, 0, XTC_AUTOMATION_GAIN, 0.1f};
+        REQUIRE(xtc_project_set_automation(project, &point) == XTC_OK);
+        point.tick = 384000;
+        point.value = 0.2f;
+        REQUIRE(xtc_project_set_automation(project, &point) == XTC_OK);
+    }
+    REQUIRE(xtc_project_add_midi_clip(project, track, 1000000, 10000, &future) == XTC_OK);
+    for (i = 32; i < XTC_PROJECT_MAX_NOTES; ++i)
+        REQUIRE(xtc_project_add_note(project, future, (i - 32) * 2, 1, 60, 100, 0, &note) ==
+                XTC_OK);
+    REQUIRE(xtc_graph_create(64, &graph) == XTC_OK &&
+            xtc_graph_add_instrument(graph, &source) == XTC_OK);
+    previous = source;
+    for (i = 0; i < 61; ++i) {
+        REQUIRE(xtc_graph_add_gain(graph, 0.99f, &node) == XTC_OK &&
+                xtc_graph_connect(graph, previous, node) == XTC_OK);
+        previous = node;
+    }
+    REQUIRE(xtc_graph_add_meter(graph, &node) == XTC_OK &&
+            xtc_graph_connect(graph, previous, node) == XTC_OK);
+    previous = node;
+    REQUIRE(xtc_graph_add_output(graph, &node) == XTC_OK &&
+            xtc_graph_connect(graph, previous, node) == XTC_OK);
+    REQUIRE(xtc_graph_compile(graph, 48000, 128, &plan) == XTC_OK &&
+            xtc_midi_plan_create(project, 48000, &midi) == XTC_OK);
+    prepare_ms = (now() - begin) * 1000;
+    for (i = 0; i < (size_t)blocks; ++i) {
+        begin = now();
+        REQUIRE(xtc_midi_plan_render_stereo(midi, i * 128, NULL, 0, left, right, 128) == XTC_OK);
+        REQUIRE(xtc_plan_process_stereo(plan, NULL, NULL, NULL, left, right, out_left, out_right,
+                                        128) == XTC_OK);
+        times[i] = now() - begin;
+        total += times[i];
+        for (j = 0; j < 128; ++j)
+            REQUIRE(isfinite(out_left[j]) && isfinite(out_right[j]));
+    }
+    qsort(times, (size_t)blocks, sizeof(*times), compare);
+    printf("Offline 48kHz/128, 64 nodes, 4096 notes (32 active), 16 automated tracks; %ld blocks\n",
+           blocks);
+    printf("Prepare %.3f ms; block mean %.3f us, p99 %.3f us, max %.3f us; deadline 2666.667 us; "
+           "CPU/audio %.4f\n",
+           prepare_ms, total / (double)blocks * 1e6, times[(size_t)blocks * 99 / 100] * 1e6,
+           times[blocks - 1] * 1e6, total / ((double)blocks * 128 / 48000));
+    result = EXIT_SUCCESS;
+cleanup:
+    free(times);
+    xtc_midi_plan_destroy(midi);
+    xtc_plan_destroy(plan);
+    xtc_graph_destroy(graph);
+    xtc_project_destroy(project);
+    return result;
+}

tests/integration/test_recovery.c

@@ -0,0 +1,153 @@
+#define _POSIX_C_SOURCE 200809L
+#include "xtc/xtc_audio.h"
+#include <assert.h>
+#include <dirent.h>
+#include <errno.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/resource.h>
+#include <sys/wait.h>
+#include <time.h>
+#include <unistd.h>
+static void reap(pid_t child)
+{
+    int status;
+    assert(kill(child, SIGKILL) == 0);
+    assert(waitpid(child, &status, 0) == child && WIFSIGNALED(status) &&
+           WTERMSIG(status) == SIGKILL);
+}
+static size_t read_bytes(const char *path, unsigned char *bytes, size_t size)
+{
+    FILE *f = fopen(path, "rb");
+    size_t n;
+    assert(f);
+    n = fread(bytes, 1, size, f);
+    assert(!ferror(f) && fclose(f) == 0);
+    return n;
+}
+static int temporaries(const char *directory, int remove_files)
+{
+    DIR *dir = opendir(directory);
+    struct dirent *entry;
+    int count = 0;
+    assert(dir);
+    while ((entry = readdir(dir)) != NULL)
+        if (strstr(entry->d_name, ".tmp.")) {
+            char path[512];
+            ++count;
+            if (remove_files) {
+                assert(snprintf(path, sizeof(path), "%s/%s", directory, entry->d_name) > 0);
+                assert(unlink(path) == 0);
+            }
+        }
+    assert(closedir(dir) == 0);
+    return count;
+}
+int main(void)
+{
+    char directory[] = "/tmp/xtc-recovery-XXXXXX", song[256], wave[256];
+    xtc_project *old = NULL, *next = NULL, *loaded = NULL;
+    uint64_t track, clip;
+    size_t i, n, before_size;
+    unsigned char before[4096], after[4096];
+    pid_t child;
+    int status, pipefd[2];
+    char ready;
+    struct timespec delay = {0, 100000};
+    assert(mkdtemp(directory));
+    snprintf(song, sizeof(song), "%s/song.xtc", directory);
+    snprintf(wave, sizeof(wave), "%s/take.wav", directory);
+    assert(xtc_project_create(&old) == XTC_OK && xtc_project_create(&next) == XTC_OK &&
+           xtc_project_create(&loaded) == XTC_OK);
+    assert(xtc_project_add_track(old, "Old", &track) == XTC_OK &&
+           xtc_project_save(old, song) == XTC_OK);
+    before_size = read_bytes(song, before, sizeof(before));
+    assert(xtc_project_add_track(next, "New", &track) == XTC_OK);
+    for (i = 0; i < XTC_PROJECT_MAX_CLIPS; ++i)
+        assert(xtc_project_add_clip(next, track, "Recovery region", i * 960, 960, &clip) == XTC_OK);
+    // Deterministic per-process file-size failure; no server/system setting change.
+    child = fork();
+    assert(child >= 0);
+    if (!child) {
+        struct rlimit limit = {128, 128};
+        signal(SIGXFSZ, SIG_IGN);
+        if (setrlimit(RLIMIT_FSIZE, &limit) != 0)
+            _exit(2);
+        _exit(xtc_project_save(next, song) == XTC_ERR_IO ? 0 : 3);
+    }
+    assert(waitpid(child, &status, 0) == child && WIFEXITED(status) && WEXITSTATUS(status) == 0);
+    assert(read_bytes(song, after, sizeof(after)) == before_size &&
+           memcmp(before, after, before_size) == 0 && temporaries(directory, 0) == 0);
+    // Kill a writer while its temporary is visible. Destination must be complete old/new.
+    for (i = 0; i < 8; ++i) {
+        size_t attempts;
+        xtc_track_info info;
+        assert(xtc_project_save(old, song) == XTC_OK);
+        assert(pipe(pipefd) == 0);
+        child = fork();
+        assert(child >= 0);
+        if (!child) {
+            close(pipefd[0]);
+            if (write(pipefd[1], "R", 1) != 1)
+                _exit(4);
+            close(pipefd[1]);
+            for (;;)
+                if (xtc_project_save(next, song) != XTC_OK)
+                    _exit(5);
+        }
+        close(pipefd[1]);
+        assert(read(pipefd[0], &ready, 1) == 1);
+        close(pipefd[0]);
+        for (attempts = 0; attempts < 10000 && !temporaries(directory, 0); ++attempts)
+            nanosleep(&delay, NULL);
+        reap(child);
+        assert(attempts < 10000);
+        assert(xtc_project_load(loaded, song) == XTC_OK &&
+               xtc_project_get_track(loaded, 0, &info) == XTC_OK);
+        assert(strcmp(info.name, "Old") == 0 || strcmp(info.name, "New") == 0);
+        temporaries(directory, 1);
+    }
+    // A killed recording leaves the previously finalized destination untouched.
+    {
+        xtc_recorder *recorder = NULL;
+        float samples[128];
+        for (i = 0; i < 128; ++i)
+            samples[i] = 0.25f;
+        assert(xtc_recorder_create(wave, 48000, 256, &recorder) == XTC_OK);
+        xtc_recorder_push(recorder, samples, 128);
+        assert(xtc_recorder_finish(recorder) == XTC_OK);
+        xtc_recorder_destroy(recorder);
+        before_size = read_bytes(wave, before, sizeof(before));
+        assert(pipe(pipefd) == 0);
+        child = fork();
+        assert(child >= 0);
+        if (!child) {
+            close(pipefd[0]);
+            if (xtc_recorder_create(wave, 48000, 256, &recorder) != XTC_OK)
+                _exit(6);
+            xtc_recorder_push(recorder, samples, 128);
+            if (xtc_recorder_drain(recorder) != XTC_OK)
+                _exit(7);
+            if (write(pipefd[1], "R", 1) != 1)
+                _exit(8);
+            for (;;)
+                pause();
+        }
+        close(pipefd[1]);
+        assert(read(pipefd[0], &ready, 1) == 1);
+        close(pipefd[0]);
+        reap(child);
+        n = read_bytes(wave, after, sizeof(after));
+        assert(n == before_size && memcmp(before, after, n) == 0);
+        assert(temporaries(directory, 1) > 0);
+    }
+    xtc_project_destroy(old);
+    xtc_project_destroy(next);
+    xtc_project_destroy(loaded);
+    assert(unlink(song) == 0 && unlink(wave) == 0 && rmdir(directory) == 0);
+    puts("Save write failure, 8 killed saves and killed recording preserve complete destinations; "
+         "orphan temporaries remain explicit.");
+    return 0;
+}

tests/integration/test_release.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+"""Rebuild source exports in different paths and compare release artifacts."""
+import argparse
+import hashlib
+import os
+from pathlib import Path
+import shlex
+import subprocess
+import tarfile
+import tempfile
+
+
+def digest(path):
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('archive', type=Path)
+    parser.add_argument('--gui', action='store_true')
+    args = parser.parse_args()
+    archive = args.archive.resolve()
+    with tempfile.TemporaryDirectory(prefix='xtc-release-') as directory:
+        root = Path(directory)
+        with tarfile.open(archive) as source:
+            names = [m.name for m in source.getmembers()]
+            assert len(set(names)) == len(names)
+            assert all(m.isfile() for m in source.getmembers())
+            top = names[0].split('/')[0]
+            epoch = source.getmembers()[0].mtime
+            assert all(n.startswith(top + '/') and '..' not in Path(n).parts for n in names)
+            assert all('__pycache__' not in Path(n).parts and '.git' not in Path(n).parts for n in names)
+            for name in ('first', 'different-path'):
+                source.extractall(root / name, filter='data')
+        env = dict(os.environ, SOURCE_DATE_EPOCH=str(epoch), MAKEFLAGS='', MFLAGS='')
+        builds = []
+        for name in ('first', 'different-path'):
+            project = root / name / top
+            log = root / (name + '.log')
+            make = shlex.split(env.get('MAKE', 'make'))
+            with log.open('w') as output:
+                try:
+                    subprocess.run(make + ['release'], cwd=project, env=env, stdout=output,
+                                   stderr=subprocess.STDOUT, check=True)
+                except subprocess.CalledProcessError:
+                    print(log.read_text()[-12000:])
+                    raise
+            # Exported source can package itself without a .git directory.
+            repacked = root / (name + '.tar.gz')
+            subprocess.run([env.get('PYTHON', 'python3'), 'tools/dist.py', '--output', str(repacked)],
+                           cwd=project, env=env, check=True)
+            assert digest(repacked) == digest(archive), 'Source archive is not reproducible'
+            release = project / 'build/release'
+            files = ['libxtc.a', 'xtc-check'] + (['xtc'] if args.gui else [])
+            builds.append({file: digest(release / file) for file in files})
+        assert builds[0] == builds[1], (builds[0], builds[1])
+        for file, sha in builds[0].items():
+            print(f'Reproducible {file}: {sha}')
+    print('Two clean source-export release builds and archives are byte-identical.')
+
+
+if __name__ == '__main__':
+    main()