From de403b37d7a0acaf31d0b68958977ccc33747860 Mon Sep 17 00:00:00 2001 From: hanez Date: Tue, 27 Oct 2026 01:21:04 +0100 Subject: [PATCH] Phase 9: Add native processors and reproducible Linux release tooling --- .gitignore | 4 + Makefile | 69 +++++- README.md | 5 + docs/ARCHITECTURE.md | 11 + docs/AUDIO_ENGINE.md | 10 + docs/BUILD.md | 26 ++ docs/LICENSES.md | 39 +++ docs/LINUX.md | 58 +++++ docs/NATIVE_API.md | 83 +++++++ docs/PERFORMANCE.md | 35 +++ docs/PLUGIN_HOSTING.md | 24 ++ docs/PROJECT_FORMAT.md | 9 + docs/RECOVERY.md | 48 ++++ docs/RELEASING.md | 56 +++++ docs/ROADMAP.md | 11 + ...8-native-processors-and-source-releases.md | 19 ++ examples/README.md | 5 +- examples/native_gain.c | 64 +++++ include/xtc/xtc.h | 5 +- include/xtc/xtc_engine.h | 3 +- include/xtc/xtc_native.h | 56 +++++ include/xtc/xtc_version.h | 3 + packaging/xtc.desktop | 10 + packaging/xtc.pc.in | 11 + src/engine/offline_engine.c | 34 ++- src/graph/graph.c | 22 +- src/graph/graph_internal.h | 3 +- src/gui/graph_widget.cpp | 1 + src/gui/main.cpp | 14 ++ src/gui/main_window.cpp | 2 +- src/main.c | 20 +- src/native/README.md | 5 + src/native/native.c | 105 ++++++++ src/persistence/project_file.c | 4 +- tests/integration/README.md | 26 ++ tests/integration/test_install.py | 79 ++++++ tests/integration/test_performance.c | 110 +++++++++ tests/integration/test_recovery.c | 153 ++++++++++++ tests/integration/test_release.py | 63 +++++ tests/unit/test_native.c | 229 ++++++++++++++++++ tools/dist.py | 81 +++++++ tools/pkgconfig.py | 16 ++ tools/uninstall.py | 44 ++++ 43 files changed, 1656 insertions(+), 19 deletions(-) create mode 100644 docs/LICENSES.md create mode 100644 docs/LINUX.md create mode 100644 docs/NATIVE_API.md create mode 100644 docs/PERFORMANCE.md create mode 100644 docs/PLUGIN_HOSTING.md create mode 100644 docs/RECOVERY.md create mode 100644 docs/RELEASING.md create mode 100644 docs/decisions/0008-native-processors-and-source-releases.md create mode 100644 examples/native_gain.c create mode 100644 include/xtc/xtc_native.h create mode 100644 packaging/xtc.desktop create mode 100644 packaging/xtc.pc.in create mode 100644 src/native/README.md create mode 100644 src/native/native.c create mode 100644 tests/integration/test_install.py create mode 100644 tests/integration/test_performance.c create mode 100644 tests/integration/test_recovery.c create mode 100644 tests/integration/test_release.py create mode 100644 tests/unit/test_native.c create mode 100644 tools/dist.py create mode 100644 tools/pkgconfig.py create mode 100644 tools/uninstall.py diff --git a/.gitignore b/.gitignore index 009e903..75999c8 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,7 @@ # Local test outputs *.wav *.mid + +# Python packaging/test caches +__pycache__/ +*.py[cod] diff --git a/Makefile b/Makefile index 22b604b..7918c36 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,8 @@ PKG_CONFIG ?= pkg-config FLTK_CONFIG ?= fltk-config BUILD_DIR ?= build PREFIX ?= /usr/local +PYTHON ?= python3 +VERSION := $(shell sed -n 's/^\#define XTC_VERSION_STRING "\([^"]*\)"/\1/p' include/xtc/xtc_version.h) CFLAGS ?= -O2 -g CFLAGS += -std=c99 -Wall -Wextra -Wpedantic -Wconversion -Wshadow -Iinclude @@ -13,7 +15,7 @@ CXXFLAGS ?= -O2 -g CXXFLAGS += -std=c++17 -Wall -Wextra -Wpedantic -Iinclude LDFLAGS ?= -CORE_SOURCES := src/mixer/model.c src/mixer/render.c src/project/history.c src/midi/model.c src/midi/instrument.c src/audio/wave.c src/audio/arrangement.c src/audio/recorder.c src/core/xtc.c src/project/project.c src/persistence/project_file.c src/graph/graph.c src/dsp/gain.c src/engine/offline_engine.c src/engine/realtime.c src/backend/jack_backend.c +CORE_SOURCES := src/native/native.c src/mixer/model.c src/mixer/render.c src/project/history.c src/midi/model.c src/midi/instrument.c src/audio/wave.c src/audio/arrangement.c src/audio/recorder.c src/core/xtc.c src/project/project.c src/persistence/project_file.c src/graph/graph.c src/dsp/gain.c src/engine/offline_engine.c src/engine/realtime.c src/backend/jack_backend.c JACK_AVAILABLE := $(shell $(PKG_CONFIG) --exists jack 2>/dev/null && echo yes) ifeq ($(JACK_AVAILABLE),yes) JACK_CFLAGS := $(shell $(PKG_CONFIG) --cflags jack) @@ -42,7 +44,7 @@ $(BUILD_DIR)/xtc-check: $(BUILD_DIR)/src/main.o $(BUILD_DIR)/libxtc.a $(BUILD_DIR)/libxtc.a: $(CORE_OBJECTS) @mkdir -p $(@D) - $(AR) rcs $@ $^ + $(AR) rcsD $@ $^ $(BUILD_DIR)/%.o: %.c @mkdir -p $(@D) @@ -83,8 +85,16 @@ $(BUILD_DIR)/test_midi: $(BUILD_DIR)/tests/unit/test_midi.o $(BUILD_DIR)/libxtc. $(BUILD_DIR)/test_mixer: $(BUILD_DIR)/tests/unit/test_mixer.o $(BUILD_DIR)/libxtc.a $(CC) $(LDFLAGS) -o $@ $^ -lm -test: $(BUILD_DIR)/test_mixer $(BUILD_DIR)/test_midi $(BUILD_DIR)/test_audio $(BUILD_DIR)/test_project $(BUILD_DIR)/test_graph_edits $(BACKEND_TEST) $(BUILD_DIR)/test_realtime $(BUILD_DIR)/test_core $(BUILD_DIR)/test_offline $(BUILD_DIR)/xtc-check +$(BUILD_DIR)/test_native: $(BUILD_DIR)/tests/unit/test_native.o $(BUILD_DIR)/libxtc.a + $(CC) $(LDFLAGS) -o $@ $^ -lm + +$(BUILD_DIR)/native-gain: $(BUILD_DIR)/examples/native_gain.o $(BUILD_DIR)/libxtc.a + $(CC) $(LDFLAGS) -o $@ $^ -lm + +test: $(BUILD_DIR)/test_native $(BUILD_DIR)/native-gain $(BUILD_DIR)/test_mixer $(BUILD_DIR)/test_midi $(BUILD_DIR)/test_audio $(BUILD_DIR)/test_project $(BUILD_DIR)/test_graph_edits $(BACKEND_TEST) $(BUILD_DIR)/test_realtime $(BUILD_DIR)/test_core $(BUILD_DIR)/test_offline $(BUILD_DIR)/xtc-check $(if $(BACKEND_TEST),./$(BACKEND_TEST),@echo "JACK fault tests skipped: JACK development files unavailable") + ./$(BUILD_DIR)/test_native + ./$(BUILD_DIR)/native-gain ./$(BUILD_DIR)/test_mixer ./$(BUILD_DIR)/test_midi ./$(BUILD_DIR)/test_audio @@ -158,10 +168,42 @@ gui: @exit 1 endif -install: all - install -d "$(DESTDIR)$(PREFIX)/include/xtc" "$(DESTDIR)$(PREFIX)/lib" +# install retains the core-only behavior; install-gui adds the application. +.PHONY: install uninstall install-gui dist release test-install test-release test-performance +$(BUILD_DIR)/xtc.pc: packaging/xtc.pc.in tools/pkgconfig.py tools/dist.py include/xtc/xtc_version.h FORCE + $(PYTHON) tools/pkgconfig.py --prefix "$(PREFIX)" --jack "$(if $(JACK_AVAILABLE),jack)" --output "$@" +.PHONY: FORCE +FORCE: + +install: all $(BUILD_DIR)/xtc.pc + install -d "$(DESTDIR)$(PREFIX)/include/xtc" "$(DESTDIR)$(PREFIX)/lib/pkgconfig" "$(DESTDIR)$(PREFIX)/bin" "$(DESTDIR)$(PREFIX)/share/doc/xtc/decisions" "$(DESTDIR)$(PREFIX)/share/licenses/xtc" install -m 0644 include/xtc/*.h "$(DESTDIR)$(PREFIX)/include/xtc/" install -m 0644 "$(BUILD_DIR)/libxtc.a" "$(DESTDIR)$(PREFIX)/lib/" + install -m 0644 "$(BUILD_DIR)/xtc.pc" "$(DESTDIR)$(PREFIX)/lib/pkgconfig/" + install -m 0755 "$(BUILD_DIR)/xtc-check" "$(DESTDIR)$(PREFIX)/bin/" + install -m 0644 README.md docs/*.md "$(DESTDIR)$(PREFIX)/share/doc/xtc/" + install -m 0644 docs/decisions/*.md "$(DESTDIR)$(PREFIX)/share/doc/xtc/decisions/" + install -m 0644 LICENSE "$(DESTDIR)$(PREFIX)/share/licenses/xtc/" + +install-gui: install gui + install -d "$(DESTDIR)$(PREFIX)/share/applications" + install -m 0755 "$(BUILD_DIR)/xtc" "$(DESTDIR)$(PREFIX)/bin/" + install -m 0644 packaging/xtc.desktop "$(DESTDIR)$(PREFIX)/share/applications/" + +uninstall: + $(PYTHON) tools/uninstall.py --root "$(DESTDIR)$(PREFIX)" + +dist: + $(PYTHON) tools/dist.py --output "$(BUILD_DIR)/xtc-$(VERSION).tar.gz" + +release: + $(MAKE) BUILD_DIR="$(BUILD_DIR)/release" CFLAGS="-O2 -g0 -std=c99 -Wall -Wextra -Wpedantic -Wconversion -Wshadow -Iinclude -ffile-prefix-map=$(CURDIR)=." CXXFLAGS="-O2 -g0 -std=c++17 -Wall -Wextra -Wpedantic -Iinclude -ffile-prefix-map=$(CURDIR)=." FLTK_CXXFLAGS="$(filter -I% -D% -pthread,$(FLTK_CXXFLAGS))" all test $(if $(GUI_AVAILABLE),gui) + +test-install: all $(if $(GUI_AVAILABLE),gui) + $(PYTHON) tests/integration/test_install.py --build-dir "$(abspath $(BUILD_DIR))" $(if $(GUI_AVAILABLE),--gui) + +test-release: dist + $(PYTHON) tests/integration/test_release.py "$(BUILD_DIR)/xtc-$(VERSION).tar.gz" $(if $(GUI_AVAILABLE),--gui) clean: rm -rf "$(BUILD_DIR)" @@ -182,3 +224,20 @@ clean: -include $(BUILD_DIR)/src/gui/mixer_widget.d -include $(BUILD_DIR)/tests/unit/test_mixer.d + +-include $(BUILD_DIR)/tests/unit/test_native.d $(BUILD_DIR)/examples/native_gain.d + +$(BUILD_DIR)/test_performance: $(BUILD_DIR)/tests/integration/test_performance.o $(BUILD_DIR)/libxtc.a + $(CC) $(LDFLAGS) -o $@ $^ -lm + +$(BUILD_DIR)/test_recovery: $(BUILD_DIR)/tests/integration/test_recovery.o $(BUILD_DIR)/libxtc.a + $(CC) $(LDFLAGS) -o $@ $^ -lm + +.PHONY: test-recovery +test-performance: $(BUILD_DIR)/test_performance + ./$(BUILD_DIR)/test_performance + +test-recovery: $(BUILD_DIR)/test_recovery + ./$(BUILD_DIR)/test_recovery + +-include $(BUILD_DIR)/tests/integration/test_performance.d $(BUILD_DIR)/tests/integration/test_recovery.d diff --git a/README.md b/README.md index 0dc0cab..3e5bd4a 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ This repository is a starter scaffold, not a production DAW. It currently provid - MIDI clips, a sine instrument, JACK MIDI input and a basic piano roll; - stereo track gain/pan/mute/solo, output meters and gain/pan automation lanes; - bounded undo/redo with keyboard shortcuts; +- native C processors, deterministic source packaging and staged install/uninstall; - architecture, format, GUI, roadmap, and decision documents. The JACK backend has passed software-loopback testing; physical-device playback @@ -81,6 +82,10 @@ connected automatically. In Modular Graph, drag output to input to connect; select a node or wire and press Delete to remove it. Invalid edits leave the active graph intact; queued changes are labeled until the engine adopts them. See [GUI documentation](docs/GUI.md). +For Linux installation, routing and CLI options see [LINUX.md](docs/LINUX.md); +for native effects/instruments see [NATIVE_API.md](docs/NATIVE_API.md). +`make release`, `make dist`, `make test-install` and `make test-release` provide +release preparation; see [RELEASING.md](docs/RELEASING.md) for exact guarantees. With JACK development files and an existing JACK-compatible server, run the isolated software-loopback test with `make test-jack` and the record/reopen/play acceptance test with `make test-jack-audio` and MIDI acceptance with diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 2b76c9a..e95beaf 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -130,3 +130,14 @@ stop transport before subsequent preparation. `src/project/history.c` provides cursor only after successful GUI/backend installation. No samples or FLTK state are stored in history. `mixer_widget.cpp` is only controls/drawing/gestures. See MIXER.md and PROJECT_FORMAT.md for ownership, limits and v4 migration. + +## Phase 9 native lifecycle and distribution + +`xtc_native.h` defines copied versioned C descriptors and bounded block contexts. +Native effect/source nodes prepare independent instances on control-thread graph +compilation and reclaim them only on compile failure, quiescent destruction or +retirement collection. Event instruments can render into the public graph source +buffer. No descriptor/state pointers enter project files; unsupported native +saves preserve files. See NATIVE_API.md and decision 0008. Release tooling lives +outside the runtime, with source manifests, deterministic exports and staged +installation/consumer checks. See RELEASING.md for reproducibility scope. diff --git a/docs/AUDIO_ENGINE.md b/docs/AUDIO_ENGINE.md index cfb558b..9725436 100644 --- a/docs/AUDIO_ENGINE.md +++ b/docs/AUDIO_ENGINE.md @@ -263,3 +263,13 @@ and publishes independently atomic output peak snapshots. Callback work remains bounded and allocation/I/O/lock free. Meter UI reads no active plan memory. See MIXER.md for exact pan law, automation interpolation, event/source exceptions, memory/history bounds and verification. Input/recording remain mono. + +## Phase 9 native processing + +Graph-native effects/sources use copied versioned descriptors, two independently +prepared mono instances and the existing plan ownership/retirement protocol. +Reset/process callbacks must be realtime-safe; prepare/destroy never run in +processing. Callback errors silence the block and reset the failing processor, +without rolling back already advanced DSP. Native event instruments can render +explicitly into graph instrument buffers. See NATIVE_API.md for exact lifecycle +and serialization limits and PERFORMANCE.md for measured workload and budgets. diff --git a/docs/BUILD.md b/docs/BUILD.md index 24e817c..4a58312 100644 --- a/docs/BUILD.md +++ b/docs/BUILD.md @@ -196,3 +196,29 @@ ASan/UBSan core/GUI suite passed without compiler warnings. GUI leak checking retains only the existing fontconfig suppression. ThreadSanitizer also passed the recorder concurrency and realtime publication tests. Physical devices and long-running stereo performance remain unverified. + +## Phase 9 release preparation + +```sh +make all test gui debug test-gui test-gui-jack test-jack test-jack-audio test-jack-midi +make test-install test-recovery test-performance +make test-release +``` + +Core test now includes native ABI/lifecycle/retirement and the native-gain example. +Optional release tests use Python 3.12+ standard library, Git for indexed source +exports, GNU/LLVM deterministic ar and temporary directories. No live prefix is +installed. test-install checks core/GUI files, a prefix containing spaces, +headless CLI, pkg-config C/C++ consumers, exact/idempotent uninstall and preservation +of unrelated files. test-release compares two clean exported optimized builds and +repacked archives byte-for-byte. test-recovery kills private child writers and +forces a child-local write limit, checking complete finalized destinations and +explicit orphan temporaries. Performance timings are reported, not used as a +universal realtime guarantee. See RELEASING.md, RECOVERY.md and PERFORMANCE.md. + +Phase 9 verification passed a fresh warning-enabled core/debug/GUI build, all +software JACK targets, staged install/uninstall, recovery and performance tests. +No-JACK core/GUI/install checks and Clang ASan/UBSan core/GUI checks also passed +(the GUI retains only its existing fontconfig leak suppression). TSan recorder +and realtime-handoff checks passed. Two independent optimized source exports +and their repacked source archives were byte-identical on this toolchain. diff --git a/docs/LICENSES.md b/docs/LICENSES.md new file mode 100644 index 0000000..c22a7f2 --- /dev/null +++ b/docs/LICENSES.md @@ -0,0 +1,39 @@ +# Licensing and dependency review + +The repository's original code, example and documentation are under the MIT +license in LICENSE (copyright Johannes Findeisen). No energyXT source, artwork, +external plugin SDK, vendored DSP or binary asset is included. The source archive +contains repository files and an export manifest; it does not bundle system +libraries. Installation includes LICENSE and these documents. + +The Phase 9 host review inspected installed package metadata and local license +files. Package-level license sets can include utilities/docs that differ from the +specific library being linked; retain the relevant upstream notices when +redistributing dependencies. The default install uses system shared libraries. + +| Component | Inspected license / role | +| --- | --- | +| FLTK 1.4.5 | LGPL-2.0-only with FLTK exception; optional GUI. `/usr/share/licenses/fltk/COPYING` contains the linking/subclass exceptions. No local modifications. | +| PipeWire JACK client library | Installed client COPYING is MIT (Wim Taymans); package metadata also lists GPL/LGPL components. No PipeWire binaries are bundled. | +| Traditional JACK client | Alternative backend library commonly LGPL; inspect the actual distribution package instead of assuming PipeWire's license. | +| glibc/libm | LGPL-2.1-or-later library code; package also contains GPL components. System runtime. | +| libstdc++/libgcc | GPL-3.0-or-later with GCC Runtime Library Exception 3.1. System C++ runtime. | +| Pango/GLib | LGPL family; installed Pango metadata LGPL-2.0-or-later. FLTK text stack. | +| Cairo | LGPL-2.1-only OR MPL-1.1. FLTK graphics stack. | +| Fontconfig | HPND AND Unicode-DFS-2016 in inspected package. | +| FreeType | FTL OR GPL-2.0-or-later; distribution can select FTL. | +| HarfBuzz, Wayland, libdecor, xkbcommon | MIT-family licenses; check each shipped package. | +| X11 libraries | MIT/X11-family notices. | +| D-Bus | AFL-2.1 OR GPL-2.0-or-later. | +| Python 3 | PSF-2.0; optional build/test/package scripts use only the standard library. Not an application runtime dependency. | + +GNU Make, GCC/Clang, binutils, pkg-config and optional Xvfb/xdotool/sanitizers are +build/test tools, not shipped dependencies. New Python tooling needs Python 3.12+ +for safe tar extraction in release tests; this host has Python 3.14.8. No packages +were installed and no application dependency was added in Phase 9. + +Binary redistributors must inspect their own `readelf -d`/`ldd` dependency set +and supply the notices/source/relinking facilities required by those exact +licenses and linking choices. FLTK's exception does not remove obligations of +other linked libraries. This review covers the current unmodified system-linked +build; static all-in-one bundles and external plugins need a separate review. diff --git a/docs/LINUX.md b/docs/LINUX.md new file mode 100644 index 0000000..e3a06a8 --- /dev/null +++ b/docs/LINUX.md @@ -0,0 +1,58 @@ +# Linux setup and installation + +Use a C99 compiler, C++17 compiler for the GUI, GNU Make, binutils and pkg-config. +The core needs only the C/math runtime. FLTK 1.4 development files enable the GUI; +JACK-compatible development files enable the optional audio/MIDI backend. +Packaging/install tests additionally use Python 3.12+ (standard library only). +No command starts a server, installs dependencies or changes system settings. + +```sh +make all test +make gui +./build/xtc +``` + +Start/configure a JACK or PipeWire JACK-compatible server through your existing +system setup first. xtc opens without starting one and requires reconnect after +server loss/rate changes. In a separate JACK routing tool connect desired MIDI +to `midi_in`, audio capture to `input`, and `output`/`output_right` to left/right +playback ports. No physical routes are automatic. Input/recording remain mono. +See GUI.md, AUDIO_FILES.md, MIDI.md and MIXER.md for composing/recording/editing. + +`xtc --help` and `xtc --version` work without a display/server. `xtc project.xtc` +opens one project; no-file startup retains FLTK options such as `-display`. +`xtc-check --validate project.xtc` validates metadata only; native graphs cannot +be serialized. See RECOVERY.md before replacing damaged files. + +## Staged installation + +Core-only installation retains the original `make install` behavior and now +adds xtc-check, pkg-config metadata, license and documentation. install-gui adds +the GUI executable and desktop launcher. Defaults to PREFIX=/usr/local: + +```sh +make install-gui DESTDIR="$PWD/build/stage" PREFIX=/usr +make uninstall DESTDIR="$PWD/build/stage" PREFIX=/usr +``` + +Review staged contents before using a package manager or choosing a writable +live prefix. For a user installation, use an absolute PREFIX such as +`$HOME/.local` and ensure its bin is in PATH. Prefixes containing spaces are +tested; shell metacharacters/newlines, quotes, dollar signs and backslashes are +not supported. No root installation is performed by tests. Uninstall from the +same source version and prefix: it removes only known files, keeps unrelated +files and removes directories only if empty. It is idempotent. It is not a +system package manager and cannot undo replacement of another installation. + +Installed files: bin/xtc-check, optional bin/xtc, include/xtc/*.h, lib/libxtc.a, +lib/pkgconfig/xtc.pc, share/doc/xtc, share/licenses/xtc/LICENSE and optional +share/applications/xtc.desktop. No daemon, service, bundled library or MIME +registration is installed. Desktop caches are left to the packager/environment. +The launcher uses the system audio icon and can open a single file argument. + +C/C++ consumers use `pkg-config --cflags --libs --static xtc` plus the chosen +compiler; expose PREFIX/lib/pkgconfig through PKG_CONFIG_PATH when needed. +Static backend consumers need JACK if that optional backend was built; the pc +file records its private dependency. Public headers contain no JACK/FLTK types. +To build without optional libraries, set PKG_CONFIG=false FLTK_CONFIG=false. +Python is not needed to run the installed executables or library. diff --git a/docs/NATIVE_API.md b/docs/NATIVE_API.md new file mode 100644 index 0000000..35067df --- /dev/null +++ b/docs/NATIVE_API.md @@ -0,0 +1,83 @@ +# Native C processor API, version 1 + +`xtc_native.h` defines a small compile-time extension boundary for trusted C99 +sources, effects and instruments. Build `make build/native-gain` and run it for a +complete graph-effect example in `examples/native_gain.c`. This is a source API +for this release, not a promise of binary compatibility across builds/platforms. +There is no module scanner, dynamic loader, SDK dependency or sandbox. + +## Descriptor and ownership + +Set `abi_version = XTC_NATIVE_ABI_VERSION`, `struct_size = sizeof(descriptor)`, +a terminated printable non-space ASCII identifier (1..63 bytes), kind and all +four callbacks. Unknown ABI/size returns UNSUPPORTED; invalid fields return +INVALID_ARGUMENT. The host copies the entire descriptor, including its name. +Callback code must remain loaded until all graphs, plans and instances are gone. +No plugin/global registry is consulted or mutated by rendering. + +`xtc_native_create` validates rate (positive, <=768000), frames (1..65536) and a +finite scalar parameter. It runs prepare on the control thread with a NULL +initial state pointer. Prepare may allocate/preload; on failure the host calls +destroy even for NULL state. Destroy must accept NULL. The host then resets the +new instance. On failure the output handle is NULL; successful instances belong +to the caller. Destroy only after the processing owner is stopped/joined. + +Reset and process have one audio owner (or run while stopped). They must never +allocate/free, perform I/O/logging, block, call GUI functions or mutate shared +control data. Reset must be idempotent. Prepare/destroy run only on the control +thread, including failed compilation and retired-plan collection. Code executed +inside callbacks is trusted: xtc cannot enforce its realtime behavior or protect +against memory corruption. C++ processors must use C linkage and never throw +across this boundary; domain/DSP code in this repository remains C. + +## Block contract + +A block contains mono input/output, frames, absolute sample position and optional +MIDI events. Input NULL means silence; exact input/output aliasing is allowed and +processors must support it. Output is writable for every frame, including silent +frames; do not retain any borrowed block pointers. Other overlapping buffers are +not supported. Only instruments receive events; event count <=256, offsets are +sorted and strictly less than frames, with equal offsets retaining caller order. +Pitch/velocity/channel/type bounds follow xtc_midi.h. Zero frames is a no-op. + +The host rejects invalid blocks before changing state/output, and reports +CAPACITY above the prepared block limit. Position arithmetic cannot overflow. +A discontinuous position resets held voices/state before processing. Call reset +for Stop/panic when hosting instruments yourself; send channel ALL_OFF for +channel-specific panic. The native host does not implement sustain or routing. + +On successful processing, nonfinite samples become silence. Callback failure +silences the block, resets that processor and returns its error. Already advanced +processors in a graph are not rolled back. Never rely on failure to transactionally +rewind DSP; prepare fallible resources before publication. There is no latency +compensation, tail protocol, parameter ramp or dynamic port negotiation in ABI 1. + +## Graph and instrument integration + +`xtc_graph_add_native` adds effects (input 0/output 1) or sources (output 1), with +a scalar immutable parameter. Compilation creates two independent mono instances +for stereo; no mutable state is shared between sides. Graph-only edits retain +the existing pending/retired publication contract. Reset clears native state and +its graph-local frame clock; graph positions count successfully rendered frames +since compilation/reset, independently of backend seek/transport. Sources needing +transport time should be hosted explicitly with a block position. + +For an instrument, use create/process directly with sample-offset events, then +feed its output to an existing XTC_NODE_INSTRUMENT through process_sources or +process_stereo. The host owns event routing and panic/reset delivery. Graph-native +instrument addition returns UNSUPPORTED because current graph ports carry audio, +not MIDI. This separation avoids silently dropping instrument events. The built-in +JACK MIDI sine instrument remains the GUI's instrument; custom instrument selection +and event graph ports are future work. + +Descriptors survive graph/project cloning and transactional remove/connect edits; +custom ADD_NODE commands use the dedicated constructor rather than a GUI menu. +Native nodes have no project-file representation: Save returns UNSUPPORTED and +preserves the destination. Format v4 still accepts only builtin types 1..6. +A durable identifier/registry, versioned processor state and missing-module policy +must precede serialized native extensions or external plugin hosting. + +Tests cover ABI/size/field rejection, descriptor copying, aliased buffers, +prepare/process failures, stereo compile cleanup, retirement/destruction outside +processing, event offsets, seek note reset, source ports, instrument-buffer graph +routing and unsupported-save preservation. Run `make test`. diff --git a/docs/PERFORMANCE.md b/docs/PERFORMANCE.md new file mode 100644 index 0000000..5dd74f7 --- /dev/null +++ b/docs/PERFORMANCE.md @@ -0,0 +1,35 @@ +# Performance and realtime qualification + +`make test-performance` runs a hardware-independent 48 kHz, 128-frame stereo +workload: 64 graph nodes, 4096 MIDI notes (32 simultaneous), 16 tracks and gain +automation. Preparation and block timing use CLOCK_MONOTONIC outside the engine; +callback code itself does not clock or log. The test checks finite output and +reports mean, p99, maximum and CPU-time/audio-duration ratio. A run is a workload +measurement, not a universal realtime threshold. Run more blocks with: + +```sh +./build/test_performance 10000 +``` + +Initial Phase 9 measurement on an AMD Ryzen Threadripper 1920X, GCC 16.2.1, -O2, +2000 blocks: preparation 3.045 ms, mean 308.301 us, p99 422.153 us, maximum +665.096 us, versus a 2666.667 us block deadline; CPU/audio ratio 0.1156. These are +one host/run's observed values and depend on scheduling/load/compiler. The test +excludes WAV disk preparation, external/native plugin complexity, GUI redraw, +server scheduling and actual hardware latency. Native callbacks must provide +their own bounded realtime implementation. + +Existing software JACK loopback, stereo MIDI/save/reopen, record/reopen, +restart/close and concurrent graph handoff tests remain available. Runtime +fault injection covers loss/rate/xrun/null-buffer and lifecycle failures without +altering the shared server. ASan/UBSan validate memory/error paths; TSan checks +recorder/handoff races, not timing. Sanitized runs are not latency qualification. + +Current worst-case storage includes up to 256 MiB preloaded mono WAV samples, +paired graph buffers (128 MiB at 256 nodes/65536 frames; GUI limit 32 MiB), bounded +MIDI/mix metadata and up to 33 metadata/graph history snapshots. Pending/retired +plans can retain additional graph memory. Native preparation can allocate more; +the API cannot enforce an extension's memory budget. There is no CPU overload +scheduler, disk streaming, latency compensation or callback allocator audit of +arbitrary third-party code. High-density interactive projects, physical devices, +long-duration stereo/recording and live server termination remain unqualified. diff --git a/docs/PLUGIN_HOSTING.md b/docs/PLUGIN_HOSTING.md new file mode 100644 index 0000000..2dc8014 --- /dev/null +++ b/docs/PLUGIN_HOSTING.md @@ -0,0 +1,24 @@ +# External plugin hosting evaluation + +Decision: defer external hosting after Phase 9. The native source API is useful +for proving processor contracts without adding an SDK or scanning/loading code. + +| Candidate | Benefits | Work required before adoption | +| --- | --- | --- | +| LV2 | Open specifications, established Linux hosts/plugins | RDF discovery and feature negotiation, URIDs, state, UI lifetime, worker requests and event mapping; likely additional discovery dependency | +| CLAP | C ABI, explicit lifecycle and event model | Extension negotiation, parameter/state persistence, host callbacks, main/audio-thread scheduling and UI integration | +| VST3 | Broad plugin availability | SDK/license review for the selected SDK release, C++ adapter outside the C core, component/controller and editor lifecycle, state and event translation | + +No candidate was linked, scanned or loaded in this phase. API/SDK and plugin +licenses must be checked at the revision actually adopted; they are separate +from xtc's MIT license. Supporting a format does not grant redistribution rights +to individual plugins. Do not search user plugin directories from the callback. + +Before implementation, require versioned module identity and state, a missing- +plugin recovery policy, preparation failure/partial cleanup tests, bounded events +and parameter publication, latency/tail handling, callback-safe unload after +retirement, a GUI/main-thread boundary and an explicit crash-containment decision. +Native ABI tests cover some ownership mechanics but do not prove third-party +code safe. Event graph ports and custom-instrument GUI selection remain gaps. +Choose one format with concrete user needs after those contracts are specified; +do not introduce a general multi-format framework now. See decision 0008. diff --git a/docs/PROJECT_FORMAT.md b/docs/PROJECT_FORMAT.md index 3fabd46..9392586 100644 --- a/docs/PROJECT_FORMAT.md +++ b/docs/PROJECT_FORMAT.md @@ -105,3 +105,12 @@ external absolute paths, not copied on Save As. Metadata loading preserves missing references; audio-plan preparation reports unreadable/invalid assets and renders them silently. Relinking replaces a selected clip's path without changing its IDs or arrangement. See AUDIO_FILES.md for recovery and limitations. + +## Phase 9 extension and recovery policy + +Format remains v4. Native processor descriptors are process-local and have no +serialized representation; saving a native graph returns UNSUPPORTED and retains +the existing destination. Unknown file versions/types remain rejected. See +NATIVE_API.md for the extension boundary and RECOVERY.md for write-failure, killed +writer and unfinished-recording behavior. Metadata validation is available via +`xtc-check --validate PROJECT` without a GUI or server; it does not load assets. diff --git a/docs/RECOVERY.md b/docs/RECOVERY.md new file mode 100644 index 0000000..f4bd65c --- /dev/null +++ b/docs/RECOVERY.md @@ -0,0 +1,48 @@ +# Persistence and failure recovery + +The project format remains v4. Readers migrate v1/v2/v3, validate bounded input +and atomically install a candidate only after complete success. Existing tests +cover exact migration/roundtrip, truncations, malformed references, counts, +IDs, nonfinite values and failed GUI/backend installation. Native processors +are session-only; saving their graph returns UNSUPPORTED rather than writing +process addresses or an unreadable project. See PROJECT_FORMAT.md. + +Project Save writes a mode-0600 same-directory `NAME.tmp.XXXXXX`, flushes and +fsyncs it, closes and renames it over the destination. Before rename, handled +failures remove the temporary and retain the destination. POSIX rename gives +complete old-or-new destination contents during process failure. The directory +is not fsynced, so this is not a power-loss durability guarantee. Symlinks are +replaced rather than followed at rename; original file permissions are not kept. + +Recording similarly writes/finalizes a same-directory temporary, then renames +it. Stop must synchronize callbacks before draining/finalizing. Ring overflow +returns an explicit error and may finalize a partial take for manual recovery; +I/O failure retains the previous destination. A killed recorder does not finish +its WAV header. Source WAV files remain external absolute paths; Save As does +not collect them. Back up the project and assets together. + +## After a crash + +1. Preserve the destination and any `.tmp.*` files before attempting repair. +2. Run `xtc-check --validate project.xtc` for metadata validation without a GUI, + display or audio server. This does not verify assets or realtime preparation. +3. A valid destination can be opened normally. If only a temporary contains useful + data, copy it to a separate candidate and validate that copy before replacing + anything. Partial project files are rejected; there is no salvage parser. +4. An unfinished WAV temporary may have a zero/incorrect data length and needs + a separate recovery tool or manual header repair on a copy. xtc does not guess + its intended length or automatically rename orphan temporaries. +5. Missing assets preserve silent clips. Use Relink selected audio, then Play to + revalidate. For server loss/format changes, Stop/Reconnect and explicit routing + are required. Panic/F9 clears held notes. Failed close retains ownership for retry. + +There is no autosave, crash journal, automatic orphan recovery or persisted Undo +history. Existing finalized files are protected against the tested process-level +failures; unsaved edits/in-flight takes can still be lost. + +`make test-recovery` uses temporary directories and child processes only. It +forces an EFBIG-style write failure with a child-local file-size limit, kills +eight project writers while temporaries are visible, and kills a recorder after +draining but before finalization. Complete old/new destinations and explicit +orphan temporaries are verified. It does not cut power, terminate the shared +audio server, simulate a failing filesystem, or qualify disk/device endurance. diff --git a/docs/RELEASING.md b/docs/RELEASING.md new file mode 100644 index 0000000..9e27f6c --- /dev/null +++ b/docs/RELEASING.md @@ -0,0 +1,56 @@ +# Release preparation (0.1.0 development prototype) + +Phase 9 makes source packaging, staged install/uninstall and repeatable builds +testable. It does not turn the prototype into a production-qualified DAW. The +version remains 0.1.0; no tag, external release or binary publication is created +by `make dist` or `make release`. + +## Build and package + +```sh +make release +make test-install +make test-recovery test-performance +make dist +make test-release +``` + +release builds optimized, warning-enabled C/C++ code, tests and the optional GUI +in BUILD_DIR/release with no debug paths, a source-prefix map, deterministic ar +metadata and FLTK include/define flags rather than distribution-specific LTO +flags. It retains assertions so the test harness actually runs. dist creates +BUILD_DIR/xtc-0.1.0.tar.gz using Python's standard library. It includes only Git +indexed regular files (working-tree contents, including staged new files), plus +a source manifest; stage intended files first and check `git diff --cached`. +Untracked files, ignored build output and .git are excluded. Exported archives +can reproduce themselves without Git using SOURCE_MANIFEST.json. + +Archive order, permissions, ownership, gzip filename and timestamps are normalized. +SOURCE_DATE_EPOCH defaults to 0; set an unsigned 32-bit epoch explicitly if desired. +The archive does not claim to identify a commit: release from a clean checkout +and record its commit/hash externally. Do not treat a dirty-tree archive as an +attested release. No artifact is uploaded by these targets. + +test-release unpacks the source into two different temporary paths, runs release +in each, compares libxtc.a/xtc-check/optional xtc SHA-256 values, and repacks each +export to compare archive bytes. This verifies reproducibility for the same host, +compiler, flags and dependency versions. It does not promise bit-identical output +across distributions, compilers or library updates. Compiler/debug/sanitizer +build directories are intentionally separate; changing toolchain/flags in place +requires a clean directory. Python release tests require version 3.12+. + +## Qualification record + +Run normal core/GUI and live software JACK tests, no-JACK builds and ASan/UBSan. +GUI leak tests retain the narrow fontconfig suppression; core tests do not. +TSan recorder/publication tests complement lifecycle tests. Native API tests +verify callback failures, cleanup and retirement; they cannot certify arbitrary +extensions. See BUILD.md and tests/integration/README.md for commands. + +Record versions, archive hashes, installation/consumer checks and measured timing. +Use LICENSES.md for dependency/license review and PLUGIN_HOSTING.md for the +external-hosting decision. Review RECOVERY.md for migration and process-crash +semantics. Physical devices, power loss, sustained recording/performance, user +acceptance on other distributions and binary redistribution are still separate +qualification work. There is no DEB/RPM/AppImage, bundled runtime, autosave, +external plugin host or stable shared-library extension ABI in this release. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 6ed4936..3df08aa 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -105,4 +105,15 @@ implemented composition/mixing workflow. Native node API, evaluate plugin hosting, licensing review, packaging, installation tests, performance and recovery tests. +Implemented: versioned native C processor lifecycle and graph source/effect API, +explicit instrument-buffer integration, example and error/retirement tests; +external-hosting evaluation (deferred); dependency/license review; deterministic +source/release targets; core/GUI staged install/uninstall and installed C/C++ +consumer tests; headless version/help/metadata validation; offline performance +and process-crash/write-failure recovery tests. Source format stays v4, reading +v1–v3. See NATIVE_API.md, RELEASING.md, LINUX.md, LICENSES.md, PERFORMANCE.md and +RECOVERY.md. Native graphs remain session-only, with unsupported saves preserving +files. No stable binary plugin ABI, external host, system package, autosave or +physical-device/endurance qualification is claimed. + Do not skip phases without documenting why and preserving acceptance criteria. diff --git a/docs/decisions/0008-native-processors-and-source-releases.md b/docs/decisions/0008-native-processors-and-source-releases.md new file mode 100644 index 0000000..5f5efa8 --- /dev/null +++ b/docs/decisions/0008-native-processors-and-source-releases.md @@ -0,0 +1,19 @@ +# 0008 — Native processors and source release preparation + +Status: accepted, Phase 9. + +Add a versioned copied C descriptor with control-thread prepare/destroy and +single-owner realtime reset/process. Prove effects/sources in the existing graph, +with two independent mono instances for stereo, and event instruments rendered +through the public instrument-source buffer boundary. Keep trusted code statically +linked. Native project serialization and graph event ports need more contracts; +reject unsupported saves without replacing files. Defer LV2/CLAP/VST3 after a +concrete evaluation rather than linking SDKs speculatively. + +Keep format v4 and test migration plus process-crash/write-failure behavior. +Provide deterministic source exports and optimized release builds, staged +core/GUI installation, exact uninstall and installed C/C++ consumer tests. Use +Python standard-library tools only for packaging/testing; no application runtime +dependency or system changes. Document dependency licenses, Linux routing and +measured offline performance separately from unverified hardware/endurance. +See NATIVE_API.md, PLUGIN_HOSTING.md, RELEASING.md and RECOVERY.md. diff --git a/examples/README.md b/examples/README.md index 06d512d..53ae476 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,4 +1,5 @@ # Examples -Add small runnable examples here as the C API stabilizes. Keep examples focused -on one behavior and make sure they build through the normal Makefile. +`native_gain.c` is a complete trusted native C effect compiled into an offline +graph. Build/run with `make build/native-gain && ./build/native-gain`; it needs +no GUI/server/plugin SDK. See ../docs/NATIVE_API.md for lifecycle/event contracts. diff --git a/examples/native_gain.c b/examples/native_gain.c new file mode 100644 index 0000000..50a0179 --- /dev/null +++ b/examples/native_gain.c @@ -0,0 +1,64 @@ +/* Compile-time native effect example. No module loading or external SDK. */ +#include "xtc/xtc_native.h" +#include +#include +typedef struct { + float gain; +} gain_state; +static xtc_result prepare(double rate, size_t frames, float gain, void **out) +{ + gain_state *state = malloc(sizeof(*state)); + (void)rate; + (void)frames; + if (!state) + return XTC_ERR_OUT_OF_MEMORY; + state->gain = gain; + *out = state; + return XTC_OK; +} +static void reset(void *state) { (void)state; } +static void destroy(void *state) { free(state); } +static xtc_result process(void *context, const xtc_native_block *block) +{ + const gain_state *state = context; + size_t i; + for (i = 0; i < block->frames; ++i) + block->output[i] = block->input ? block->input[i] * state->gain : 0; + return XTC_OK; +} +int main(void) +{ + const xtc_native_descriptor descriptor = {XTC_NATIVE_ABI_VERSION, + sizeof(xtc_native_descriptor), + "org.xtc.example-gain", + XTC_NATIVE_EFFECT, + prepare, + reset, + process, + destroy}; + xtc_graph *graph = NULL; + xtc_plan *plan = NULL; + size_t effect, output; + float input[4] = {1, 0.5f, -1, 0}, rendered[4]; + xtc_result result = xtc_graph_create(2, &graph); + if (result == XTC_OK) + result = xtc_graph_add_native(graph, &descriptor, 0.25f, &effect); + if (result == XTC_OK) + result = xtc_graph_add_output(graph, &output); + if (result == XTC_OK) + result = xtc_graph_connect(graph, effect, output); + if (result == XTC_OK) + result = xtc_graph_compile(graph, 48000, 4, &plan); + if (result == XTC_OK) + result = xtc_plan_process(plan, input, rendered, 4); + if (result == XTC_OK && (rendered[0] != 0.25f || rendered[2] != -0.25f)) + result = XTC_ERR_INVALID_STATE; + xtc_plan_destroy(plan); + xtc_graph_destroy(graph); + if (result != XTC_OK) { + fprintf(stderr, "Native example: %s\n", xtc_result_string(result)); + return EXIT_FAILURE; + } + puts("Native gain graph: 0.25, 0.125, -0.25, 0"); + return EXIT_SUCCESS; +} diff --git a/include/xtc/xtc.h b/include/xtc/xtc.h index 775b726..f329716 100644 --- a/include/xtc/xtc.h +++ b/include/xtc/xtc.h @@ -2,15 +2,12 @@ #define XTC_H #include "xtc_types.h" +#include "xtc_version.h" #ifdef __cplusplus extern "C" { #endif -#define XTC_VERSION_MAJOR 0 -#define XTC_VERSION_MINOR 1 -#define XTC_VERSION_PATCH 0 - typedef struct xtc_app xtc_app; /* Creates an application context. On success, *out_app is caller-owned. */ diff --git a/include/xtc/xtc_engine.h b/include/xtc/xtc_engine.h index 70dc69a..9159568 100644 --- a/include/xtc/xtc_engine.h +++ b/include/xtc/xtc_engine.h @@ -34,7 +34,8 @@ typedef enum { XTC_NODE_METER, XTC_NODE_OUTPUT, XTC_NODE_AUDIO, /* Timeline mix source, output only. */ - XTC_NODE_INSTRUMENT /* Internal MIDI instrument, output only. */ + XTC_NODE_INSTRUMENT, /* Internal MIDI instrument, output only. */ + XTC_NODE_NATIVE /* Trusted source/effect; see xtc_native.h. */ } xtc_node_type; typedef struct { diff --git a/include/xtc/xtc_native.h b/include/xtc/xtc_native.h new file mode 100644 index 0000000..dff4956 --- /dev/null +++ b/include/xtc/xtc_native.h @@ -0,0 +1,56 @@ +#ifndef XTC_NATIVE_H +#define XTC_NATIVE_H +#include "xtc_midi.h" +#ifdef __cplusplus +extern "C" { +#endif +#define XTC_NATIVE_ABI_VERSION 1U +#define XTC_NATIVE_ID_BYTES 64U +typedef enum { XTC_NATIVE_EFFECT = 1, XTC_NATIVE_SOURCE, XTC_NATIVE_INSTRUMENT } xtc_native_kind; +typedef struct { + const float *input; /* NULL is silence; may equal output. Borrowed for call. */ + float *output; /* Write every frame, including silence. */ + size_t frames; + uint64_t position; /* Absolute sample position, no wall clock. */ + const xtc_midi_event *events; /* Instruments only, sorted block offsets. */ + size_t event_count; +} xtc_native_block; +typedef struct { + unsigned int abi_version; + size_t struct_size; /* Must equal sizeof(xtc_native_descriptor) for ABI 1. */ + char identifier[XTC_NATIVE_ID_BYTES]; /* Nonempty printable ASCII, terminated. */ + xtc_native_kind kind; + /* Control thread. On error, any non-NULL *state is destroyed by host. */ + xtc_result (*prepare)(double sample_rate, size_t max_frames, float parameter, void **state); + /* Single audio owner (or stopped). No allocation/free, I/O, locks or GUI. */ + void (*reset)(void *state); + xtc_result (*process)(void *state, const xtc_native_block *block); + /* Control thread after quiescence; must accept NULL. */ + void (*destroy)(void *state); +} xtc_native_descriptor; +typedef struct xtc_native xtc_native; +/* Descriptors are copied; callback code must remain loaded through destruction. + * Trusted statically linked C code, not a sandbox or stable shared-library ABI. */ +xtc_result xtc_native_validate(const xtc_native_descriptor *); +xtc_result xtc_native_create(const xtc_native_descriptor *, double rate, size_t max_frames, + float parameter, xtc_native **out_processor); +void xtc_native_destroy(xtc_native *); +void xtc_native_reset(xtc_native *); +/* Bounded, single-owner, realtime-safe host path (callbacks must obey contract). + * Invalid arguments leave state/output unchanged. Zero frames is a no-op. + * Discontinuity resets before processing. Callback failure silences this block + * and resets the processor. Successful nonfinite output samples become zero. */ +xtc_result xtc_native_process(xtc_native *, const xtc_native_block *); +/* Graph accepts effects/sources; creates independent L/R instances on compile. + * For instruments, process events explicitly then feed the existing instrument + * source buffer through xtc_plan_process_sources/stereo. Graph add returns + * UNSUPPORTED for instrument descriptors (no event port in current topology). + * Native graphs are session-only: project save returns UNSUPPORTED. */ +xtc_result xtc_graph_add_native(xtc_graph *, const xtc_native_descriptor *, float parameter, + size_t *out_id); +xtc_result xtc_graph_get_native(const xtc_graph *, size_t node_id, + xtc_native_descriptor *out_descriptor); +#ifdef __cplusplus +} +#endif +#endif diff --git a/include/xtc/xtc_version.h b/include/xtc/xtc_version.h index dd3aa6a..c8a5fd2 100644 --- a/include/xtc/xtc_version.h +++ b/include/xtc/xtc_version.h @@ -1,6 +1,9 @@ #ifndef XTC_VERSION_H #define XTC_VERSION_H +#define XTC_VERSION_MAJOR 0 +#define XTC_VERSION_MINOR 1 +#define XTC_VERSION_PATCH 0 #define XTC_VERSION_STRING "0.1.0" #define XTC_PROJECT_NAME "xtc" diff --git a/packaging/xtc.desktop b/packaging/xtc.desktop new file mode 100644 index 0000000..4bb1f75 --- /dev/null +++ b/packaging/xtc.desktop @@ -0,0 +1,10 @@ +[Desktop Entry] +Type=Application +Name=xtc +Comment=Modular audio workstation +Exec=xtc %f +TryExec=xtc +Icon=audio-x-generic +Terminal=false +Categories=AudioVideo;Audio;Midi;Sequencer; +StartupNotify=false diff --git a/packaging/xtc.pc.in b/packaging/xtc.pc.in new file mode 100644 index 0000000..c3782eb --- /dev/null +++ b/packaging/xtc.pc.in @@ -0,0 +1,11 @@ +prefix=@PREFIX@ +exec_prefix=${prefix} +libdir=${prefix}/lib +includedir=${prefix}/include + +Name: xtc +Description: Modular C audio engine and project library +Version: @VERSION@ +Requires.private: @JACK@ +Libs: -L"${libdir}" -lxtc -lm +Cflags: -I"${includedir}" diff --git a/src/engine/offline_engine.c b/src/engine/offline_engine.c index 9009c55..f1357e7 100644 --- a/src/engine/offline_engine.c +++ b/src/engine/offline_engine.c @@ -14,11 +14,17 @@ struct xtc_plan { double *phases; double sample_rate; float input_gain; + uint64_t position; + xtc_native **native; }; void xtc_plan_destroy(xtc_plan *plan) { if (plan == NULL) return; + if (plan->native) { + size_t i; for (i = 0; i < plan->node_count * 2; ++i) xtc_native_destroy(plan->native[i]); + } + free(plan->native); free(plan->nodes); free(plan->order); free(plan->offsets); @@ -55,6 +61,7 @@ xtc_result xtc_graph_compile(const xtc_graph *graph, double sample_rate, plan->max_frames = max_frames; plan->sample_rate = sample_rate; plan->input_gain = 1.0f; + plan->native = calloc(storage_nodes * 2U, sizeof(*plan->native)); plan->nodes = calloc(storage_nodes, sizeof(*plan->nodes)); plan->order = calloc(storage_nodes, sizeof(*plan->order)); plan->offsets = calloc(plan->node_count + 1U, sizeof(*plan->offsets)); @@ -63,7 +70,7 @@ xtc_result xtc_graph_compile(const xtc_graph *graph, double sample_rate, plan->buffers = calloc(storage_nodes * max_frames * 2U, sizeof(*plan->buffers)); plan->peaks = calloc(storage_nodes, sizeof(*plan->peaks)); plan->phases = calloc(storage_nodes, sizeof(*plan->phases)); - if (plan->nodes == NULL || plan->order == NULL || plan->offsets == NULL || + if (plan->native == NULL || plan->nodes == NULL || plan->order == NULL || plan->offsets == NULL || plan->sources == NULL || plan->render == NULL || plan->buffers == NULL || plan->peaks == NULL || plan->phases == NULL) { xtc_plan_destroy(plan); @@ -83,6 +90,16 @@ xtc_result xtc_graph_compile(const xtc_graph *graph, double sample_rate, plan->render[node] = (unsigned char)(has_output || graph->require_output_nodes ? plan->nodes[node].type == XTC_NODE_OUTPUT : terminal); } + for (node = 0; node < plan->node_count; ++node) { + if (plan->nodes[node].type == XTC_NODE_NATIVE) { + size_t side; + for (side = 0; side < 2; ++side) { + result = xtc_native_create(&plan->nodes[node].native, sample_rate, max_frames, + plan->nodes[node].value, &plan->native[side * plan->node_count + node]); + if (result != XTC_OK) { xtc_plan_destroy(plan); return result; } + } + } + } plan->offsets[plan->node_count] = cursor; *out_plan = plan; return XTC_OK; @@ -91,6 +108,8 @@ xtc_result xtc_graph_compile(const xtc_graph *graph, double sample_rate, xtc_result xtc_plan_reset(xtc_plan *plan) { if (plan == NULL) return XTC_ERR_INVALID_ARGUMENT; + { size_t i; for (i = 0; i < plan->node_count * 2; ++i) xtc_native_reset(plan->native[i]); } + plan->position = 0; memset(plan->phases, 0, plan->node_count * sizeof(*plan->phases)); memset(plan->peaks, 0, plan->node_count * sizeof(*plan->peaks)); return XTC_OK; @@ -115,7 +134,7 @@ xtc_result xtc_plan_process_stereo(xtc_plan *plan, const float *input, { size_t index, frame, edge, channel; if (plan == NULL || output == NULL || (right && right == output)) return XTC_ERR_INVALID_ARGUMENT; - if (frame_count > plan->max_frames) return XTC_ERR_CAPACITY; + if (frame_count > plan->max_frames || plan->position > UINT64_MAX - frame_count) return XTC_ERR_CAPACITY; if (frame_count == 0U) return XTC_OK; /* All external input reads finish before output is written, allowing aliasing. * Nonfinite input/arithmetic becomes silence per sample, never persistent DSP state. */ @@ -150,6 +169,16 @@ xtc_result xtc_plan_process_stereo(xtc_plan *plan, const float *input, buffer[frame] = sample; if (fabsf(sample) > peak) peak = fabsf(sample); } + if (spec.type == XTC_NODE_NATIVE) { + xtc_native_block block = {spec.native.kind == XTC_NATIVE_EFFECT ? buffer : NULL, + buffer, frame_count, plan->position, NULL, 0}; + xtc_result result = xtc_native_process(plan->native[channel * plan->node_count + node], &block); + if (result != XTC_OK) { + memset(output, 0, frame_count * sizeof(*output)); + if (right) memset(right, 0, frame_count * sizeof(*right)); + return result; + } + } if (!right || channel == 1) plan->phases[node] = phase; if (spec.type == XTC_NODE_METER) plan->peaks[node] = channel && plan->peaks[node] > peak ? plan->peaks[node] : peak; } @@ -160,6 +189,7 @@ xtc_result xtc_plan_process_stereo(xtc_plan *plan, const float *input, destination[frame] = isfinite(sample) ? sample : 0.0f; } } + plan->position += frame_count; return XTC_OK; } diff --git a/src/graph/graph.c b/src/graph/graph.c index 3f07f70..045f48d 100644 --- a/src/graph/graph.c +++ b/src/graph/graph.c @@ -74,7 +74,8 @@ xtc_result xtc_graph_get_port(const xtc_graph *graph, size_t node_id, node_id = node_index(graph, node_id); if (node_id == SIZE_MAX) return XTC_ERR_NOT_FOUND; type = graph->nodes[node_id].type; - if (port_id > 1U || (port_id == 0U && (type == XTC_NODE_OSCILLATOR || type == XTC_NODE_AUDIO || type == XTC_NODE_INSTRUMENT)) || + if (port_id > 1U || (port_id == 0U && (type == XTC_NODE_OSCILLATOR || type == XTC_NODE_AUDIO || type == XTC_NODE_INSTRUMENT || + (type == XTC_NODE_NATIVE && graph->nodes[node_id].native.kind != XTC_NATIVE_EFFECT))) || (port_id == 1U && type == XTC_NODE_OUTPUT)) return XTC_ERR_NOT_FOUND; out_port->direction = port_id == 0U ? XTC_PORT_INPUT : XTC_PORT_OUTPUT; out_port->signal_type = XTC_SIGNAL_AUDIO; @@ -250,3 +251,22 @@ xtc_result xtc_graph_apply_edit(const xtc_graph *source, const xtc_graph_edit *e *out_graph = graph; return XTC_OK; } + +xtc_result xtc_graph_add_native(xtc_graph *graph, const xtc_native_descriptor *descriptor, float value, size_t *out) +{ + xtc_result result = xtc_native_validate(descriptor); + if (result != XTC_OK) return result; + if (descriptor->kind == XTC_NATIVE_INSTRUMENT) return XTC_ERR_UNSUPPORTED; + result = add_node(graph, XTC_NODE_NATIVE, value, out); + if (result == XTC_OK) graph->nodes[graph->node_count - 1].native = *descriptor; + return result; +} +xtc_result xtc_graph_get_native(const xtc_graph *graph, size_t id, xtc_native_descriptor *out) +{ + size_t i; + if (!graph || !out) return XTC_ERR_INVALID_ARGUMENT; + i = node_index(graph, id); + if (i == SIZE_MAX) return XTC_ERR_NOT_FOUND; + if (graph->nodes[i].type != XTC_NODE_NATIVE) return XTC_ERR_INVALID_ARGUMENT; + *out = graph->nodes[i].native; return XTC_OK; +} diff --git a/src/graph/graph_internal.h b/src/graph/graph_internal.h index fb8ffdc..ab334f9 100644 --- a/src/graph/graph_internal.h +++ b/src/graph/graph_internal.h @@ -1,9 +1,10 @@ #ifndef XTC_GRAPH_INTERNAL_H #define XTC_GRAPH_INTERNAL_H #include "xtc/xtc_engine.h" +#include "xtc/xtc_native.h" #define XTC_MAX_NODES 256U #define XTC_MAX_FRAMES 65536U -typedef struct { xtc_node_type type; float value; size_t id; } xtc_node; +typedef struct { xtc_node_type type; float value; size_t id; xtc_native_descriptor native; } xtc_node; typedef struct { size_t source; size_t destination; } xtc_edge; struct xtc_graph { size_t max_nodes, node_count, edge_count, next_id; diff --git a/src/gui/graph_widget.cpp b/src/gui/graph_widget.cpp index 1dd2658..658ce53 100644 --- a/src/gui/graph_widget.cpp +++ b/src/gui/graph_widget.cpp @@ -14,6 +14,7 @@ const char *node_name(xtc_node_type type) case XTC_NODE_OSCILLATOR: return "Oscillator"; case XTC_NODE_GAIN: return "Gain"; case XTC_NODE_METER: return "Meter"; + case XTC_NODE_NATIVE: return "Native processor"; case XTC_NODE_INSTRUMENT: return "Instrument"; case XTC_NODE_AUDIO: return "Audio clips"; case XTC_NODE_OUTPUT: return "Output"; diff --git a/src/gui/main.cpp b/src/gui/main.cpp index 395538d..a8c9f2a 100644 --- a/src/gui/main.cpp +++ b/src/gui/main.cpp @@ -3,15 +3,29 @@ #include #include #include "main_window.hpp" +#include "xtc/xtc_version.h" +#include int main(int argc, char **argv) { + if (argc == 2 && !std::strcmp(argv[1], "--version")) { + std::printf("%s %s\n", XTC_PROJECT_NAME, XTC_VERSION_STRING); return EXIT_SUCCESS; + } + if (argc == 2 && !std::strcmp(argv[1], "--help")) { + std::puts("Usage: xtc [PROJECT] or xtc [FLTK options]\n--version and --help work without a display or audio server."); return EXIT_SUCCESS; + } + const char *project_path = nullptr; + if (argc == 2 && argv[1][0] != '-') { project_path = argv[1]; argc = 1; argv[1] = nullptr; } try { Fl::scheme("flat"); Fl::background(34, 36, 40); Fl::background2(24, 26, 30); Fl::foreground(230, 232, 236); MainWindow window; + if (project_path) { + xtc_result result = window.open_project(project_path); + if (result != XTC_OK) { std::fprintf(stderr, "Cannot open project: %s\n", xtc_result_string(result)); return EXIT_FAILURE; } + } window.show(argc, argv); return Fl::run(); } catch (const std::exception &error) { diff --git a/src/gui/main_window.cpp b/src/gui/main_window.cpp index 6025cda..b718ed7 100644 --- a/src/gui/main_window.cpp +++ b/src/gui/main_window.cpp @@ -384,7 +384,7 @@ void MainWindow::view_callback(Fl_Widget *widget, void *view) } void MainWindow::about_callback(Fl_Widget *, void *) { - fl_message("%s %s\nModular audio workstation\nPhase 8: stereo track mixer, automation, meters and undo/redo.", + fl_message("%s %s\nModular audio workstation\nPhase 9: native C processors and Linux release preparation.", XTC_PROJECT_NAME, XTC_VERSION_STRING); } void MainWindow::timer_callback(void *context) diff --git a/src/main.c b/src/main.c index 16dc3aa..f46fee3 100644 --- a/src/main.c +++ b/src/main.c @@ -3,9 +3,27 @@ #include #include +#include +#include "xtc/xtc_project.h" -int main(void) +int main(int argc, char **argv) { + if (argc == 2 && !strcmp(argv[1], "--version")) { + printf("%s %s\n", XTC_PROJECT_NAME, XTC_VERSION_STRING); return EXIT_SUCCESS; + } + if (argc == 2 && !strcmp(argv[1], "--help")) { + puts("Usage: xtc-check [--version | --help | --validate PROJECT]\nNo arguments checks core startup/shutdown. Validation checks metadata only."); return EXIT_SUCCESS; + } + if (argc == 3 && !strcmp(argv[1], "--validate")) { + xtc_project *project = NULL; + xtc_result result = xtc_project_create(&project); + if (result == XTC_OK) result = xtc_project_load(project, argv[2]); + xtc_project_destroy(project); + if (result != XTC_OK) { fprintf(stderr, "Project validation failed: %s\n", xtc_result_string(result)); return EXIT_FAILURE; } + puts("Project metadata valid (external assets not loaded)."); return EXIT_SUCCESS; + } + if (argc != 1) { fputs("Usage: xtc-check [--version | --help | --validate PROJECT]\n", stderr); return EXIT_FAILURE; } + xtc_app *app = NULL; xtc_result result = xtc_app_create(&app); diff --git a/src/native/README.md b/src/native/README.md new file mode 100644 index 0000000..93acbf6 --- /dev/null +++ b/src/native/README.md @@ -0,0 +1,5 @@ +# Native processor host + +`native.c` validates and copies ABI-1 descriptors, prepares/destroys control-thread +instances and validates bounded realtime blocks. User callback code is trusted. +See ../../docs/NATIVE_API.md for ownership, failures and graph/event integration. diff --git a/src/native/native.c b/src/native/native.c new file mode 100644 index 0000000..079d923 --- /dev/null +++ b/src/native/native.c @@ -0,0 +1,105 @@ +#include "xtc/xtc_native.h" +#include +#include +struct xtc_native { + xtc_native_descriptor descriptor; + void *state; + size_t max_frames; + uint64_t next_position; +}; +xtc_result xtc_native_validate(const xtc_native_descriptor *d) +{ + size_t i; + if (!d) + return XTC_ERR_INVALID_ARGUMENT; + if (d->abi_version != XTC_NATIVE_ABI_VERSION || d->struct_size != sizeof(*d)) + return XTC_ERR_UNSUPPORTED; + if (d->kind < XTC_NATIVE_EFFECT || d->kind > XTC_NATIVE_INSTRUMENT || !d->prepare || + !d->reset || !d->process || !d->destroy || !d->identifier[0]) + return XTC_ERR_INVALID_ARGUMENT; + for (i = 0; i < sizeof(d->identifier); ++i) { + unsigned char c = (unsigned char)d->identifier[i]; + if (!c) + return XTC_OK; + if (c < 33 || c > 126) + return XTC_ERR_INVALID_ARGUMENT; + } + return XTC_ERR_INVALID_ARGUMENT; +} +xtc_result xtc_native_create(const xtc_native_descriptor *d, double rate, size_t frames, + float value, xtc_native **out) +{ + xtc_native *p; + xtc_result r; + if (!out) + return XTC_ERR_INVALID_ARGUMENT; + *out = NULL; + r = xtc_native_validate(d); + if (r != XTC_OK) + return r; + if (!isfinite(rate) || rate <= 0 || rate > 768000 || !frames || frames > 65536 || + !isfinite(value)) + return XTC_ERR_INVALID_ARGUMENT; + p = calloc(1, sizeof(*p)); + if (!p) + return XTC_ERR_OUT_OF_MEMORY; + p->descriptor = *d; + p->max_frames = frames; + p->next_position = UINT64_MAX; + r = d->prepare(rate, frames, value, &p->state); + if (r != XTC_OK) { + xtc_native_destroy(p); + return r; + } + p->descriptor.reset(p->state); + *out = p; + return XTC_OK; +} +void xtc_native_destroy(xtc_native *p) +{ + if (p) { + p->descriptor.destroy(p->state); + free(p); + } +} +void xtc_native_reset(xtc_native *p) +{ + if (p) { + p->descriptor.reset(p->state); + p->next_position = UINT64_MAX; + } +} +xtc_result xtc_native_process(xtc_native *p, const xtc_native_block *b) +{ + size_t i; + xtc_result r; + if (!p || !b || !b->output || b->position > UINT64_MAX - b->frames || + b->event_count > XTC_MIDI_MAX_EVENTS || (b->event_count && !b->events) || + (b->event_count && p->descriptor.kind != XTC_NATIVE_INSTRUMENT)) + return XTC_ERR_INVALID_ARGUMENT; + if (b->frames > p->max_frames) + return XTC_ERR_CAPACITY; + for (i = 0; i < b->event_count; ++i) { + const xtc_midi_event *e = b->events + i; + if (e->offset >= b->frames || (i && e->offset < b->events[i - 1].offset) || + e->type < XTC_MIDI_NOTE_ON || e->type > XTC_MIDI_ALL_OFF || e->channel > 15 || + e->pitch > 127 || e->velocity > 127 || (e->type == XTC_MIDI_NOTE_ON && !e->velocity)) + return XTC_ERR_INVALID_ARGUMENT; + } + if (!b->frames) + return XTC_OK; + if (p->next_position != b->position) + p->descriptor.reset(p->state); + r = p->descriptor.process(p->state, b); + if (r != XTC_OK) { + for (i = 0; i < b->frames; ++i) + b->output[i] = 0; + xtc_native_reset(p); + return r; + } + for (i = 0; i < b->frames; ++i) + if (!isfinite(b->output[i])) + b->output[i] = 0; + p->next_position = b->position + b->frames; + return XTC_OK; +} diff --git a/src/persistence/project_file.c b/src/persistence/project_file.c index 25c056c..9b3cc6b 100644 --- a/src/persistence/project_file.c +++ b/src/persistence/project_file.c @@ -102,7 +102,7 @@ static xtc_result read_project(reader *input, xtc_project *project) if (value > 1U || next_id > SIZE_MAX || node_count > 64U || edge_count > node_count * node_count) return XTC_ERR_FORMAT; graph->require_output_nodes = (int)value; graph->next_id = (size_t)next_id; for (i = 0U; i < (size_t)node_count; ++i) { - xtc_node node; + xtc_node node = {0}; READ("NODE", 4U); NUM(1U, value); NUM(2U, version); if (value >= next_id || value >= SIZE_MAX || version < XTC_NODE_GAIN || version > (format_version == 1U ? XTC_NODE_OUTPUT : format_version == 2U ? XTC_NODE_AUDIO : XTC_NODE_INSTRUMENT) || !decode_float(input->tokens[3], &node.value)) return XTC_ERR_FORMAT; @@ -203,6 +203,8 @@ static xtc_result write_project(FILE *file, const xtc_project *project) { const xtc_graph *graph = project->graph; size_t i; + for (i = 0; i < graph->node_count; ++i) + if (graph->nodes[i].type == XTC_NODE_NATIVE) return XTC_ERR_UNSUPPORTED; if (sizeof(float) != sizeof(uint32_t) || FLT_RADIX != 2 || FLT_MANT_DIG != 24 || FLT_MAX_EXP != 128) return XTC_ERR_UNSUPPORTED; if (fprintf(file, "XTC_PROJECT 4\nAPP %d %d %d\nTEMPO %u\nIDS %" PRIu64 " %" PRIu64 "\nGRAPH %d %zu %zu %zu\n", XTC_VERSION_MAJOR, XTC_VERSION_MINOR, XTC_VERSION_PATCH, project->tempo, diff --git a/tests/integration/README.md b/tests/integration/README.md index 0f05d5d..2ddc19f 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -156,3 +156,29 @@ ASan/UBSan core/GUI suite passed without compiler warnings. GUI leak checking retains only the existing fontconfig suppression. ThreadSanitizer also passed the recorder concurrency and realtime publication tests. Physical devices and long-running stereo performance remain unverified. + +## Phase 9 release preparation + +```sh +make all test gui debug test-gui test-gui-jack test-jack test-jack-audio test-jack-midi +make test-install test-recovery test-performance +make test-release +``` + +Core test now includes native ABI/lifecycle/retirement and the native-gain example. +Optional release tests use Python 3.12+ standard library, Git for indexed source +exports, GNU/LLVM deterministic ar and temporary directories. No live prefix is +installed. test-install checks core/GUI files, a prefix containing spaces, +headless CLI, pkg-config C/C++ consumers, exact/idempotent uninstall and preservation +of unrelated files. test-release compares two clean exported optimized builds and +repacked archives byte-for-byte. test-recovery kills private child writers and +forces a child-local write limit, checking complete finalized destinations and +explicit orphan temporaries. Performance timings are reported, not used as a +universal realtime guarantee. See RELEASING.md, RECOVERY.md and PERFORMANCE.md. + +Phase 9 verification passed a fresh warning-enabled core/debug/GUI build, all +software JACK targets, staged install/uninstall, recovery and performance tests. +No-JACK core/GUI/install checks and Clang ASan/UBSan core/GUI checks also passed +(the GUI retains only its existing fontconfig leak suppression). TSan recorder +and realtime-handoff checks passed. Two independent optimized source exports +and their repacked source archives were byte-identical on this toolchain. diff --git a/tests/integration/test_install.py b/tests/integration/test_install.py new file mode 100644 index 0000000..afb4131 --- /dev/null +++ b/tests/integration/test_install.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Exercise core/GUI install and exact uninstall solely below a temporary DESTDIR.""" +import argparse +import os +from pathlib import Path +import shlex +import shutil +import subprocess +import tempfile + + +def run(args, **kwargs): + return subprocess.run(args, check=True, **kwargs) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--build-dir', type=Path, required=True) + parser.add_argument('--gui', action='store_true') + args = parser.parse_args() + source = Path.cwd() + with tempfile.TemporaryDirectory(prefix='xtc-install-') as directory: + stage = Path(directory) + prefix = '/opt/xtc test' + root = stage / prefix.lstrip('/') + sentinel = root / 'lib/keep-unrelated.txt' + sentinel.parent.mkdir(parents=True) + sentinel.write_text('keep\n') + make = shlex.split(os.environ.get('MAKE', 'make')) + # MAKEFLAGS jobserver descriptors do not cross Python subprocesses. + env = dict(os.environ, MAKEFLAGS='', MFLAGS='') + variables = [f'BUILD_DIR={args.build_dir}', f'PREFIX={prefix}', f'DESTDIR={stage}'] + run(make + ['install'] + variables, env=env) + assert not (root / 'bin/xtc').exists() + assert (root / 'share/licenses/xtc/LICENSE').read_bytes() == (source / 'LICENSE').read_bytes() + assert (root / 'share/doc/xtc/PROJECT_FORMAT.md').is_file() + assert os.access(root / 'bin/xtc-check', os.X_OK) + run([str(root / 'bin/xtc-check')]) + version = subprocess.check_output([str(root / 'bin/xtc-check'), '--version'], text=True) + assert version.startswith('xtc ') + valid = stage / 'valid.xtc' + valid.write_text('XTC_PROJECT 4\nAPP 0 1 0\nTEMPO 120000\nIDS 0 0\nGRAPH 1 0 0 0\nTRACKS 0\nCLIPS 0\nNOTES 0 0\nAUTOMATION 0\nEND\n') + run([str(root / 'bin/xtc-check'), '--validate', str(valid)]) + valid.write_text('damaged\n') + assert subprocess.run([str(root / 'bin/xtc-check'), '--validate', str(valid)], + stdout=subprocess.PIPE, stderr=subprocess.PIPE).returncode != 0 + pkg_env = dict(env, PKG_CONFIG_PATH=str(root / 'lib/pkgconfig') + ':' + env.get('PKG_CONFIG_PATH', ''), + PKG_CONFIG_SYSROOT_DIR=str(stage)) + pkg = shlex.split(os.environ.get('PKG_CONFIG', 'pkg-config')) + # A no-JACK test can be invoked with PKG_CONFIG=false; package consumer + # discovery itself still uses the installed pkg-config tool. + if pkg == ['false']: + pkg = ['pkg-config'] + flags = shlex.split(subprocess.check_output(pkg + ['--cflags', '--libs', '--static', 'xtc'], env=pkg_env, text=True)) + includes = ''.join(f'#include "xtc/{p.name}"\n' for p in sorted((source / 'include/xtc').glob('*.h'))) + consumer = stage / 'consumer.c' + consumer.write_text(includes + '\nint main(void) { xtc_project *p = 0; xtc_result r = xtc_project_create(&p); xtc_project_destroy(p); return r != XTC_OK; }\n') + for compiler, standard in [('CC', 'c99'), ('CXX', 'c++17')]: + output = stage / ('consumer-' + compiler) + command = shlex.split(env.get(compiler, 'cc' if compiler == 'CC' else 'c++')) + run(command + [f'-std={standard}', '-Wall', '-Wextra', '-Werror', str(consumer)] + flags + ['-o', str(output)]) + run([str(output)]) + if args.gui: + run(make + ['install-gui'] + variables, env=env) + gui_env = dict(env, DISPLAY='', WAYLAND_DISPLAY='') + assert subprocess.check_output([str(root / 'bin/xtc'), '--version'], env=gui_env, text=True) == version + run([str(root / 'bin/xtc'), '--help'], env=gui_env) + if shutil.which('desktop-file-validate'): + run(['desktop-file-validate', str(root / 'share/applications/xtc.desktop')]) + run(make + ['uninstall'] + variables, env=env) + run(make + ['uninstall'] + variables, env=env) + remaining = sorted(p.relative_to(root).as_posix() for p in root.rglob('*') if p.is_file() or p.is_symlink()) + assert remaining == ['lib/keep-unrelated.txt'], remaining + assert sentinel.read_text() == 'keep\n' + print('Staged install/uninstall, C/C++ pkg-config consumers and headless CLI passed.') + + +if __name__ == '__main__': + main() diff --git a/tests/integration/test_performance.c b/tests/integration/test_performance.c new file mode 100644 index 0000000..41a9a91 --- /dev/null +++ b/tests/integration/test_performance.c @@ -0,0 +1,110 @@ +#define _POSIX_C_SOURCE 200809L +/* Offline workload timing, not hardware or hard realtime qualification. */ +#include "xtc/xtc_midi.h" +#include "xtc/xtc_mixer.h" +#include +#include +#include +#include +#define REQUIRE(c) \ + do { \ + if (!(c)) { \ + fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #c); \ + goto cleanup; \ + } \ + } while (0) +static double now(void) +{ + struct timespec t; + clock_gettime(CLOCK_MONOTONIC, &t); + return (double)t.tv_sec + (double)t.tv_nsec / 1e9; +} +static int compare(const void *left, const void *right) +{ + double a = *(const double *)left, b = *(const double *)right; + return a < b ? -1 : a > b; +} +int main(int argc, char **argv) +{ + xtc_project *project = NULL; + xtc_graph *graph = NULL; + xtc_plan *plan = NULL; + xtc_midi_plan *midi = NULL; + long blocks = 2000; + char *end; + size_t i, j, source, previous, node; + uint64_t track, clip, note, future; + float left[128], right[128], out_left[128], out_right[128]; + double *times = NULL, total = 0, begin, prepare_ms; + int result = EXIT_FAILURE; + if (argc > 2) + return EXIT_FAILURE; + if (argc == 2) { + blocks = strtol(argv[1], &end, 10); + if (!*argv[1] || *end || blocks < 1 || blocks > 100000) + return EXIT_FAILURE; + } + times = calloc((size_t)blocks, sizeof(*times)); + REQUIRE(times); + begin = now(); + REQUIRE(xtc_project_create(&project) == XTC_OK); + for (i = 0; i < 16; ++i) { + xtc_automation_point point; + REQUIRE(xtc_project_add_track(project, "Performance", &track) == XTC_OK); + REQUIRE(xtc_project_add_midi_clip(project, track, 0, 384000, &clip) == XTC_OK); + REQUIRE(xtc_project_set_mix(project, track, 0.1f, i % 2 ? 0.5f : -0.5f, 0, 0) == XTC_OK); + for (j = 0; j < 2; ++j) + REQUIRE(xtc_project_add_note(project, clip, 0, 384000, 48 + (unsigned int)i, 100, 0, + ¬e) == XTC_OK); + point = (xtc_automation_point){track, 0, XTC_AUTOMATION_GAIN, 0.1f}; + REQUIRE(xtc_project_set_automation(project, &point) == XTC_OK); + point.tick = 384000; + point.value = 0.2f; + REQUIRE(xtc_project_set_automation(project, &point) == XTC_OK); + } + REQUIRE(xtc_project_add_midi_clip(project, track, 1000000, 10000, &future) == XTC_OK); + for (i = 32; i < XTC_PROJECT_MAX_NOTES; ++i) + REQUIRE(xtc_project_add_note(project, future, (i - 32) * 2, 1, 60, 100, 0, ¬e) == + XTC_OK); + REQUIRE(xtc_graph_create(64, &graph) == XTC_OK && + xtc_graph_add_instrument(graph, &source) == XTC_OK); + previous = source; + for (i = 0; i < 61; ++i) { + REQUIRE(xtc_graph_add_gain(graph, 0.99f, &node) == XTC_OK && + xtc_graph_connect(graph, previous, node) == XTC_OK); + previous = node; + } + REQUIRE(xtc_graph_add_meter(graph, &node) == XTC_OK && + xtc_graph_connect(graph, previous, node) == XTC_OK); + previous = node; + REQUIRE(xtc_graph_add_output(graph, &node) == XTC_OK && + xtc_graph_connect(graph, previous, node) == XTC_OK); + REQUIRE(xtc_graph_compile(graph, 48000, 128, &plan) == XTC_OK && + xtc_midi_plan_create(project, 48000, &midi) == XTC_OK); + prepare_ms = (now() - begin) * 1000; + for (i = 0; i < (size_t)blocks; ++i) { + begin = now(); + REQUIRE(xtc_midi_plan_render_stereo(midi, i * 128, NULL, 0, left, right, 128) == XTC_OK); + REQUIRE(xtc_plan_process_stereo(plan, NULL, NULL, NULL, left, right, out_left, out_right, + 128) == XTC_OK); + times[i] = now() - begin; + total += times[i]; + for (j = 0; j < 128; ++j) + REQUIRE(isfinite(out_left[j]) && isfinite(out_right[j])); + } + qsort(times, (size_t)blocks, sizeof(*times), compare); + printf("Offline 48kHz/128, 64 nodes, 4096 notes (32 active), 16 automated tracks; %ld blocks\n", + blocks); + printf("Prepare %.3f ms; block mean %.3f us, p99 %.3f us, max %.3f us; deadline 2666.667 us; " + "CPU/audio %.4f\n", + prepare_ms, total / (double)blocks * 1e6, times[(size_t)blocks * 99 / 100] * 1e6, + times[blocks - 1] * 1e6, total / ((double)blocks * 128 / 48000)); + result = EXIT_SUCCESS; +cleanup: + free(times); + xtc_midi_plan_destroy(midi); + xtc_plan_destroy(plan); + xtc_graph_destroy(graph); + xtc_project_destroy(project); + return result; +} diff --git a/tests/integration/test_recovery.c b/tests/integration/test_recovery.c new file mode 100644 index 0000000..3aaf51a --- /dev/null +++ b/tests/integration/test_recovery.c @@ -0,0 +1,153 @@ +#define _POSIX_C_SOURCE 200809L +#include "xtc/xtc_audio.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +static void reap(pid_t child) +{ + int status; + assert(kill(child, SIGKILL) == 0); + assert(waitpid(child, &status, 0) == child && WIFSIGNALED(status) && + WTERMSIG(status) == SIGKILL); +} +static size_t read_bytes(const char *path, unsigned char *bytes, size_t size) +{ + FILE *f = fopen(path, "rb"); + size_t n; + assert(f); + n = fread(bytes, 1, size, f); + assert(!ferror(f) && fclose(f) == 0); + return n; +} +static int temporaries(const char *directory, int remove_files) +{ + DIR *dir = opendir(directory); + struct dirent *entry; + int count = 0; + assert(dir); + while ((entry = readdir(dir)) != NULL) + if (strstr(entry->d_name, ".tmp.")) { + char path[512]; + ++count; + if (remove_files) { + assert(snprintf(path, sizeof(path), "%s/%s", directory, entry->d_name) > 0); + assert(unlink(path) == 0); + } + } + assert(closedir(dir) == 0); + return count; +} +int main(void) +{ + char directory[] = "/tmp/xtc-recovery-XXXXXX", song[256], wave[256]; + xtc_project *old = NULL, *next = NULL, *loaded = NULL; + uint64_t track, clip; + size_t i, n, before_size; + unsigned char before[4096], after[4096]; + pid_t child; + int status, pipefd[2]; + char ready; + struct timespec delay = {0, 100000}; + assert(mkdtemp(directory)); + snprintf(song, sizeof(song), "%s/song.xtc", directory); + snprintf(wave, sizeof(wave), "%s/take.wav", directory); + assert(xtc_project_create(&old) == XTC_OK && xtc_project_create(&next) == XTC_OK && + xtc_project_create(&loaded) == XTC_OK); + assert(xtc_project_add_track(old, "Old", &track) == XTC_OK && + xtc_project_save(old, song) == XTC_OK); + before_size = read_bytes(song, before, sizeof(before)); + assert(xtc_project_add_track(next, "New", &track) == XTC_OK); + for (i = 0; i < XTC_PROJECT_MAX_CLIPS; ++i) + assert(xtc_project_add_clip(next, track, "Recovery region", i * 960, 960, &clip) == XTC_OK); + // Deterministic per-process file-size failure; no server/system setting change. + child = fork(); + assert(child >= 0); + if (!child) { + struct rlimit limit = {128, 128}; + signal(SIGXFSZ, SIG_IGN); + if (setrlimit(RLIMIT_FSIZE, &limit) != 0) + _exit(2); + _exit(xtc_project_save(next, song) == XTC_ERR_IO ? 0 : 3); + } + assert(waitpid(child, &status, 0) == child && WIFEXITED(status) && WEXITSTATUS(status) == 0); + assert(read_bytes(song, after, sizeof(after)) == before_size && + memcmp(before, after, before_size) == 0 && temporaries(directory, 0) == 0); + // Kill a writer while its temporary is visible. Destination must be complete old/new. + for (i = 0; i < 8; ++i) { + size_t attempts; + xtc_track_info info; + assert(xtc_project_save(old, song) == XTC_OK); + assert(pipe(pipefd) == 0); + child = fork(); + assert(child >= 0); + if (!child) { + close(pipefd[0]); + if (write(pipefd[1], "R", 1) != 1) + _exit(4); + close(pipefd[1]); + for (;;) + if (xtc_project_save(next, song) != XTC_OK) + _exit(5); + } + close(pipefd[1]); + assert(read(pipefd[0], &ready, 1) == 1); + close(pipefd[0]); + for (attempts = 0; attempts < 10000 && !temporaries(directory, 0); ++attempts) + nanosleep(&delay, NULL); + reap(child); + assert(attempts < 10000); + assert(xtc_project_load(loaded, song) == XTC_OK && + xtc_project_get_track(loaded, 0, &info) == XTC_OK); + assert(strcmp(info.name, "Old") == 0 || strcmp(info.name, "New") == 0); + temporaries(directory, 1); + } + // A killed recording leaves the previously finalized destination untouched. + { + xtc_recorder *recorder = NULL; + float samples[128]; + for (i = 0; i < 128; ++i) + samples[i] = 0.25f; + assert(xtc_recorder_create(wave, 48000, 256, &recorder) == XTC_OK); + xtc_recorder_push(recorder, samples, 128); + assert(xtc_recorder_finish(recorder) == XTC_OK); + xtc_recorder_destroy(recorder); + before_size = read_bytes(wave, before, sizeof(before)); + assert(pipe(pipefd) == 0); + child = fork(); + assert(child >= 0); + if (!child) { + close(pipefd[0]); + if (xtc_recorder_create(wave, 48000, 256, &recorder) != XTC_OK) + _exit(6); + xtc_recorder_push(recorder, samples, 128); + if (xtc_recorder_drain(recorder) != XTC_OK) + _exit(7); + if (write(pipefd[1], "R", 1) != 1) + _exit(8); + for (;;) + pause(); + } + close(pipefd[1]); + assert(read(pipefd[0], &ready, 1) == 1); + close(pipefd[0]); + reap(child); + n = read_bytes(wave, after, sizeof(after)); + assert(n == before_size && memcmp(before, after, n) == 0); + assert(temporaries(directory, 1) > 0); + } + xtc_project_destroy(old); + xtc_project_destroy(next); + xtc_project_destroy(loaded); + assert(unlink(song) == 0 && unlink(wave) == 0 && rmdir(directory) == 0); + puts("Save write failure, 8 killed saves and killed recording preserve complete destinations; " + "orphan temporaries remain explicit."); + return 0; +} diff --git a/tests/integration/test_release.py b/tests/integration/test_release.py new file mode 100644 index 0000000..2217178 --- /dev/null +++ b/tests/integration/test_release.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""Rebuild source exports in different paths and compare release artifacts.""" +import argparse +import hashlib +import os +from pathlib import Path +import shlex +import subprocess +import tarfile +import tempfile + + +def digest(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('archive', type=Path) + parser.add_argument('--gui', action='store_true') + args = parser.parse_args() + archive = args.archive.resolve() + with tempfile.TemporaryDirectory(prefix='xtc-release-') as directory: + root = Path(directory) + with tarfile.open(archive) as source: + names = [m.name for m in source.getmembers()] + assert len(set(names)) == len(names) + assert all(m.isfile() for m in source.getmembers()) + top = names[0].split('/')[0] + epoch = source.getmembers()[0].mtime + assert all(n.startswith(top + '/') and '..' not in Path(n).parts for n in names) + assert all('__pycache__' not in Path(n).parts and '.git' not in Path(n).parts for n in names) + for name in ('first', 'different-path'): + source.extractall(root / name, filter='data') + env = dict(os.environ, SOURCE_DATE_EPOCH=str(epoch), MAKEFLAGS='', MFLAGS='') + builds = [] + for name in ('first', 'different-path'): + project = root / name / top + log = root / (name + '.log') + make = shlex.split(env.get('MAKE', 'make')) + with log.open('w') as output: + try: + subprocess.run(make + ['release'], cwd=project, env=env, stdout=output, + stderr=subprocess.STDOUT, check=True) + except subprocess.CalledProcessError: + print(log.read_text()[-12000:]) + raise + # Exported source can package itself without a .git directory. + repacked = root / (name + '.tar.gz') + subprocess.run([env.get('PYTHON', 'python3'), 'tools/dist.py', '--output', str(repacked)], + cwd=project, env=env, check=True) + assert digest(repacked) == digest(archive), 'Source archive is not reproducible' + release = project / 'build/release' + files = ['libxtc.a', 'xtc-check'] + (['xtc'] if args.gui else []) + builds.append({file: digest(release / file) for file in files}) + assert builds[0] == builds[1], (builds[0], builds[1]) + for file, sha in builds[0].items(): + print(f'Reproducible {file}: {sha}') + print('Two clean source-export release builds and archives are byte-identical.') + + +if __name__ == '__main__': + main() diff --git a/tests/unit/test_native.c b/tests/unit/test_native.c new file mode 100644 index 0000000..5d787a5 --- /dev/null +++ b/tests/unit/test_native.c @@ -0,0 +1,229 @@ +#define _POSIX_C_SOURCE 200809L +#include "xtc/xtc_native.h" +#include "xtc/xtc_realtime.h" +#include +#include +#include +#include +#include +#include +typedef struct { + float gain; + int held; +} state; +static unsigned int prepared, destroyed, resets, fail_prepare; +static int fail_process, nonfinite; +static xtc_result prepare(double rate, size_t frames, float gain, void **out) +{ + state *s = calloc(1, sizeof(*s)); + assert(rate == 48000 && frames == 64); + if (!s) + return XTC_ERR_OUT_OF_MEMORY; + s->gain = gain; + *out = s; + ++prepared; + return prepared == fail_prepare ? XTC_ERR_IO : XTC_OK; +} +static void reset(void *context) +{ + state *s = context; + if (s) + s->held = 0; + ++resets; +} +static void destroy(void *context) +{ + if (context) + ++destroyed; + free(context); +} +static xtc_result effect(void *context, const xtc_native_block *b) +{ + const state *s = context; + size_t i; + for (i = 0; i < b->frames; ++i) + b->output[i] = nonfinite ? NAN : (b->input ? b->input[i] : 0) * s->gain; + return fail_process ? XTC_ERR_IO : XTC_OK; +} +static xtc_result instrument(void *context, const xtc_native_block *b) +{ + state *s = context; + size_t i, event = 0; + for (i = 0; i < b->frames; ++i) { + while (event < b->event_count && b->events[event].offset == i) { + s->held = b->events[event].type == XTC_MIDI_NOTE_ON; + ++event; + } + b->output[i] = s->held ? s->gain : 0; + } + return XTC_OK; +} +static xtc_native_descriptor descriptor(void) +{ + xtc_native_descriptor d = {XTC_NATIVE_ABI_VERSION, + sizeof(d), + "org.xtc.test-gain", + XTC_NATIVE_EFFECT, + prepare, + reset, + effect, + destroy}; + return d; +} +int main(void) +{ + xtc_native_descriptor d = descriptor(), bad; + xtc_native *processor = NULL; + float input[64], out[64], right[64]; + size_t i, node, output, nodes, edges; + xtc_native_block block = {input, out, 64, 0, NULL, 0}; + xtc_graph *graph = NULL, *copy = NULL; + xtc_plan *plan = NULL; + xtc_realtime *engine = NULL; + xtc_project *project = NULL; + xtc_port_info port; + xtc_graph_edit edit = {0}; + char dir[] = "/tmp/xtc-native-XXXXXX", path[256], sentinel[8] = {0}; + FILE *f; + assert(mkdtemp(dir)); + snprintf(path, sizeof(path), "%s/song.xtc", dir); + for (i = 0; i < 64; ++i) + input[i] = 0.25f; + bad = d; + bad.abi_version = 99; + assert(xtc_native_create(&bad, 48000, 64, 1, &processor) == XTC_ERR_UNSUPPORTED && !processor); + bad = d; + --bad.struct_size; + assert(xtc_native_validate(&bad) == XTC_ERR_UNSUPPORTED); + bad = d; + bad.process = NULL; + assert(xtc_native_validate(&bad) == XTC_ERR_INVALID_ARGUMENT); + bad = d; + memset(bad.identifier, 'x', sizeof(bad.identifier)); + assert(xtc_native_validate(&bad) == XTC_ERR_INVALID_ARGUMENT); + assert(xtc_native_create(&d, NAN, 64, 1, &processor) == XTC_ERR_INVALID_ARGUMENT); + fail_prepare = prepared + 1; + assert(xtc_native_create(&d, 48000, 64, 1, &processor) == XTC_ERR_IO && !processor && + destroyed == prepared); + fail_prepare = 0; + assert(xtc_native_create(&d, 48000, 64, 0.5f, &processor) == XTC_OK); + d.process = instrument; // descriptor was copied, not borrowed. + assert(xtc_native_process(processor, &block) == XTC_OK && out[0] == 0.125f); + block.frames = 65; + out[0] = 17; + assert(xtc_native_process(processor, &block) == XTC_ERR_CAPACITY && out[0] == 17); + block.frames = 64; + fail_process = 1; + assert(xtc_native_process(processor, &block) == XTC_ERR_IO && out[0] == 0 && out[63] == 0); + fail_process = 0; + nonfinite = 1; + assert(xtc_native_process(processor, &block) == XTC_OK && out[1] == 0); + nonfinite = 0; + block.input = input; + block.output = input; + assert(xtc_native_process(processor, &block) == XTC_OK && input[0] == 0.125f); + xtc_native_destroy(processor); + processor = NULL; + d = descriptor(); + assert(xtc_graph_create(2, &graph) == XTC_OK && + xtc_graph_add_native(graph, &d, 2, &node) == XTC_OK); + assert(xtc_graph_get_native(graph, node, &bad) == XTC_OK && + strcmp(bad.identifier, d.identifier) == 0); + assert(xtc_graph_add_output(graph, &output) == XTC_OK && + xtc_graph_connect(graph, node, output) == XTC_OK); + assert(xtc_graph_get_port(graph, node, 0, &port) == XTC_OK); + // Partial compile destroys every prepared state on the control thread. + fail_prepare = prepared + 2; + assert(xtc_graph_compile(graph, 48000, 64, &plan) == XTC_ERR_IO && !plan && + destroyed == prepared); + fail_prepare = 0; + assert(xtc_graph_compile(graph, 48000, 64, &plan) == XTC_OK); + assert(xtc_plan_process_stereo(plan, input, NULL, NULL, NULL, NULL, out, right, 64) == XTC_OK && + out[0] == 0.25f && right[0] == 0.25f); + fail_process = 1; + assert(xtc_plan_process_stereo(plan, input, NULL, NULL, NULL, NULL, out, right, 64) == + XTC_ERR_IO); + assert(out[0] == 0 && right[0] == 0 && out[63] == 0 && right[63] == 0); + fail_process = 0; + assert(xtc_plan_reset(plan) == XTC_OK); + assert(xtc_plan_process_stereo(plan, input, NULL, NULL, NULL, NULL, out, right, 64) == XTC_OK && + out[0] == 0.25f); + xtc_plan_destroy(plan); + plan = NULL; + assert(xtc_project_create(&project) == XTC_OK && + xtc_project_set_graph(project, graph) == XTC_OK); + f = fopen(path, "wb"); + assert(f && fwrite("retain", 1, 6, f) == 6 && fclose(f) == 0); + assert(xtc_project_save(project, path) == XTC_ERR_UNSUPPORTED); + f = fopen(path, "rb"); + assert(f && fread(sentinel, 1, 8, f) == 6 && fclose(f) == 0 && !strcmp(sentinel, "retain")); + xtc_project_destroy(project); + edit.kind = XTC_EDIT_REMOVE_NODE; + edit.data.node_id = output; + assert(xtc_graph_apply_edit(graph, &edit, ©, NULL) == XTC_OK); + xtc_graph_destroy(graph); + graph = copy; + assert(xtc_graph_get_counts(graph, &nodes, &edges) == XTC_OK && nodes == 1 && edges == 0); + assert(xtc_realtime_create(48000, 64, &engine) == XTC_OK && + xtc_realtime_submit(engine, graph) == XTC_OK); + i = destroyed; + assert(xtc_realtime_process(engine, input, out, 64) == XTC_OK && out[0] == 0.25f && + destroyed == i); + assert(xtc_realtime_submit(engine, graph) == XTC_OK); + assert(xtc_realtime_process(engine, input, out, 64) == XTC_OK && destroyed == i); + xtc_realtime_collect(engine); + assert(destroyed == i + 2); + xtc_realtime_destroy(engine); + xtc_graph_destroy(graph); + d.kind = XTC_NATIVE_SOURCE; + assert(xtc_graph_create(1, &graph) == XTC_OK && + xtc_graph_add_native(graph, &d, 1, &node) == XTC_OK); + assert(xtc_graph_get_port(graph, node, 0, &port) == XTC_ERR_NOT_FOUND); + assert(xtc_graph_compile(graph, 48000, 64, &plan) == XTC_OK); + assert(xtc_plan_process(plan, input, out, 64) == XTC_OK && out[0] == 0 && out[63] == 0); + xtc_plan_destroy(plan); + + xtc_graph_destroy(graph); + // An event instrument renders a source buffer that routes through the graph. + d.kind = XTC_NATIVE_INSTRUMENT; + d.process = instrument; + assert(xtc_native_create(&d, 48000, 64, 0.2f, &processor) == XTC_OK); + { + xtc_midi_event events[] = {{4, XTC_MIDI_NOTE_ON, 0, 60, 100}, + {8, XTC_MIDI_NOTE_OFF, 0, 60, 0}}; + block = (xtc_native_block){NULL, out, 64, 0, events, 2}; + assert(xtc_native_process(processor, &block) == XTC_OK && out[3] == 0 && out[4] == 0.2f && + out[7] == 0.2f && out[8] == 0); + events[1].offset = 2; + out[0] = 17; + i = resets; + assert(xtc_native_process(processor, &block) == XTC_ERR_INVALID_ARGUMENT && out[0] == 17 && + resets == i); + block.event_count = 1; + block.position = 64; + assert(xtc_native_process(processor, &block) == XTC_OK && out[63] == 0.2f); + block.event_count = 0; + block.position = 512; + assert(xtc_native_process(processor, &block) == XTC_OK && + out[63] == 0); // seek resets held notes + assert(xtc_graph_create(2, &graph) == XTC_OK); + assert(xtc_graph_add_native(graph, &d, 1, &node) == XTC_ERR_UNSUPPORTED); + assert(xtc_graph_add_instrument(graph, &node) == XTC_OK && + xtc_graph_add_output(graph, &output) == XTC_OK && + xtc_graph_connect(graph, node, output) == XTC_OK); + block.position = 0; + block.event_count = 2; + events[1].offset = 8; + assert(xtc_native_process(processor, &block) == XTC_OK); + assert(xtc_graph_compile(graph, 48000, 64, &plan) == XTC_OK && + xtc_plan_process_sources(plan, NULL, NULL, out, right, 64) == XTC_OK && + memcmp(out, right, sizeof(out)) == 0); + xtc_plan_destroy(plan); + xtc_graph_destroy(graph); + } + xtc_native_destroy(processor); + assert(prepared == destroyed); + assert(unlink(path) == 0 && rmdir(dir) == 0); + puts("Native ABI/lifecycle/failure cleanup/graph retirement/instrument events passed."); + return 0; +} diff --git a/tools/dist.py b/tools/dist.py new file mode 100644 index 0000000..96e70ea --- /dev/null +++ b/tools/dist.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +"""Deterministic source archive using indexed files, or an exported manifest.""" +import argparse +import gzip +import io +import json +import os +from pathlib import Path, PurePosixPath +import re +import subprocess +import tarfile +import tempfile + + +def version(root): + text = (root / 'include/xtc/xtc_version.h').read_text() + match = re.search(r'^#define XTC_VERSION_STRING "([0-9]+\.[0-9]+\.[0-9]+)"$', text, re.M) + if not match: + raise ValueError('Invalid project version') + return match.group(1) + + +def manifest(root): + if (root / '.git').exists(): + data = subprocess.check_output(['git', 'ls-files', '--stage', '-z'], cwd=root) + entries = [] + for record in data.split(b'\0'): + if not record: + continue + metadata, name = record.split(b'\t', 1) + mode, _, stage = metadata.split() + if stage != b'0' or mode not in (b'100644', b'100755'): + raise ValueError('Only resolved regular indexed files can be packaged') + entries.append([name.decode('utf-8'), mode == b'100755']) + else: + entries = json.loads((root / 'SOURCE_MANIFEST.json').read_text()) + seen = set() + for name, executable in entries: + path = PurePosixPath(name) + if (not name or path.is_absolute() or '..' in path.parts or str(path) != name + or name in seen or name == 'SOURCE_MANIFEST.json' or not isinstance(executable, bool)): + raise ValueError('Invalid source manifest entry') + seen.add(name) + if (root / name).is_symlink() or not (root / name).is_file(): + raise ValueError(f'Not a regular source file: {name}') + return sorted(entries) + + +def archive(root, output): + entries = manifest(root) + epoch = int(os.environ.get('SOURCE_DATE_EPOCH', '0')) + if epoch < 0 or epoch > 4294967295: + raise ValueError('SOURCE_DATE_EPOCH must fit an unsigned 32-bit timestamp') + top = 'xtc-' + version(root) + output.parent.mkdir(parents=True, exist_ok=True) + fd, temporary = tempfile.mkstemp(prefix='.xtc-dist-', dir=output.parent) + try: + with os.fdopen(fd, 'wb') as raw, gzip.GzipFile(filename='', mode='wb', fileobj=raw, + mtime=epoch, compresslevel=9) as compressed: + with tarfile.open(fileobj=compressed, mode='w', format=tarfile.PAX_FORMAT) as tar: + contents = [(name, (root / name).read_bytes(), executable) for name, executable in entries] + contents.append(('SOURCE_MANIFEST.json', (json.dumps(entries, indent=2) + '\n').encode(), False)) + for name, data, executable in sorted(contents): + info = tarfile.TarInfo(top + '/' + name) + info.size = len(data) + info.mode = 0o755 if executable else 0o644 + info.mtime = epoch + tar.addfile(info, io.BytesIO(data)) + os.replace(temporary, output) + finally: + if os.path.exists(temporary): + os.unlink(temporary) + print(output) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--root', type=Path, default=Path.cwd()) + parser.add_argument('--output', type=Path, required=True) + args = parser.parse_args() + archive(args.root.resolve(), args.output.resolve()) diff --git a/tools/pkgconfig.py b/tools/pkgconfig.py new file mode 100644 index 0000000..bfe23c4 --- /dev/null +++ b/tools/pkgconfig.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 +"""Generate pkg-config metadata without shell/sed interpolation of paths.""" +import argparse +from pathlib import Path +from dist import version +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('--prefix', required=True) +parser.add_argument('--jack', default='') +parser.add_argument('--output', type=Path, required=True) +args = parser.parse_args() +if not args.prefix.startswith('/') or any(c in args.prefix for c in '\n\r\0"$\\'): + parser.error('PREFIX must be absolute, without newlines, quotes, $, or backslashes') +text = Path('packaging/xtc.pc.in').read_text().replace('@PREFIX@', args.prefix) +text = text.replace('@VERSION@', version(Path.cwd())).replace('@JACK@', args.jack) +args.output.parent.mkdir(parents=True, exist_ok=True) +args.output.write_text(text) diff --git a/tools/uninstall.py b/tools/uninstall.py new file mode 100644 index 0000000..43cebd6 --- /dev/null +++ b/tools/uninstall.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +"""Remove only this source version's installed files; never recurse directories.""" +import argparse +from pathlib import Path +import stat + + +def owned_files(source): + files = ['lib/libxtc.a', 'lib/pkgconfig/xtc.pc', 'bin/xtc-check', 'bin/xtc', + 'share/applications/xtc.desktop', 'share/licenses/xtc/LICENSE', + 'share/doc/xtc/README.md'] + files += ['include/xtc/' + p.name for p in sorted((source / 'include/xtc').glob('*.h'))] + files += ['share/doc/xtc/' + p.name for p in sorted((source / 'docs').glob('*.md'))] + files += ['share/doc/xtc/decisions/' + p.name for p in sorted((source / 'docs/decisions').glob('*.md'))] + return files + + +def uninstall(root, source): + if not root.is_absolute() or root == Path('/'): + raise ValueError('Installation root must be absolute and not /') + files = [root / name for name in owned_files(source)] + # Preflight before changing anything; never unlink a directory at a file path. + for path in files: + try: + mode = path.lstat().st_mode + except FileNotFoundError: + continue + if not (stat.S_ISREG(mode) or stat.S_ISLNK(mode)): + raise ValueError(f'Refusing non-file: {path}') + for path in files: + path.unlink(missing_ok=True) + parents = {parent for path in files for parent in path.parents if parent == root or root in parent.parents} + for parent in sorted(parents, key=lambda p: len(p.parts), reverse=True): + try: + parent.rmdir() + except (FileNotFoundError, OSError): + pass # Shared/nonempty directories remain untouched. + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--root', type=Path, required=True) + args = parser.parse_args() + uninstall(args.root, Path(__file__).resolve().parent.parent)