xtc/tools/dist.py

#!/usr/bin/env python3
"""Deterministic source archive using indexed files, or an exported manifest."""
import argparse
import gzip
import hashlib
import io
import json
import os
from pathlib import Path, PurePosixPath
import re
import subprocess
import tarfile
import tempfile


def version(root):
    text = (root / 'include/xtc/xtc_version.h').read_text()
    match = re.search(r'^#define XTC_VERSION_STRING "([0-9]+\.[0-9]+\.[0-9]+)"$', text, re.M)
    if not match:
        raise ValueError('Invalid project version')
    return match.group(1)


def manifest(root):
    if (root / '.git').exists():
        data = subprocess.check_output(['git', 'ls-files', '--stage', '-z'], cwd=root)
        entries = []
        for record in data.split(b'\0'):
            if not record:
                continue
            metadata, name = record.split(b'\t', 1)
            mode, _, stage = metadata.split()
            if stage != b'0' or mode not in (b'100644', b'100755'):
                raise ValueError('Only resolved regular indexed files can be packaged')
            entries.append([name.decode('utf-8'), mode == b'100755'])
    else:
        entries = json.loads((root / 'SOURCE_MANIFEST.json').read_text())
    seen = set()
    for name, executable in entries:
        path = PurePosixPath(name)
        if (not name or path.is_absolute() or '..' in path.parts or str(path) != name
                or name in seen or name == 'SOURCE_MANIFEST.json' or not isinstance(executable, bool)):
            raise ValueError('Invalid source manifest entry')
        seen.add(name)
        if (root / name).is_symlink() or not (root / name).is_file():
            raise ValueError(f'Not a regular source file: {name}')
    return sorted(entries)


def archive(root, output):
    entries = manifest(root)
    epoch = int(os.environ.get('SOURCE_DATE_EPOCH', '0'))
    if epoch < 0 or epoch > 4294967295:
        raise ValueError('SOURCE_DATE_EPOCH must fit an unsigned 32-bit timestamp')
    top = 'xtc-' + version(root)
    output.parent.mkdir(parents=True, exist_ok=True)
    fd, temporary = tempfile.mkstemp(prefix='.xtc-dist-', dir=output.parent)
    try:
        with os.fdopen(fd, 'wb') as raw, gzip.GzipFile(filename='', mode='wb', fileobj=raw,
                                                       mtime=epoch, compresslevel=9) as compressed:
            with tarfile.open(fileobj=compressed, mode='w', format=tarfile.PAX_FORMAT) as tar:
                contents = [(name, (root / name).read_bytes(), executable) for name, executable in entries]
                contents.append(('SOURCE_MANIFEST.json', (json.dumps(entries, indent=2) + '\n').encode(), False))
                for name, data, executable in sorted(contents):
                    info = tarfile.TarInfo(top + '/' + name)
                    info.size = len(data)
                    info.mode = 0o755 if executable else 0o644
                    info.mtime = epoch
                    tar.addfile(info, io.BytesIO(data))
        os.replace(temporary, output)
    finally:
        if os.path.exists(temporary):
            os.unlink(temporary)
    sha = hashlib.sha256(output.read_bytes()).hexdigest()
    output.with_name(output.name + '.sha256').write_text(f'{sha}  {output.name}\n')
    print(f'{output}: {sha}')


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--root', type=Path, default=Path.cwd())
    parser.add_argument('--output', type=Path, required=True)
    args = parser.parse_args()
    archive(args.root.resolve(), args.output.resolve())