"""dataset_manifest.py: capture a dataset arm's content digest (IMPLEMENTATION.md B; PLAN-v4 3.5 S9).

    python dataset_manifest.py <arm_dir> <catalog_json> --label at_creation|captured_post_hoc [--workers 16] [--verify]

Scope = the catalog json + every file it references (string values under its "paths" key, plus any other string
value that resolves to an existing path, relative to the arm dir) + every file under the arm dir.  Directory
symlinks are traversed once (a directory whose realpath was already walked is skipped, which also makes cycles
safe).  Entries are [path, type, sha256, size]: path = canonical relative path from the arm dir (posix, no leading
./; "../x" for referenced files outside it), type = "file" | "symlink-><readlink target>", sha256 = content hash
of the resolved file, size = resolved size.  A directory symlink gets an entry with sha256 = null and size = null so
the digest also pins where the arm's directories point.  digest = sha256 over the sorted lines
"<sha256 or ->  <size or ->  <type>  <path>".  Output `<arm_dir>/dataset_manifest.json` = {digest, captured
(UTC ISO), label ("at_creation" | "captured_post_hoc <date>"), arm_dir, catalog, n_entries, bytes, entry_fields,
entries}; written only if absent (an existing file is reported, never overwritten; --verify recomputes and compares
its digest, exit 1 on mismatch).  Hashing is parallel (ProcessPoolExecutor, like the frozen s1_manifest.py inputs).
"""
import argparse, datetime, hashlib, json, os, sys
from concurrent.futures import ProcessPoolExecutor


def sha256_file(path):
    h = hashlib.sha256()
    with open(path, 'rb') as f:
        for b in iter(lambda: f.read(1 << 22), b''):
            h.update(b)
    return h.hexdigest()


def catalog_refs(catalog_path, arm_dir):
    """Paths referenced by the catalog: every string value (recursively) that resolves to an existing path."""
    try:
        cat = json.load(open(catalog_path))
    except (OSError, ValueError) as e:
        sys.exit(f'catalog {catalog_path}: {e}')
    refs = []

    def walk(v):
        if isinstance(v, dict):
            for x in v.values():
                walk(x)
        elif isinstance(v, list):
            for x in v:
                walk(x)
        elif isinstance(v, str) and v and not v.startswith(('http://', 'https://', 's3://')):
            for base in (os.path.dirname(os.path.realpath(catalog_path)), arm_dir):
                p = v if os.path.isabs(v) else os.path.join(base, v)
                if os.path.lexists(p):
                    refs.append(p); break
    walk(cat)
    return refs


def collect(arm_dir, catalog_path):
    arm_dir = os.path.abspath(arm_dir)
    seen_dirs = set()
    items = {}  # relpath -> (type, resolved path or None)

    def add_file(p):
        rel = os.path.relpath(p, arm_dir).replace(os.sep, '/')
        if rel in items:
            return
        if os.path.islink(p):
            tgt = os.readlink(p)
            items[rel] = (f'symlink->{tgt}', os.path.realpath(p) if os.path.isfile(p) else None)
        else:
            items[rel] = ('file', p)

    def walk_dir(d):
        rp = os.path.realpath(d)
        if rp in seen_dirs:
            return
        seen_dirs.add(rp)
        try:
            names = sorted(os.listdir(d))
        except OSError as e:
            print(f'warning: {d}: {e}', file=sys.stderr); return
        for n in names:
            p = os.path.join(d, n)
            if os.path.realpath(d) == os.path.realpath(arm_dir) and n == 'dataset_manifest.json':
                continue  # the tool's own output is never part of the digest (otherwise --verify can never match; found 2026-11-06)
            if os.path.isdir(p):
                if os.path.islink(p):
                    rel = os.path.relpath(p, arm_dir).replace(os.sep, '/')
                    items.setdefault(rel, (f'symlink->{os.readlink(p)}', None))
                walk_dir(p)
            else:
                add_file(p)

    add_file(catalog_path if os.path.isabs(catalog_path) else os.path.abspath(catalog_path))
    for ref in catalog_refs(catalog_path, arm_dir):
        if os.path.isdir(ref):
            walk_dir(ref)
        else:
            add_file(ref)
    walk_dir(arm_dir)
    return items


def build(arm_dir, catalog_path, workers):
    items = collect(arm_dir, catalog_path)
    rels = sorted(items)
    to_hash = [(r, items[r][1]) for r in rels if items[r][1] is not None]
    with ProcessPoolExecutor(workers) as pool:
        hashes = list(pool.map(sha256_file, [p for _, p in to_hash], chunksize=256))
    sha = dict(zip([r for r, _ in to_hash], hashes))
    entries, total = [], 0
    for r in rels:
        typ, resolved = items[r]
        size = os.path.getsize(resolved) if resolved is not None else None
        total += size or 0
        entries.append([r, typ, sha.get(r), size])
    lines = [f'{e[2] or "-"}  {e[3] if e[3] is not None else "-"}  {e[1]}  {e[0]}' for e in entries]
    digest = hashlib.sha256('\n'.join(lines).encode()).hexdigest()
    return digest, entries, total


def main():
    ap = argparse.ArgumentParser(description=__doc__.split('\n\n')[0])
    ap.add_argument('arm_dir'); ap.add_argument('catalog_json')
    ap.add_argument('--label', choices=('at_creation', 'captured_post_hoc'), required=True)
    ap.add_argument('--workers', type=int, default=16)
    ap.add_argument('--verify', action='store_true', help='if the manifest exists, recompute and compare the digest')
    a = ap.parse_args()
    arm_dir = os.path.abspath(a.arm_dir)
    out = os.path.join(arm_dir, 'dataset_manifest.json')
    if os.path.exists(out):
        with open(out) as fh:
            head = fh.read(2048)
        print(f'{out} exists; not overwritten. head: {head[:300]!r}')
        if a.verify:
            digest, _, _ = build(arm_dir, a.catalog_json, a.workers)
            old = json.load(open(out))['digest']
            print(f'recomputed {digest} vs stored {old}: {"MATCH" if digest == old else "MISMATCH"}')
            sys.exit(0 if digest == old else 1)
        return
    started = datetime.datetime.now(datetime.timezone.utc)
    digest, entries, total = build(arm_dir, a.catalog_json, a.workers)
    label = 'at_creation' if a.label == 'at_creation' else f'captured_post_hoc {started.date().isoformat()}'
    doc = {'digest': digest, 'captured': started.isoformat(timespec='seconds'), 'label': label, 'arm_dir': arm_dir,
           'catalog': os.path.relpath(os.path.abspath(a.catalog_json), arm_dir).replace(os.sep, '/'),
           'n_entries': len(entries), 'bytes': total, 'entry_fields': ['path', 'type', 'sha256', 'size'],
           'entries': entries}
    tmp = f'{out}.tmp.{os.getpid()}'
    with open(tmp, 'w') as fh:
        json.dump(doc, fh, separators=(',', ':'))
        fh.flush(); os.fsync(fh.fileno())
    try:
        os.link(tmp, out)  # create-once: EEXIST if someone wrote it meanwhile
    except FileExistsError:
        os.unlink(tmp)
        sys.exit(f'{out} appeared while hashing; not overwritten')
    os.unlink(tmp)
    secs = (datetime.datetime.now(datetime.timezone.utc) - started).total_seconds()
    print(json.dumps({k: v for k, v in doc.items() if k != 'entries'}) + f' seconds={secs:.1f}')


if __name__ == '__main__':
    main()
