#!/usr/bin/env python3
"""assemble_arm.py: rebuild one PHerc0332 fit dataset arm from the published data, by content hash.

    python3 assemble_arm.py --data DATA --manifest arms/<arm>/dataset_manifest.json --root ROOT [--workers 8]

DATA     the downloaded dataset directory (this ash directory: loose files at their published paths, the published
         .tar.zst archives NOT extracted, repro/members.sha256 and repro/delta/*.members.sha256 present).
ROOT     an empty directory. The arm is rebuilt as a mirror of the original absolute layout under ROOT: the arm
         dataset_tta_... lands at ROOT/media/djosey/nvme4/spiral_0332/<arm>, and every absolute symlink target T of
         the original arm becomes a RELATIVE link to ROOT/T (the documented relocation). Nothing outside ROOT is read
         except DATA; nothing outside ROOT is written.

Rules (repro/README.md, PLAN-v5 A2):
- a regular-file entry [path, "file", sha256, size] is resolved by sha256 to any published file or archive member
  with that hash (members.sha256 + delta members), size-checked, sha256-verified, and placed at ROOT/<arm>/<path>;
  paths that leave the arm ("../dataset_tta_hub1030/...") land beside it under the same mirror.
- a file symlink ["symlink->T", sha256, size]: the content is placed at ROOT/T and the link points there relatively.
- a directory symlink ["symlink->T", null, null] becomes a relative link to the directory ROOT/T, which is filled
  by the entries listed beneath the link; a directory symlink with nothing listed beneath it fails.
- any missing, mismatched or unresolvable entry fails (exit 1) and nothing is reported as assembled.
- every content entry gets its recorded mtime (arms/<arm>/mtimes.json.gz beside the manifest); hard links are shared
  only between entries with the same mtime. ROOT must be new or empty; nothing outside ROOT is written or resolved.
Output: ROOT/<arm>/... plus ROOT/assembled_<arm>.json, a fresh dataset_manifest-format listing (same collector as the
original, claude/tools/dataset_manifest.py semantics), with the comparison result. Exit 0 only if every entry
(path, type, sha256, size, mtime) equals the original, symlink targets compared after the relocation.
"""
import argparse, hashlib, json, os, shutil, subprocess, sys, tarfile
from concurrent.futures import ProcessPoolExecutor

def sha256_file(path):
    h = hashlib.sha256()
    with open(path, 'rb') as f:
        for b in iter(lambda: f.read(1 << 22), b''):
            h.update(b)
    return h.hexdigest()


def load_members(data):
    """sha256 -> list of (size, source) where source is a published relpath or (archive relpath, member)."""
    files = [os.path.join(data, 'repro', 'members.sha256')]
    ddir = os.path.join(data, 'repro', 'delta')
    if os.path.isdir(ddir):
        files += sorted(os.path.join(ddir, f) for f in os.listdir(ddir) if f.endswith('.members.sha256'))
    idx = {}
    for fn in files:
        prefix = 'repro/' if os.path.dirname(fn).endswith('delta') else ''
        for line in open(fn):
            h, size, p = line.rstrip('\n').split('  ', 2)
            if h.startswith('symlink:'):
                continue
            src = tuple((prefix + p).split('::', 1)) if '::' in p else prefix + p
            idx.setdefault(h, []).append((int(size), src))
    return idx


def mirror(root, abs_path):
    p = os.path.normpath(os.path.join(root, abs_path.lstrip('/')))
    if os.path.commonpath([p, root]) != root:
        sys.exit(f'path escapes ROOT: {abs_path}')
    return p


def inside(root, p):
    return os.path.commonpath([os.path.realpath(p), os.path.realpath(root)]) == os.path.realpath(root)


def safe_dir(root, d):
    """Refuse to create or write through a directory unless it is lexically under ROOT and its nearest existing
    ancestor resolves (through any links already made) to a place under ROOT."""
    d = os.path.normpath(d)
    if os.path.commonpath([d, root]) != root:
        sys.exit(f'path escapes ROOT: {d}')
    e = d
    while not os.path.lexists(e):
        e = os.path.dirname(e)
    if not inside(root, e):
        sys.exit(f'path resolves outside ROOT: {d}')


def collect_listing(arm_dir, catalog, workers):
    """Recompute the arm listing exactly as dataset_manifest.py does (imported from the bundle copy)."""
    sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
    import dataset_manifest as dm
    digest, entries, total = dm.build(arm_dir, os.path.join(arm_dir, catalog), workers)
    return digest, entries, total


def main():
    ap = argparse.ArgumentParser(description=__doc__.split('\n\n')[0])
    ap.add_argument('--data', required=True); ap.add_argument('--manifest', required=True)
    ap.add_argument('--root', required=True); ap.add_argument('--workers', type=int, default=8)
    ap.add_argument('--verify-only', action='store_true', help='skip assembly; re-list and compare an existing ROOT')
    a = ap.parse_args()
    data, root = os.path.abspath(a.data), os.path.abspath(a.root)
    man = json.load(open(a.manifest))
    arm_orig = man['arm_dir']
    arm_dir = mirror(root, arm_orig)
    name = os.path.basename(arm_orig)
    if not a.verify_only and os.path.lexists(root) and (os.path.islink(root) or os.listdir(root)):
        sys.exit(f'{root} must be a new or empty directory (not a symlink)')
    os.makedirs(root, exist_ok=True)
    mt_path = os.path.join(os.path.dirname(os.path.abspath(a.manifest)), 'mtimes.json.gz')
    import gzip
    with gzip.open(mt_path, 'rt') as f:
        mtimes = json.load(f)  # path -> st_mtime_ns of the resolved file (the track caches check DBM mtimes)
    idx = load_members(data)
    entries = man['entries']
    fail = []

    def target_abs(path, typ):
        t = typ[len('symlink->'):]
        if not t.startswith('/'):  # every link in the published arms is absolute; a relative one would need the original real parent
            sys.exit(f'unsupported relative symlink in the manifest: {path} -> {t}')
        return t

    if a.verify_only:
        return compare(a, root, arm_dir, name, man, entries, target_abs, mtimes)
    # 1. directory symlinks first, shallowest first, so later paths resolve through them
    dirs = sorted((e for e in entries if e[1].startswith('symlink->') and e[2] is None), key=lambda e: e[0].count('/'))
    listed = sorted(e[0] for e in entries)
    import bisect
    for path, typ, _, _ in dirs:
        i = bisect.bisect_left(listed, path + '/')
        if not (i < len(listed) and listed[i].startswith(path + '/')):
            fail.append(f'directory symlink with nothing listed beneath: {path}'); continue
        link = os.path.normpath(os.path.join(arm_dir, path))  # write through the path that safe_dir() validated
        tgt = mirror(root, target_abs(path, typ))
        safe_dir(root, tgt); safe_dir(root, os.path.dirname(link))
        os.makedirs(tgt, exist_ok=True); os.makedirs(os.path.dirname(link), exist_ok=True)
        os.symlink(os.path.relpath(os.path.realpath(tgt), os.path.realpath(os.path.dirname(link))), link)
    # 2. content placements: (sha, size, destination)
    place = []
    for path, typ, h, size in entries:
        if h is None:
            continue
        if path not in mtimes:
            fail.append(f'no recorded mtime for {path}'); continue
        if typ == 'file':
            place.append((h, size, os.path.normpath(os.path.join(arm_dir, path)), None, mtimes[path]))
        else:
            tgt = mirror(root, target_abs(path, typ))
            place.append((h, size, tgt, os.path.normpath(os.path.join(arm_dir, path)), mtimes[path]))
    need = {}
    for h, size, dst, _, _ in place:
        cands = [c for c in idx.get(h, []) if c[0] == size]
        if not cands:
            fail.append(f'unresolvable content {h} ({size} B) for {dst}'); continue
        need.setdefault(h, cands)
    if fail:
        print('\n'.join(fail[:50]), f'\n{len(fail)} failures', file=sys.stderr); sys.exit(1)
    # 3. content store: loose files copied, archive members streamed once per archive
    cs = os.path.join(root, '.content'); os.makedirs(cs, exist_ok=True)
    by_archive = {}
    for h, cands in need.items():
        loose = [c for c in cands if isinstance(c[1], str)]
        if loose:
            shutil.copyfile(os.path.join(data, loose[0][1]), os.path.join(cs, h))
        else:
            arch, member = cands[0][1]
            by_archive.setdefault(arch, {})[member] = h
    for arch, members in sorted(by_archive.items()):
        z = subprocess.Popen(['zstd', '-dcqf', os.path.join(data, arch)], stdout=subprocess.PIPE)
        with tarfile.open(fileobj=z.stdout, mode='r|') as t:
            for m in t:
                h = members.pop(m.name, None)
                if h is None or not m.isfile():
                    continue
                with t.extractfile(m) as src, open(os.path.join(cs, h), 'wb') as dst:
                    shutil.copyfileobj(src, dst, 1 << 22)
        z.wait()
        if members:
            sys.exit(f'{arch}: {len(members)} members not found, e.g. {next(iter(members))}')
    with ProcessPoolExecutor(a.workers) as pool:
        hs = list(pool.map(sha256_file, [os.path.join(cs, h) for h in need], chunksize=64))
    bad = [h for h, got in zip(need, hs) if got != h]
    if bad:
        sys.exit(f'{len(bad)} content files fail sha256, e.g. {bad[0]}')
    # 4. place content (hard links into the content store when the recorded mtime agrees, else a copy) and file symlinks
    cs_mtime, copies = {}, []
    for h, size, dst, link, mt in place:
        safe_dir(root, os.path.dirname(dst))
        os.makedirs(os.path.dirname(dst), exist_ok=True)
        if not os.path.lexists(dst):
            if cs_mtime.setdefault(h, mt) == mt:
                try:
                    os.link(os.path.join(cs, h), dst)
                except OSError:
                    shutil.copyfile(os.path.join(cs, h), dst); copies.append((dst, mt))
            else:
                shutil.copyfile(os.path.join(cs, h), dst); copies.append((dst, mt))
        if link is not None:
            safe_dir(root, os.path.dirname(link)); os.makedirs(os.path.dirname(link), exist_ok=True)
            os.symlink(os.path.relpath(os.path.realpath(dst), os.path.realpath(os.path.dirname(link))), link)
    for h, mt in cs_mtime.items():
        os.utime(os.path.join(cs, h), ns=(mt, mt))
    for dst, mt in copies:
        os.utime(dst, ns=(mt, mt))
    shutil.copyfile(a.manifest, os.path.join(arm_dir, 'dataset_manifest.json'))  # skipped by the collector, kept for reference
    return compare(a, root, arm_dir, name, man, entries, target_abs, mtimes)


def compare(a, root, arm_dir, name, man, entries, target_abs, mtimes):
    # 5. fresh listing and comparison
    digest, fresh, total = collect_listing(arm_dir, man['catalog'], a.workers)
    orig = {e[0]: e for e in entries}; new = {e[0]: e for e in fresh}
    diffs = [f'missing {p}' for p in orig if p not in new] + [f'extra {p}' for p in new if p not in orig]
    for p in orig.keys() & new.keys():
        o, n = orig[p], new[p]
        full = os.path.join(arm_dir, p)
        if not inside(root, full):
            diffs.append(f'resolves outside ROOT: {p}'); continue
        if o[2] is not None and os.stat(full).st_mtime_ns != mtimes.get(p):
            diffs.append(f'mtime {p}: {os.stat(full).st_mtime_ns} vs recorded {mtimes.get(p)}')
        if (o[2], o[3]) != (n[2], n[3]):
            diffs.append(f'content {p}: {o[2:]} vs {n[2:]}')
        elif o[1] == 'file' or n[1] == 'file':
            if o[1] != n[1]:
                diffs.append(f'type {p}: {o[1]} vs {n[1]}')
        else:  # both symlinks: the new link must resolve to the mirror of the original target
            want = mirror(root, target_abs(p, o[1]))
            got = os.path.realpath(os.path.join(arm_dir, p))  # the kernel resolves a relative link from the REAL parent
            if got != os.path.realpath(want) or n[1][len('symlink->'):].startswith('/'):
                diffs.append(f'symlink {p}: -> {got} vs mirror {want}')
    report = dict(arm=name, original_digest=man['digest'], original_entries=len(entries), fresh_entries=len(fresh),
                  fresh_digest=digest, bytes=total, relocation=f'absolute symlink targets T -> relative links to {root}/T',
                  differences=len(diffs), first_differences=diffs[:50], entries=fresh)
    with open(os.path.join(root, f'assembled_{name}.json'), 'w') as f:
        json.dump(report, f, separators=(',', ':'))
    print(json.dumps({k: v for k, v in report.items() if k != 'entries'}, indent=1))
    sys.exit(1 if diffs else 0)


if __name__ == '__main__':
    main()
