#!/usr/bin/env bash
# run_reference.sh: relocated, equivalent replay of the recorded PHerc0332 reference commands (NEW code; the frozen
# bytes it copies are untouched in the archive).
#
#   run_reference.sh score --data DATA --src SRC --ref REF --seed s1|s2 --out OUT [--py PY]
#       rescore the PUBLISHED Vb checkpoint of REF/seed (DATA/repro/runs/REF/SEED/Vb) with the launchers' own trimmed
#       scoring command (s1_score.py on CPU with the trimmed records and its defaults: step 8, the 51 slice lines, the
#       frozen work/s1/holdout_eval_set.json), then compare with the published scores_trim.json (exit 0 = equal).
#   run_reference.sh fit --data DATA --src SRC --ref REF --seed s1|s2 --out OUT --arm-root ROOT [--py PY] [--stage both|Vb] [--dry-run]
#       refit through the frozen chain_s1.sh stage()/valid(), sourced exactly as the launchers sourced it: Va (10k from
#       scratch) -> Vb (25k resumed from Va; --stage Vb resumes from the published Va), each stage's manifest, untrimmed
#       scores.json and validity check as recorded, the launcher's FIT_CACHE name and density-hook mode (#81: none;
#       AW/M6/M8: capture; SWA: compare against AW s1's published fingerprint), then the trimmed scoring and a
#       comparison report. GPU, hours per stage; a refit is a seed-noise-level comparison, not bit identity.
#
# DATA = the downloaded ash dataset dir. SRC = the extracted frozen source tar (tar -xf
# eval/frozen_source_s1_f327248db2c1f694.tar.zst -C SRC  ->  SRC/{spiral-fitting,vesuvius,work,source.sha256,...}).
# PY = the interpreter of a fresh env built per repro/README.md.
# Relocation: OUT/D stands in for /media/djosey/nvme4/spiral_0332 and OUT/D/work for its work/ (the frozen work/ scripts
# plus the launch scripts); OUT/F stands in for the frozen root. In every copied .py/.sh ONLY the five workstation path
# constants in SUBST are replaced, and afterwards no /media/djosey or /home/djosey string may remain (else abort);
# diff in OUT/relocation.diff. JSON data files are copied byte for byte. Every input is checked against its published
# sha256 (repro/members.sha256) before use.
set -euo pipefail
MODE=${1:?score|fit}; shift
REF= SEED= OUT= DATA= SRC= PY=python3 ARMROOT= STAGES=both DRY=
while [ $# -gt 0 ]; do case $1 in
  --ref) REF=$2; shift 2;; --seed) SEED=$2; shift 2;; --out) OUT=$2; shift 2;; --data) DATA=$2; shift 2;;
  --src) SRC=$2; shift 2;; --py) PY=$2; shift 2;; --arm-root) ARMROOT=$2; shift 2;; --stage) STAGES=$2; shift 2;; --dry-run) DRY=1; shift;;
  *) echo "unknown argument $1" >&2; exit 2;; esac; done
die() { echo "run_reference: $*" >&2; exit 1; }
for v in REF SEED OUT DATA SRC; do [ -n "${!v}" ] || die "--${v,,} is required"; done
case $STAGES in both|Vb) ;; *) die "--stage must be both or Vb";; esac
[ ! -e "$OUT" ] || [ -z "$(ls -A "$OUT")" ] || die "--out must be a new or empty directory (stage() picks run dirs by tag)"
DATA=$(readlink -f "$DATA"); SRC=$(readlink -f "$SRC"); mkdir -p "$OUT"; OUT=$(readlink -f "$OUT")
[ -z "$ARMROOT" ] || ARMROOT=$(readlink -f "$ARMROOT")
PY=$(command -v "$PY") || die "python not found"
for p in "$DATA" "$SRC" "$OUT" "$PY" "$ARMROOT"; do  # these are pasted into shell and Python code by the relocation
  [[ "$p" =~ ^[A-Za-z0-9._/+-]*$ ]] || die "paths may contain only letters, digits and . _ / + - : $p"; done
[ -d "$SRC/work/s1" ] && [ -d "$SRC/spiral-fitting" ] || die "--src must be the extracted frozen source tar (SRC/work/s1 missing)"
REC=$DATA/repro/recipes/$REF/$SEED; [ -d "$REC" ] || die "no recipe $REC"
HERE=$(dirname "$(readlink -f "$0")")
sha() { sha256sum "$1" | cut -d' ' -f1; }
pubsha() { awk -v p="$1" -F'  ' '$3 == p {print $1; exit}' "$DATA/repro/members.sha256"; }
check() { local want; want=$(pubsha "$2"); [ -n "$want" ] || die "$2 is not a published file"
  [ "$(sha "$1")" = "$want" ] || die "$2: sha256 differs from the published one"; }
lin() { "$PY" -c "import json; print(json.load(open('$REC/lineage.json'))$1)"; }

# --- published inputs, hash-checked ---
check "$DATA/umbilicus/umbilicus.json" umbilicus/umbilicus.json
check "$DATA/anchors/anchors_train_records.json" anchors/anchors_train_records.json
check "$DATA/anchors/anchors_held_records.json" anchors/anchors_held_records.json
check "$DATA/eval/frozen_s1_scorer/winding_gauge.json" eval/frozen_s1_scorer/winding_gauge.json
LINES=$DATA/slice_lines/PHerc0332_vcslices.volpkg.json
"$PY" - "$DATA" <<'PY' || die "slice-line files differ from the published set"
import hashlib, os, sys
data = sys.argv[1]; pre = 'slice_lines/PHerc0332_vcslices.volpkg.json/'
want = {}
for l in open(f'{data}/repro/members.sha256'):
    h, _, p = l.rstrip('\n').split('  ', 2)
    if p.startswith(pre): want[p] = h
have, root = {}, f'{data}/{pre}'
for d, _, fs in os.walk(root):
    for f in fs:
        full = os.path.join(d, f)
        have[pre + os.path.relpath(full, root)] = hashlib.sha256(open(full, 'rb').read()).hexdigest()
bad = sorted(set(want) ^ set(have)) + sorted(p for p in set(want) & set(have) if want[p] != have[p])
print(f'slice lines: {len(have)} files, {len(bad)} differences from the published set')
sys.exit(1 if bad or not want else 0)
PY

# --- relocated tree ---
ND=$OUT/D; NW=$ND/work; F=$OUT/F
rm -rf "$NW" "$F"; mkdir -p "$ND/dataset" "$ND/out" "$NW" "$F"
cp "$DATA/umbilicus/umbilicus.json" "$ND/dataset/umbilicus.json"
cp -a "$SRC/work/." "$NW/"
cp "$DATA/eval/frozen_s1_scorer/winding_gauge.json" "$NW/winding_gauge.json"   # DEPENDENCIES.md; read, not re-applied
cp "$DATA/repro/launch/work/"{chain_whole.sh,chain_trim.sh,chain_nolas.sh,fiber_jump_map_sheet.py} "$NW/"
mkdir -p "$NW/density/hook" "$NW/s1_trim"
cp "$DATA/repro/launch/work/density/hook/sitecustomize.py" "$NW/density/hook/"
cp "$DATA/repro/launch/work/density/trackhash_Vb_AW_s1_25k.json" "$NW/density/"
cp "$DATA/anchors/anchors_train_records.json" "$DATA/anchors/anchors_held_records.json" "$NW/s1_trim/"  # = work/s1_trim (same sha256)
ln -s "$NW" "$F/work"; ln -s "$SRC/spiral-fitting" "$F/spiral-fitting"
for f in source.sha256 venv.json inputs.json; do ln -s "$SRC/$f" "$F/$f"; done
cp -a "$NW" "$OUT/work.orig"
SUBST=(  # longest first; placeholders first, so the leftover check cannot be fooled by a new path under /home
  "s#/home/djosey/Documents/villa/spiral-fitting/.venv/bin/python#@@PY@@#g"
  "s#/home/djosey/Documents/villa/spiral-fitting#@@SPIRAL@@#g"
  "s#/media/djosey/nvme4/fibers/PHerc0332_vcslices.volpkg.json#@@LINES@@#g"
  "s#/media/djosey/nvme4/spiral_0332/work#@@WORK@@#g"
  "s#/media/djosey/nvme4/spiral_0332#@@D@@#g")
mapfile -t CODE < <(find "$NW" -type f \( -name '*.py' -o -name '*.sh' \))
for f in "${CODE[@]}"; do for e in "${SUBST[@]}"; do sed -i "$e" "$f"; done; done
left=$(grep -l -E '/(media|home)/djosey' "${CODE[@]}" || true)
[ -z "$left" ] || die "workstation paths remain after substitution in: $left"
for f in "${CODE[@]}"; do sed -i -e "s#@@PY@@#$PY#g" -e "s#@@SPIRAL@@#$SRC/spiral-fitting#g" -e "s#@@LINES@@#$LINES#g" \
  -e "s#@@WORK@@#$NW#g" -e "s#@@D@@#$ND#g" "$f"; done
! grep -l '@@[A-Z]*@@' "${CODE[@]}" > /dev/null || die "unreplaced placeholder"
(cd "$OUT" && diff -ru work.orig D/work > relocation.diff || true); rm -rf "$OUT/work.orig"
check "$NW/s1/holdout_eval_set.json" "eval/frozen_source_s1_f327248db2c1f694.tar.zst::work/s1/holdout_eval_set.json"

PUB=$REC/Vb/scores_trim.json
rescore() {  # the launchers' trimmed scoring (chain_trim/chain_nolas rescore(), chain_whole.sh), then the comparison
  local R=$1
  CUDA_VISIBLE_DEVICES= "$PY" "$F/work/s1/s1_score.py" "$R" --out "$R/scores_trim.json" \
    --train-records "$ND/work/s1_trim/anchors_train_records.json" --held-records "$ND/work/s1_trim/anchors_held_records.json" \
    > "$R/scoretrim.log" 2>&1 || die "trimmed scoring failed ($R/scoretrim.log)"
  "$PY" "$HERE/compare_scores.py" "$PUB" "$R/scores_trim.json" ${2:-} | tee "$R/compare.txt"
}

case $MODE in
score)
  export SPIRAL_ROOT=$F/spiral-fitting WORK_ROOT=$F/work
  R=$OUT/published/$(lin "['Vb']['tag']"); mkdir -p "$R"
  for f in checkpoint_fitted.ckpt manifest.json satisfied_fitted.json; do cp "$DATA/repro/runs/$REF/$SEED/Vb/$f" "$R/$f"; done
  [ "$(sha "$R/checkpoint_fitted.ckpt")" = "$(lin "['Vb']['checkpoint_sha256']")" ] || die "checkpoint differs from the lineage record"
  [ "$(sha "$R/manifest.json")" = "$(lin "['Vb']['manifest_sha256']")" ] || die "manifest differs from the lineage record"
  rescore "$R";;
fit)
  [ -n "$ARMROOT" ] || die "fit needs --arm-root (assemble_arm.py output)"
  VA=$(lin "['Va']['tag']"); VB=$(lin "['Vb']['tag']"); LAUNCH=$(lin "['launcher']")
  for st in Va Vb; do  # relocated overrides where stage() reads them ($D/overrides_<tag>.json)
    TAG=$(lin "['$st']['tag']")
    "$PY" - "$REC/$st/overrides.json" "$ND/overrides_$TAG.json" "$ND/work/s1_trim/anchors_train_records.json" <<'PY'
import json, sys
o = json.load(open(sys.argv[1])); before = dict(o)
if o.get('abs_winding_audit_records') != '/media/djosey/nvme4/spiral_0332/work/s1_trim/anchors_train_records.json':
    sys.exit(f"unexpected abs_winding_audit_records {o.get('abs_winding_audit_records')!r}")
o['abs_winding_audit_records'] = sys.argv[3]   # the only absolute path in any reference overrides dict
bad = [k for k, v in before.items() if isinstance(v, str) and '/djosey' in v and k != 'abs_winding_audit_records']
if bad: sys.exit(f'unexpected workstation paths in overrides: {bad}')
json.dump(o, open(sys.argv[2], 'w'), indent=1)
open(sys.argv[2] + '.rewrite.txt', 'w').write(''.join(f'{k}: {before[k]!r} -> {o[k]!r}\n' for k in o if o[k] != before[k]))
PY
    ARM=$("$PY" -c "import json,os; print(os.path.basename(json.load(open('$REC/$st/manifest.json'))['env']['FIT_DATASET']))")
    [ -d "$ARMROOT/media/djosey/nvme4/spiral_0332/$ARM" ] || die "assembled arm missing under --arm-root: $ARM"
    ln -sfn "$ARMROOT/media/djosey/nvme4/spiral_0332/$ARM" "$ND/$ARM"
  done
  SUFFIX=${ARM#dataset_tta_hub1030h_}
  HREF=
  if [ "$LAUNCH" = work/chain_whole.sh ]; then
    INV=$DATA/repro/launch/invocations/$(basename "$(lin "['launch_records'][0]")")
    read -r P ARMX CACHE HREF < <("$PY" - "$INV/steps.log" <<'PY'
import re, sys
m = re.search(r'chain_whole\.sh \S+ (\S+) (\S+) (\S+)(?: (/\S+\.json))?', open(sys.argv[1]).read())
print(*[g or '' for g in m.groups()])
PY
)
    [ "$ARMX" = "$SUFFIX" ] || die "recorded arm $ARMX != recipe arm $SUFFIX"
    [ -z "$HREF" ] || HREF=$ND/work/density/$(basename "$HREF")
  fi
  set +e
  # the launchers' environment: chain_s1.sh sourced exactly as chain_trim/chain_nolas/chain_whole source it
  source <(sed -e '/chain_s1.pid/d' -e '/chain_dt done/d' -e '/^for ARM_TAG/,$d' "$F/work/s1/chain_s1.sh") "$F"
  case $LAUNCH in
    work/chain_trim.sh) ;;                                   # FIT_CACHE=$D/cache_hub1030 as chain_s1.sh exports it
    work/chain_nolas.sh) export FIT_CACHE=$D/cache_nolas;;
    work/chain_whole.sh) export FIT_CACHE=$D/$CACHE PYTHONPATH=$D/work/density/hook;;
    *) die "unknown launcher $LAUNCH";;
  esac
  mkdir -p "$FIT_CACHE"
  if [ -n "$DRY" ]; then  # plumbing check: show what each stage would run, fit nothing
    stage() { echo "DRY stage $*: FIT_DATASET=$D/dataset_tta_hub1030h_$2 run_fit.sh $D/overrides_$1.json $1 ${4:+(resume from $(rundir $4)/checkpoint_fitted.ckpt)}"
      echo "    FIT_CACHE=$FIT_CACHE SPIRAL_ROOT=$SPIRAL_ROOT WORK_ROOT=$WORK_ROOT PYTHONPATH=${PYTHONPATH:-} HASH_OUT=${DENSITY_TRACK_HASH_OUT:-} HASH_REF=${DENSITY_TRACK_HASH_REF:-}"
      [ -d "$D/dataset_tta_hub1030h_$2" ] && [ -f "$D/overrides_$1.json" ] || return 1; }
    rundir() { echo "$D/out/<$1>"; }
    rescore() { echo "DRY rescore $1"; }
  fi
  hook() { [ "$LAUNCH" = work/chain_whole.sh ] || return 0
    export DENSITY_TRACK_HASH_OUT=$D/work/density/trackhash_$1.json
    if [ -n "$HREF" ]; then export DENSITY_TRACK_HASH_REF=$HREF; else unset DENSITY_TRACK_HASH_REF; fi; }
  if [ "$STAGES" = Vb ]; then  # the published Va as the parent run dir (it was VALID when recorded)
    PR=$D/out/published_$VA; mkdir -p "$PR"
    for f in checkpoint_fitted.ckpt manifest.json; do cp "$DATA/repro/runs/$REF/$SEED/Va/$f" "$PR/$f"; done
    [ "$(sha "$PR/checkpoint_fitted.ckpt")" = "$(lin "['Va']['checkpoint_sha256']")" ] || die "published Va checkpoint mismatch"
    touch "$PR/stage.valid"
  else
    hook "$VA"; stage "$VA" "$SUFFIX" 10000 || die "stage $VA not VALID (see $D/work/fit_$VA.log)"
  fi
  hook "$VB"; stage "$VB" "$SUFFIX" 25000 "$VA" || die "stage $VB not VALID (see $D/work/fit_$VB.log)"
  rescore "$(rundir "$VB")" --report;;
*) die "mode must be score or fit";;
esac
