#!/usr/bin/env bash
# chain_main.sh PREFIX STORE [ITERS_A ITERS_B] : M6 on main's fitter (~/Documents/villa on main, env spiral-fitting/.venv-main; origin/main >= b7d554598; schema-22 store) from a
# WORKING COPY of the schema-21 store (link refresh writes to it; m6seam21.vcobj stays pristine), two seeds,
# Va_<P>_s<S>_10k (scratch) -> Vb_<P>_s<S>_25k (refine from it), each Vb scored with the FROZEN s1_score on the trimmed
# records (scores_trim.json), as chain_whole.sh. Stop file: store_m6/chain_main_<P>.stop. Exit 10 fit, 11 scoring.
# ITERS_A/ITERS_B (default 10000/25000) only label the expected horizons for verify_held.py; the overrides set the
# real num_iterations (a smoke chain uses its own overrides files).
set -u -o pipefail
D=/media/djosey/nvme4/spiral_0332; M=$D/store_m6; P=${1:?prefix}; STORE=${2:?store .vcobj}; IA=${3:-10000}; IB=${4:-25000}
MAIN=/home/djosey/Documents/villa/spiral-fitting; MPY=$MAIN/.venv-main/bin/python
F=$D/frozen/s1_f327248db2c1f694; PY=/home/djosey/Documents/villa/spiral-fitting/.venv/bin/python
echo $$ > $M/chain_main_${P}.pid; rm -f $M/chain_main_${P}.stop
stop() { [ -f $M/chain_main_${P}.stop ]; }
rundir() { ls -d $D/out/*_$1 2>/dev/null | tail -1; }
echo "$(date) chain_main $P start; main $(git -C $MAIN rev-parse --short HEAD); store $STORE"
fit() {  # $1 tag, [$2 parent tag]
  local T=$1 PT=${2:-} CK=""
  if [ -n "$PT" ]; then CK=$(rundir $PT)/checkpoint_fitted.ckpt; [ -f "$CK" ] || { echo "$(date) no parent checkpoint for $T"; return 1; }; fi
  echo "$(date) === $T ${PT:+from $PT}"
  # env, not a bare ${CK:+VAR=...} expansion (bash runs that as a command); scratch runs explicitly drop any
  # inherited resume path.
  local -a E=(FIT_SPIRAL_CONFIG_OVERRIDES="$(tr -d '\n' < $M/inputs/overrides_$T.json)" FIT_SPIRAL_OUT_DIR=$D/out
              FIT_SPIRAL_RUN_TAG=$T WANDB_MODE=disabled PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True)
  [ -n "$CK" ] && E+=(FIT_SPIRAL_RESUME_PATH=$CK)
  ( cd $MAIN && env -u FIT_SPIRAL_RESUME_PATH "${E[@]}" \
    $MPY fit_spiral.py --inputs $M/inputs/spiral-inputs.json --object-store $STORE --object-store-author djosey \
      --cache $M/cache_main22 ) > $M/fit_$T.log 2>&1
  local rc=$?; echo "$(date) $T exited $rc; tracebacks $(grep -c Traceback $M/fit_$T.log)"
  local nf; nf=$(grep 'non-finite-gradient steps sanitised' $M/fit_$T.log | tail -1)
  [ -n "$nf" ] && echo "$(date) NONFINITE $T:$nf"
  [ -n "$CK" ] && echo "$(date) $T resumed: $(grep -m1 -i 'resum' $M/fit_$T.log)"
  return $rc
}
held() {  # $1 tag, $2 its horizon: resume its checkpoint at its own horizon with no patch exclusions (0 training steps) so the
  # held-out patches are loaded and get satisfied areas, then score that run dir (same model) with the frozen scorer.
  local T=$1 N=$2 H=Hx_$1
  $PY -c "import json,sys; c=json.load(open('$M/inputs/overrides_$T.json')); c['object_store_patch_exclude_uuids']=[]; json.dump(c,open('$M/inputs/overrides_$H.json','w'),indent=1)"
  fit $H $T || return 1
  local R; R=$(rundir $H); $MPY $M/held_ids.py $R || return 1
  ( cd $F/work/s1 && SPIRAL_ROOT=$F/spiral-fitting WORK_ROOT=$F/work CUDA_VISIBLE_DEVICES= $PY $F/work/s1/s1_score.py $R --out $R/scores_trim.json \
    --train-records $D/work/s1_trim/anchors_train_records.json --held-records $D/work/s1_trim/anchors_held_records.json ) > $M/scoretrim_$T.log 2>&1 || return 1
  local V; V=$($MPY $M/verify_held.py $(rundir $T)/checkpoint_fitted.ckpt $R $N 2>&1); local vr=$?
  echo "$(date) TSCORE $T $V"; return $vr
}
for S in ${SEEDS:-1 2}; do  # SEEDS="1" for a smoke chain
  stop && break
  t0=$(date +%s); fit Va_${P}_s${S}_10k || { echo "$(date) FAILED Va_${P}_s${S}_10k"; exit 10; }
  echo "$(date) Va_${P}_s${S}_10k wall $(( ($(date +%s)-t0)/60 )) min"
  # early check on seed 1: the held-out pass and the frozen scorer must work on a main checkpoint before 25k is spent
  [ $S = 1 ] && { held Va_${P}_s1_10k $IA || { echo "$(date) FAILED early held-out check on Va_${P}_s1_10k"; exit 11; }; }
  stop && break
  t0=$(date +%s); fit Vb_${P}_s${S}_25k Va_${P}_s${S}_10k || { echo "$(date) FAILED Vb_${P}_s${S}_25k"; exit 10; }
  echo "$(date) Vb_${P}_s${S}_25k wall $(( ($(date +%s)-t0)/60 )) min"
  held Vb_${P}_s${S}_25k $IB || { echo "$(date) FAILED held-out pass/scoring of Vb_${P}_s${S}_25k"; exit 11; }
done
echo "$(date) chain_main ${P} done"
