#!/usr/bin/env bash
# verso3 final (init.ckpt) continued with the teacher-distilled recto recipe (--verso-anchor 3) after the real-label A/B (verso4) let verso drift to recto. P 640, half peak LR, seed 28, 9900 steps.
# RESUMED with upstream 7c3a149 (memory offloads) from the step-2500 checkpoint of the same run; verso channel, recipe from the Paris4 tests (verso-probe/bin/lam*.out): cout 8 = recto, verso, 6 affinities, from the
# open-surfaces-aug9 final model. Frozen teacher = start.ckpt (unet_use_params swaps u->live). Verso target: the teacher's
# recto channel with the radial input reversed, thinned to its ridge along the direction (>= 90/255), own loss over every
# owned voxel at weight 0.25. Recto distilled from the teacher (normal direction, soft) everywhere so verso cannot drag it.
# All 76 sources and augmentations of open-surfaces-aug9; 7 um soft labels; MX-fp4 gradients (upstream default).
F=/vesuvius/ufsm/runs/verso5-20261009
R=/vesuvius/ufsm/runs/open-surfaces-20261007
STEPS=${STEPS:-9900}
AUG="--aug-strength 1 --aug-p 0.08 --aug-noise-p 0.15 --aug-lowres-p 0.15 --aug-blur-p 0.12 --aug-tone-p 0.15 --aug-bias-p 0.12 --aug-ring-p 0.1 --aug-slice-p 0.1 --aug-seam-p 0.05 --aug-air-p 0.05 --aug-dropslice-p 0.03 --aug-cutout-p 0.03 --aug-chunk-p 0.03 --aug-drop-ignore 1 --aug-affine-p 0.15"
cd $F
UFSM_RC_KEEP_COARSE=1 $F/ufsm train $R/sources.json --out $F/${OUT:-model} --gpus 0,1 --split z --task band_affinity \
  --verso 1 --verso-ridge 90 --verso-anchor 3 --verso-lambda 0.25 \
  --resume $F/init.ckpt --finetune 1 --verso-teacher-ckpt $F/start.ckpt --widths 32,64,96,128,160,192 --scale-cond 1 --steps $STEPS --warmup 300 \
  --geometry-aug 1 --rotate-p 0 --elastic-p 0 --zoom-p 0.5 --zoom-min 0.7071 --levels 1,1,1,1 \
  --fp4 2 --mem auto --opt muon --muon-lr 0.005 --lr 0.0005 --sched wsd --cooldown 0.25 --soft 7 --soft-unit um --erode 1 --down-norm 1 --workers 16 \
  --seed 28 --input-prec 8 --gn-stats stored --val-batches 8 --log-every 20 --val-every 1000 --ckpt-every 500 --zfix 1 \
  --symmetry-p 0.5 --ct-aug 1 $AUG --axis-jitter 32 --det 1 --P 640 --B 1 > $F/${OUT:-model}-resume.log 2>&1
echo "training exit $? $(date)" >> $F/status.log
[ -n "$OUT" ] && exit 0
cp model/last.ckpt final.ckpt
P4="/vesuvius/usrm/volcomp PHercParis4/20260411134726-2.400um-0.2m-78keV-masked.zarr"
A=/vesuvius/ufsm/runs/noisy-labels-20261005/axis-0.json
for c in 0 1; do $F/ufsm predict $F/final.ckpt $P4 $F/paris4_ch$c --um 2.4 --box 48512,17408,16384,96,1024,1024 --window 544 --halo 16 --shard 512 \
  --ema 1 --q 8 --threads 16 --levels 1 --gpu $c --axis $A --channel $c & done; wait
$F/ufsm predict $F/start.ckpt $P4 $F/paris4_teacher_rev --um 2.4 --box 48512,17408,16384,96,1024,1024 --window 544 --halo 16 --shard 512 \
  --ema 1 --q 8 --threads 16 --levels 1 --gpu 0 --axis $A --radial-sign -1
PY=/home/forrest/miniconda3/bin/python3; T=/home/forrest/ufsm/tools
cd $T
( $PY score_surface.py --window 544,16 --out $F/scores --gpu 0 start=$F/init.ckpt final=$F/final.ckpt > $F/score_surface.log 2>&1 ) &
( $PY score_open.py --out $F/openscores --gpu 1 start=$F/init.ckpt final=$F/final.ckpt > $F/score_open.log 2>&1 ) &
wait
echo "scoring done $(date)" >> $F/status.log