AtlatestRepositoryfolio

folio / tree / testsabotage-f2b.sh

1#!/bin/sh
2# sabotage-f2b.sh -- prove that `test/gate-f2b.sh` can go RED.
3#
4# A gate reported only green has been shown to be quiet, not to work. Every leg
5# below breaks the PRODUCT -- source under `src/` or `tools/`, not the gate's own
6# assertions -- rebuilds, and requires the named arm to fail. A leg that stays
7# green is a finding about the gate, and this script says so in those words
8# rather than exiting 0.
9#
10# WHY THE PRODUCT LAYER SPECIFICALLY
12# Breaking an assertion only proves the assertion is wired to something.
13# Breaking the PRODUCT proves the gate executes the product rather than a mirror
14# of itself. These legs all sit in the second class: each plants a defect a
15# reasonable person might actually ship -- several are lifted verbatim from F2's
16# own sabotage list -- and asks whether the gate notices.
18# RESTORE IS VERIFIED BY HASH, NOT BY `git checkout` REPORTING SUCCESS. A leg
19# that restored the source but left a stale build artifact would make every
20# later leg a measurement of the wrong tree, so each leg rebuilds after
21# restoring and the final check re-hashes every touched file against its
22# starting digest.
24set -u
25repo_dir=$(cd "$(dirname "$0")/.." && pwd)
26cd "$repo_dir" || exit 1
28sigil="${SIGIL:-sigil}"
29redirects="$repo_dir/dev-redirects.sgl"
30export SIGIL_REDIRECTS="$redirects"
32work=$(mktemp -d /tmp/sabotage-f2b.XXXXXX) || { echo "SETUP-FAILED: no temp dir"; exit 3; }
34# Baselines live OUTSIDE `$work`, because `cleanup` deletes `$work`.
36# On the interrupt path the trap ran first, removed `$work`, and the restore
37# verification then read empty baselines and reported "4 file(s) not restored"
38# against a tree that `git status` showed was perfectly clean. A verification
39# that reads its own reference from a directory its own teardown deletes reports
40# a false alarm exactly when things went wrong -- which is when you least want to
41# be chasing a phantom. Observed 2026-08-23 on a deliberately-killed run.
42base=$(mktemp -d /tmp/sabotage-f2b-base.XXXXXX) || { echo "SETUP-FAILED: no baseline dir"; exit 3; }
44TOUCHED="src/folio/doc/discovery.sgl src/folio/doc/bounded.sgl src/folio/doc/revision.sgl src/folio/doc/service.sgl"
46# THE TREE MUST BE CLEAN BEFORE WE START.
48# Every leg restores with `git checkout --`, so a pre-existing uncommitted change
49# to a touched file would be silently destroyed, and every "restored" check would
50# compare against the wrong baseline.
51for f in $TOUCHED; do
52 if ! git diff --quiet -- "$f" 2>/dev/null; then
53 echo "SETUP-FAILED: $f has uncommitted changes. This suite restores with \`git checkout --\`" >&2
54 echo " and would DESTROY them. Commit or set them aside first." >&2
55 exit 3
56 fi
57done
59# Baseline digests, captured BEFORE anything is planted.
60for f in $TOUCHED; do
61 sha256sum "$f" | cut -d' ' -f1 > "$base/$(echo "$f" | tr / _).base"
62done
64restore_all () {
65 for f in $TOUCHED; do git checkout -- "$f" 2>/dev/null; done
68# DO NOT `exit` FROM THIS TRAP. A RETRACTED FIX, KEPT AS A WARNING.
70# A first pass here "fixed" a status-laundering bug that DID NOT EXIST. The
71# reasoning was that this script's `exit 1` was being overwritten by its EXIT
72# trap. Measured afterwards, on this host's /bin/sh (bash 5.2.37), an EXIT trap
73# does NOT reset the status in any of four variants -- plain command, function,
74# function whose last command fails, function whose last command succeeds. All
75# four exit 1.
77# The real cause was the MEASUREMENT, not the script. The run was invoked as
79# sh test/sabotage-f2b.sh 2>&1 | grep -E ... | head -80
81# and a pipeline's status is its LAST command's -- `head`, which exits 0. The
82# suite reported its failure correctly and the harness reading it threw that away.
84# The attempted fix -- `cleanup () { ...; exit "$final_status"; }` -- then
85# INTRODUCED the very defect it was meant to remove: with `final_status` still 0,
86# an INT or TERM mid-run exits 0, so a Ctrl-C'd sabotage run reports SUCCESS.
87# Verified: `kill -INT $$` under that trap exits 0.
89# So the trap cleans up and says nothing about status, which was right all along.
90# The lesson worth keeping is the general one: **when checking whether something
91# can report failure, do not read its status through a pipe.**
92cleanup () {
93 restore_all
94 "$sigil" build --redirects "$redirects" >/dev/null 2>&1
95 rm -rf "$work"
97trap cleanup EXIT INT TERM
99legs=0
100bad=0
102# $1 = leg label, $2 = arm number(s) to run, $3 = file, $4 = sed expression
103leg () {
104 label=$1; arms=$2; file=$3; expr=$4
105 legs=$((legs + 1))
106 echo
107 echo "=========================================================================="
108 echo "LEG $legs: $label"
109 echo " planting in $file, then requiring gate arm(s) [$arms] to go RED"
110 echo "=========================================================================="
112 before=$(sha256sum "$file" | cut -d' ' -f1)
113 sed -i "$expr" "$file"
114 after=$(sha256sum "$file" | cut -d' ' -f1)
116 # THE PLANT MUST HAVE LANDED. A sed that matched nothing leaves the file
117 # untouched, the gate then passes for the honest reason, and the leg would
118 # score as "the gate did not catch it" -- convicting a correct gate. This is
119 # the single most important check in the file.
120 if [ "$before" = "$after" ]; then
121 echo "LEG $legs INCONCLUSIVE: the plant did not change $file (the sed matched nothing)."
122 echo " This says NOTHING about the gate. Fix the sed expression."
123 bad=$((bad + 1))
124 git checkout -- "$file" 2>/dev/null
125 return
126 fi
128 if ! "$sigil" build --redirects "$redirects" > "$work/build.log" 2>&1; then
129 # A plant that will not compile is still a valid red for arms that need the
130 # tree, but it is a WEAKER result: it proves the build gates the product, not
131 # that the gate's assertions discriminate. Say which one happened.
132 echo "LEG $legs RED (build): the sabotaged tree does not compile, so the gate cannot run against it."
133 echo " Weaker than an assertion red -- it proves the build catches this, not the gate."
134 git checkout -- "$file" 2>/dev/null
135 "$sigil" build --redirects "$redirects" >/dev/null 2>&1
136 return
137 fi
139 sh test/gate-f2b.sh $arms > "$work/gate.log" 2>&1
140 rc=$?
141 if [ "$rc" -eq 0 ]; then
142 echo "LEG $legs STAYED GREEN -- THIS IS A GATE DEFECT, NOT A PASS."
143 echo " The product was broken as described and gate arm(s) [$arms] did not notice."
144 grep -E '^(arm|GATE)' "$work/gate.log" | tail -6
145 bad=$((bad + 1))
146 elif [ "$rc" -eq 3 ]; then
147 # SETUP-FAILED IS NOT A RED, and treating it as one is the exact conflation
148 # the gate's own header refuses. A gate that could not establish its subject
149 # has said nothing about the product, so a leg "passing" on exit 3 would be
150 # scoring the gate's inability to run as evidence that it detects defects.
151 # Found by an adversarial review.
152 echo "LEG $legs INCONCLUSIVE: the gate exited 3 (SETUP-FAILED) -- it could not establish its subject,"
153 echo " so this says NOTHING about whether the arm detects the planted defect."
154 grep -E 'SETUP-FAILED' "$work/gate.log" | head -2 | sed 's/^/ /'
155 bad=$((bad + 1))
156 else
157 echo "LEG $legs RED as required (gate exit $rc). The failure it reported:"
158 grep 'GATE-F2B FAIL' "$work/gate.log" | head -3 | sed 's/^/ /'
159 fi
161 git checkout -- "$file" 2>/dev/null
162 "$sigil" build --redirects "$redirects" >/dev/null 2>&1
165echo "sabotage-f2b: sigil $("$sigil" --version 2>&1 | sed 's/\x1b\[[0-9;]*m//g' | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1)"
167# WHICH SHAPE THE LEGS ARE PROVEN AGAINST.
169# `F2B_SHAPE` is read by `gate-f2b.sh`; exporting it here makes every leg's gate
170# invocation test the same shape as this run's banner claims.
172# THE RE-RUN IS NOT CEREMONY. Containment was proven for the DEV shape -- a
173# script under the `sigil` runtime with `-L` paths, which cannot ship. Bundling
174# changes what is spawned, so "the memory cap bites" and "the wall clock bites"
175# are claims about a program that is no longer the program. They have to be
176# re-established, not inherited. The first bundled gate run proved the point by
177# failing an arm that had been passing for a reason that had quietly stopped
178# being true.
179export F2B_SHAPE="${F2B_SHAPE:-dev}"
180echo "sabotage-f2b: SHAPE UNDER TEST = $F2B_SHAPE"
182# THE RUNTIME IS A SHARED, MOVING INPUT. RECORD IT AT BOTH ENDS.
184# `dev-redirects.sgl` resolves the sigil runtime to the sibling `../sigil`
185# CHECKOUT, not to a pin, and other people merge into it. Every leg rebuilds, so
186# a merge landing mid-run means earlier legs linked a different runtime than
187# later ones -- and the transcript would look perfectly normal.
189# That happened: a run straddled cb8d563f -> 2e9d23c0 (three commits, one of them
190# a GC hot-path perf change) and was only detectable because the sha had been
191# captured by hand outside the suite. The whole set had to be discarded.
193# So the suite now samples the sha at start AND end and says plainly whether its
194# own results are attributable. A run that cannot attribute itself should not be
195# quoted, and it now says so in its own output rather than in someone's shell
196# history.
197# EVERY redirect target, not just `../sigil`.
199# The first version of this check watched the sigil runtime alone, and that was
200# luck rather than care: `../sigil-doc` -- the PARSER this whole phase depends on
201# -- moved nine commits including a D8 feature change on the same afternoon, and
202# nothing here would have noticed. It happened to land 15 minutes before a run
203# rather than during one.
205# So the fingerprint is derived from `dev-redirects.sgl` itself: every
206# `from-path` target that is a git checkout is sampled. Deriving it means a
207# redirect added later is covered without anyone remembering to add it here,
208# which is the failure mode a hand-listed pair would have.
209redirect_fingerprint () {
210 for d in $(sed -n 's/.*from-path dir: "\([^"]*\)".*/\1/p' "$redirects" | sort -u); do
211 case "$d" in
212 /*) target="$d" ;;
213 *) target="$repo_dir/$d" ;;
214 esac
215 if [ -d "$target/.git" ] || [ -f "$target/.git" ]; then
216 printf '%s=%s\n' "$(basename "$target")" \
217 "$(git -C "$target" rev-parse --short HEAD 2>/dev/null || echo UNKNOWN)"
218 fi
219 done
221runtime_sha_start=$(redirect_fingerprint | tr '\n' ' ')
222[ -n "$runtime_sha_start" ] || { echo "SETUP-FAILED: no redirect targets resolved from $redirects; the attribution check would compare two empty strings and always pass" >&2; exit 3; }
223echo "sabotage-f2b: redirect targets (START) = $runtime_sha_start"
225# ---------------------------------------------------------------------------
226# PREFLIGHT: EVERY ARM MUST PASS ALONE ON A CLEAN TREE.
228# A PERMANENT PROPERTY OF THIS SUITE, not a one-off repair.
230# Each leg below invokes a SUBSET of the gate's arms and reads a non-zero exit as
231# "the plant was detected". That inference is only valid if the selected arms
232# pass on an unsabotaged tree. They did not: arms 2, 3 and 10 greped files that
233# only arms 1 and 5 created, so `gate-f2b.sh 3` failed on a clean tree and legs
234# 1, 2 and 8 reported "RED as required" -- with plausible `GATE-F2B FAIL` text --
235# WHETHER OR NOT THEIR PLANT DID ANYTHING. Three legs of void evidence that read
236# exactly like three legs of good evidence.
238# This preflight makes that class impossible to reintroduce: if an arm ever again
239# depends on another arm's leftovers, the suite refuses to run rather than
240# reporting confident nonsense. It is checked for every arm any leg uses, not
241# just the three that were broken.
242# ---------------------------------------------------------------------------
243echo
244echo "PREFLIGHT: every arm used by a leg must pass ALONE on this clean tree"
246# The list is DERIVED FROM THE LEG CALLS rather than written out, so adding a leg
247# that uses a new arm automatically extends the preflight. A hand-maintained
248# list would drift, and the drift would be silent in exactly the direction that
249# matters. If the derivation ever yields nothing, that is SETUP-FAILED, not an
250# empty-and-therefore-passing loop -- a for-loop over nothing succeeds, which is
251# how a preflight becomes decorative.
252preflight_arms=$(grep '^leg "' "$0" \
253 | sed 's/^leg "[^"]*"[[:space:]]*//; s/[[:space:]]*\\$//; s/"//g' \
254 | tr ' ' '\n' | grep -E '^[0-9]+$' | sort -un)
255[ -n "$preflight_arms" ] || { echo "SETUP-FAILED: could not derive the arm list from this script's leg calls; the preflight would silently check nothing" >&2; exit 3; }
256echo " arms used by legs: $(echo "$preflight_arms" | tr '\n' ' ')"
257pref_bad=0
258for a in $preflight_arms; do
259 if sh test/gate-f2b.sh "$a" > "$work/pref-$a.log" 2>&1; then
260 echo " arm $a alone on a clean tree: PASS"
261 else
262 echo " arm $a alone on a clean tree: FAIL -- this arm cannot produce its own subject," >&2
263 echo " so any leg invoking it would report RED regardless of its plant." >&2
264 grep -E 'GATE-F2B (FAIL|SETUP-FAILED)' "$work/pref-$a.log" | head -3 | sed 's/^/ /' >&2
265 pref_bad=$((pref_bad + 1))
266 fi
267done
268if [ "$pref_bad" -ne 0 ]; then
269 echo "SABOTAGE-F2B ABORTED: $pref_bad arm(s) fail on a clean tree. No leg below would mean anything." >&2
270 exit 3
271fi
272echo "PREFLIGHT OK: all arms used by legs pass standalone"
274# --- LEG 1 -----------------------------------------------------------------
275# F2's sabotage list, verbatim: "silently prefer `.sgd` duplicate over `.md`".
276# Make `find-collisions` never report a collision, which is exactly what
277# silently preferring one side looks like from the outside.
278leg "discovery silently swallows the .sgd/.md collision" 3 \
279 src/folio/doc/discovery.sgl \
280 's/(if (> (length group) 1)/(if (> (length group) 99999)/'
282# --- LEG 2 -----------------------------------------------------------------
283# Conflate "not seen" with "nothing to see": claim completeness unconditionally.
284# This is the defect that makes a scan of one readable directory out of nine
285# report "0 skipped" and read as exhaustive.
286leg "discovery calls every census COMPLETE, hiding unreadable directories" 2 \
287 src/folio/doc/discovery.sgl \
288 's/(null? (census-record-unreadable-dirs c)))/(null? (list)))/'
290# --- LEG 3 -----------------------------------------------------------------
291# Remove the memory cap from the worker shell -- F2 clause 2's named sabotage,
292# "remove the bounds -> the arm must go red by observing worker death contained,
293# not service death".
294leg "the bounded worker stops applying its memory cap" "4 5" \
295 src/folio/doc/bounded.sgl \
296 's/ulimit -v \\"\$cap\\" || exit 201; //'
298# --- LEG 4 -----------------------------------------------------------------
299# THE TRUST-BOUNDARY DEFECT ITSELF: treat `parse-error-unlocated` as a broken
300# document. This is the one that would mark healthy notes read-only whenever the
301# host is under memory pressure.
302leg "parse-error-unlocated is treated as a broken document" 7 \
303 src/folio/doc/bounded.sgl \
304 's/(unlocated? (pair? unlocated-lines))/(unlocated? #f)/'
306# --- LEG 5 -----------------------------------------------------------------
307# Drop the completion-marker requirement, so a truncated worker run scores as a
308# clean parse. This is the free-pass absence claim in its purest form.
309leg "a truncated worker run is accepted as a clean parse" 6 \
310 src/folio/doc/bounded.sgl \
311 's/((not done?)/((and #f (not done?))/'
313# --- LEG 6 -----------------------------------------------------------------
314# Emit uppercase hex. `valid-revision?` refuses uppercase precisely because a
315# case difference between two clients is a stale-write check that fails open on
316# some clients and not others -- so this must not survive the conformance arm.
317leg "revisions are emitted in uppercase hex" 8 \
318 src/folio/doc/revision.sgl \
319 's/(char-downcase (string-ref s i))/(char-upcase (string-ref s i))/'
321# --- LEG 7 -----------------------------------------------------------------
322# Report Markdown notes as `indexed` -- claiming the .sgd inspector successfully
323# inspected a file it never looked at. The counts stay internally consistent and
324# the denominator still balances; the population is simply described wrongly,
325# which is the version of this defect that survives review.
327# (An earlier spelling of this leg wrapped the map in a filter and did not
328# compile, which yields only a BUILD red -- weaker evidence, because it proves the
329# compiler catches it rather than that the gate discriminates.)
330leg "markdown notes are reported as successfully indexed" 5 \
331 src/folio/doc/service.sgl \
332 "s/status: 'not-inspectable/status: 'indexed/"
334# --- LEG 8 -----------------------------------------------------------------
335# Collapse the workspace-level duplicate-id report, so two notes carrying one id
336# are silently tolerated.
337leg "workspace-level duplicate ids are not reported" 10 \
338 src/folio/doc/service.sgl \
339 's/(if (> (length paths) 1)/(if (> (length paths) 99999)/'
341# --- LEG 9 -----------------------------------------------------------------
342# Widen the memory-signature search back over ALL of stdout, which is
343# DOCUMENT-CONTROLLED. This is the defect an adversarial pass found in the first
344# version of this module: a healthy note whose task id is the string
345# "out of memory" is then classified unindexed/memory-cap and never indexed.
347# Arm 7 gained a probe for exactly this, so the leg has something to hit.
348leg "the memory-signature search reads document-controlled output" 7 \
349 src/folio/doc/bounded.sgl \
350 's/(memory? (or (memory-signature? se)/(memory? (or (memory-signature? so) (memory-signature? se)/'
352# --- LEG 10 ----------------------------------------------------------------
353# Treat the completion marker as a SUBSTRING rather than a whole line, so a
354# document carrying the marker text in a node id can certify its own truncated
355# run. This is a DIFFERENT defect from leg 5: leg 5 removes the marker rule
356# entirely, this one leaves the rule in place and makes it forgeable by document
357# content. Arm 6's forgery probe is what distinguishes them.
358leg "the completion marker is matched as a substring, so a document can forge it" 6 \
359 src/folio/doc/bounded.sgl \
360 's/(has-exact-line? lines "F2B_WORKER_DONE")))/(contains? so "F2B_WORKER_DONE")))/'
362# ---------------------------------------------------------------------------
363restore_all
364"$sigil" build --redirects "$redirects" >/dev/null 2>&1
366echo
367echo "=========================================================================="
368echo "RESTORE VERIFICATION (by hash, not by git reporting success)"
369restore_bad=0
370for f in $TOUCHED; do
371 want=$(cat "$base/$(echo "$f" | tr / _).base")
372 got=$(sha256sum "$f" | cut -d' ' -f1)
373 if [ "$want" = "$got" ]; then
374 echo " OK $f"
375 else
376 echo " DIRTY $f -- expected $want got $got"
377 restore_bad=$((restore_bad + 1))
378 fi
379done
381echo "=========================================================================="
382# ATTRIBUTION CHECK -- does this run describe one runtime, or a boundary?
383runtime_sha_end=$(redirect_fingerprint | tr "\n" " ")
384echo "sabotage-f2b: redirect targets (END) = $runtime_sha_end"
385straddled=0
386if [ "$runtime_sha_start" != "$runtime_sha_end" ]; then
387 straddled=1
388 echo "=========================================================================="
389 echo "SABOTAGE-F2B UNATTRIBUTABLE: a redirected dependency MOVED during this run."
390 echo " start: $runtime_sha_start"
391 echo " end: $runtime_sha_end"
392 echo "Every leg rebuilds, so earlier legs linked a different runtime than later"
393 echo "ones. These results describe a BOUNDARY, not a build. DO NOT QUOTE THEM --"
394 echo "re-run against a held sha. This is not a product failure."
395fi
397echo "=========================================================================="
398if [ "$straddled" -ne 0 ]; then
399 echo "SABOTAGE-F2B: results discarded (runtime moved mid-run); $bad leg(s) failed to go red before that."
400 exit 2
401elif [ "$bad" -eq 0 ] && [ "$restore_bad" -eq 0 ]; then
402 echo "SABOTAGE-F2B: all $legs legs went RED as required; tree restored clean."
403 echo " attributable to: $runtime_sha_start (shape $F2B_SHAPE)"
404 exit 0
405else
406 echo "SABOTAGE-F2B: $bad leg(s) failed to go red, $restore_bad file(s) not restored."
407 echo "A leg that stayed green is a GATE DEFECT and must be fixed before the gate's green means anything."
408 exit 1
409fi