AtlatestRepositorysigil-doc

sigil-doc / tree / testsabotage-sd-p8.sh

1#!/usr/bin/env bash
2# SD-P8 sabotage: prove `test/gate-sd-p8.sh` and the pinned literals it guards
3#
4# WHY EVERY LEG HERE IS NEEDED, stated rather than assumed.
5#
6# SD-P8 rewrote pinned diagnostic literals -- `@chapter` became `@article` in
7# two shipped messages and a third language's name appeared. A migration that
8# updates a pinned literal to match the new output proves NOTHING: the pin and
9# the product moved together and neither was ever tested against the other. So
10# every literal this phase changed has a leg below that perturbs the PRODUCT --
11# the profile, the derivation, the multiplicity -- and requires the pin to
12# redden. No leg edits an assertion to make it fail.
14# The one exception is deliberate and labelled: LEG 06 sabotages the
15# INSTRUMENT (the differential's normalization), because an instrument carries
16# the same defect rate as the product and gets none of the scrutiny.
18# EVERY LEG RESTORES BY HASH. A leg that restored the source but left the tree
19# subtly different would poison every later leg, and the last leg -- the
20# positive control -- is what proves the tree came back.
21set -euo pipefail
22repo_dir="$(cd "$(dirname "$0")/.." && pwd)"
23cd "$repo_dir"
24work=$(mktemp -d "${TMPDIR:-/tmp}/sd-p8-sabotage.XXXXXX")
26PARSER=src/sigil/doc/private/parser.sgl
27BOOK=src/sigil/doc/book/reader.sgl
28MODEL=src/sigil/doc/model.sgl
29DUMP=test/sd-p8-lowering-dump.sgl
30PROBE=test/sd-p8-language-probe.sgl
31FIXTURE=test/fixtures/minimal.sgd
33die() { echo "$*" >&2; exit 1; }
34# The SAME compiler the gate uses, named rather than inherited from PATH: a
35# sabotage driven by a different binary than the gate measures two things at
36# once.
37compiler="${SIGIL_D1_COMPILER:-$repo_dir/../task-sigil-language-readers/build/dev/bin/sigil}"
38sxml_root="${SIGIL_SXML_ROOT:-$repo_dir/../sigil-sxml}"
40for tool in rg sed awk sha256sum cmp git; do
41 command -v "$tool" >/dev/null || die "$tool is required by this sabotage script"
42done
43test -x "$compiler" || die "dev compiler not found at $compiler"
44test -d "$sxml_root/src" || die "sigil-sxml source not found at $sxml_root/src"
46# ------------------------------------------------------------- plant/restore
48# RESTORE FROM A SAVED COPY, NOT FROM `git checkout`.
50# The first version of this script restored with
51# `git checkout -- $PARSER $BOOK ... $DUMP $PROBE $FIXTURE`, and three of those
52# files were UNTRACKED at the time. `git checkout` refuses the whole invocation
53# on an unknown pathspec, so it restored NOTHING and said so on stderr while the
54# script carried on. The sabotaged parser survived leg 1, and the next probe run
55# in that worktree reported a diagnostic naming `@chapter` on a tree whose
56# source says `@article` -- a result that looks exactly like a stale build cache
57# and is not one. Found 2026-08-22, and it is the reason this block is longer
58# than it looks like it needs to be:
60# * a copy under $work is authoritative, so tracked and untracked files
61# restore the same way and a file's git status cannot change the outcome;
62# * every restore is verified BY HASH against that copy, per file, and a
63# mismatch is fatal rather than a warning;
64# * the tree is required CLEAN of these files before anything is planted, so
65# the copies are known-good.
67# A sabotage script that cannot restore is worse than no sabotage script: it
68# leaves a poisoned worktree behind and every later measurement in it is a lie.
69for f in "$PARSER" "$BOOK" "$MODEL" "$DUMP" "$PROBE" "$FIXTURE"; do
70 test -f "$f" || die "$f does not exist; refusing to sabotage a tree that is not the one this script was written for"
71 if git ls-files --error-unmatch "$f" >/dev/null 2>&1; then
72 git diff --quiet -- "$f" \
73 || die "$f has uncommitted changes; refusing to sabotage over them"
74 fi
75 cp "$f" "$work/copy-$(echo "$f" | tr / _)"
76 sha256sum "$f" | awk '{print $1}' >"$work/pristine-$(echo "$f" | tr / _)"
77done
79pristine_hash() { cat "$work/pristine-$(echo "$1" | tr / _)"; }
80pristine_copy() { echo "$work/copy-$(echo "$1" | tr / _)"; }
82plant() { # $1 = file, $2 = pattern to replace, $3 = replacement
83 local before after matches
84 before=$(sha256sum "$1" | awk '{print $1}')
85 test "$before" = "$(pristine_hash "$1")" || die "plant: $1 is not pristine"
86 # UNIQUENESS FIRST. `sed` replaces the first match on every matching line and
87 # says nothing about how many there were, so a pattern that drifted into
88 # matching two sites would sabotage more than the leg describes -- and a leg
89 # whose red cannot be attributed to one edit is not evidence.
90 matches=$(grep -c -- "$2" "$1" || true)
91 test "${matches:-0}" -eq 1 \
92 || die "plant: '$2' matches ${matches:-0} lines of $1; a sabotage must name exactly one site"
93 sed -i "s|$2|$3|" "$1"
94 after=$(sha256sum "$1" | awk '{print $1}')
95 test "$before" != "$after" \
96 || die "plant: $1 did not change -- the sabotage never landed, so a red below would mean nothing"
99# RETURNS non-zero rather than calling `die`. `die` ends in `exit`, and an
100# `exit` inside a function still ends the SHELL -- so an `if ! restore_all`
101# guard around a dying function can never take its own failure branch. That is
102# the same class of mistake as the `git checkout` one above: a recovery path
103# that cannot run.
104restore_all() {
105 local f h failed=0
106 for f in "$PARSER" "$BOOK" "$MODEL" "$DUMP" "$PROBE" "$FIXTURE"; do
107 cp "$(pristine_copy "$f")" "$f" || { echo "restore: could not copy back $f" >&2; failed=1; continue; }
108 h=$(sha256sum "$f" | awk '{print $1}')
109 if [ "$h" != "$(pristine_hash "$f")" ]; then
110 echo "restore: $f came back as $h, not $(pristine_hash "$f")" >&2
111 failed=1
112 fi
113 done
114 return $failed
116# THE TRAP IS LOUD. `restore_all || true` would hide exactly the failure that
117# motivated the block above: a restore that did not happen, followed by a quiet
118# exit and a poisoned worktree. If the restore fails here there is nothing left
119# to do but say so as clearly as possible, and NOT delete the copies that are
120# the only way back.
121on_exit() {
122 local rc=$?
123 if ! restore_all; then
124 echo "!!! THE WORKTREE IS STILL SABOTAGED AND COULD NOT BE RESTORED." >&2
125 echo "!!! Pristine copies are kept at $work -- do NOT delete it." >&2
126 echo "!!! Restore by hand and verify by hash before trusting anything measured here." >&2
127 exit 2
128 fi
129 rm -rf "$work"
130 exit $rc
132trap on_exit EXIT
133trap 'exit 130' INT TERM
135# ------------------------------------------------------------------ runners
137# The gate's own exit status is not the evidence -- its MESSAGE is. A leg that
138# only checked for a non-zero exit would score a gate that died in `mktemp` as
139# a successful sabotage.
140run_gate() { # $1 = log
141 set +e
142 SD_P8_SKIP_REGRESSION=1 bash test/gate-sd-p8.sh >"$1" 2>&1
143 local rc=$?
144 set -e
145 sed -i 's/\x1b\[[0-9;]*m//g' "$1"
146 return $rc
149# A COLD CACHE PER RUN, and it is not hygiene theatre. The shared
150# `~/.cache/sigil` serves previously-compiled modules, so a sabotage that edits
151# a source file can be defeated by the cache and report a convincing
152# red-that-wasn't -- or, worse here, a green-that-should-have-been-red. The
153# isolation is what makes each leg a measurement of the edit.
154run_suite() { # $1 = log
155 set +e
156 XDG_CACHE_HOME="$work/cache-$(basename "$1")" \
157 "$compiler" -L "$sxml_root/src" -L src -L test -L book \
158 test test/test-root-profile.sgl >"$1" 2>&1
159 local rc=$?
160 set -e
161 sed -i 's/\x1b\[[0-9;]*m//g' "$1"
162 return $rc
165# `%s`, NOT `%02d`, AND THE REASON IS WORTH THE LINE.
167# Callers pass ZERO-PADDED labels (`01` … `09`). `printf '%02d' 08` in bash is
168# "invalid octal number": it prints `00` and RETURNS NON-ZERO. Under `set -e`
169# that killed this suite immediately after leg 8 printed a MISLABELLED marker
170# (`SD_P8_SABOTAGE_00`), so leg 9 -- the positive control, the only leg that
171# proves the gate can stand DOWN -- never ran.
173# The shape is worth more than the fix: the call worked for legs 01-07 because
174# those are valid octal, and failed for exactly 08 and 09. A formatting bug that
175# is correct for seven of nine inputs and wrong for the two that matter most is
176# not going to be found by reading it.
177leg_red() { # $1 = leg label, $2 = log, $3 = required substring
178 rg -qF -- "$3" "$2" || {
179 echo "LEG $1: expected a red naming '$3'; got:" >&2
180 tail -25 "$2" >&2
181 exit 1; }
182 printf 'SD_P8_SABOTAGE_%s RED %s\n' "$1" "$3"
185# ---------------------------------------------------------------- LEG 01
186# PRODUCT. Put the base root set back to `(chapter)`.
188# This is the single edit D8 is: if the pinned literals in
189# `test/test-root-profile.sgl` cannot see it, they are decoration.
190echo "=== LEG 01 -- the base profile reverts to (chapter)"
191plant "$PARSER" \
192 '(make-document-profile "sigil/doc" .(article) .one)' \
193 "(make-document-profile \"sigil/doc\" '(chapter) 'one)"
194run_suite "$work/leg01.log" && die "LEG 01: the suite passed with the base root set reverted"
195leg_red 01 "$work/leg01.log" "the base profile is D8's policy and D8's diagnostics, exactly"
196rg -qF 'exactly one @chapter' "$work/leg01.log" \
197 || die "LEG 01: the suite reddened, but not on the derived diagnostic; the pin may be matching something else"
198restore_all || die "restore failed; the worktree is still sabotaged"
200# ---------------------------------------------------------------- LEG 02
201# PRODUCT. Give the genre language the BASE language's name.
203# The `sigil/doc/book:` prefix is what sends an author to the right
204# documentation, and it comes from one string in one place.
206# EXPECT THE RED AT ARM 01, NOT AT ARM 04. Arm 1 reads both profiles' name,
207# roots and multiplicity through the public modules, so it catches every
208# PROFILE-LEVEL sabotage before anything downstream runs. That layering was not
209# obvious until this leg was run: the first version of it predicted a red at arm
210# 4 and got one at arm 1, which is the expectation being wrong rather than the
211# gate. Recorded rather than quietly renumbered -- an arm that reddens somewhere
212# unpredicted is a finding either way
213# (topics/an-expectation-names-which-check-must-fail).
214echo "=== LEG 02 -- the genre profile reports the base language's name"
215plant "$BOOK" \
216 '(make-document-profile "sigil/doc/book" .(chapter) .one)' \
217 "(make-document-profile \"sigil/doc\" '(chapter) 'one)"
218run_gate "$work/leg02.log" && die "LEG 02: the gate passed with the genre language misnamed"
219leg_red 02 "$work/leg02.log" "arm 01: the genre profile is not D8's"
220restore_all || die "restore failed; the worktree is still sabotaged"
222# ---------------------------------------------------------------- LEG 03
223# PRODUCT, AND IT HAS TO REACH PAST ARM 01.
225# The point is to make a REQUIRED RED go green -- the failure mode a pass/fail
226# count cannot see -- and to do it where arm 1 is blind. Widening the profile's
227# multiplicity would not work: arm 1 asserts `mult=one` for both languages and
228# would catch it first, exactly as leg 2 just showed. So this sabotages the
229# ENFORCEMENT instead, in `parse-document-parts-with-profile`, where `several?`
230# gates whether a second root is accepted. Both profiles still SAY `one`; the
231# parser stops meaning it.
233# That is a strictly better sabotage than the one it replaces: it separates
234# "the profile is configured correctly" from "the parser obeys the profile",
235# and only the second is what the required reds are about.
236echo "=== LEG 03 -- the parser stops enforcing multiplicity, while both profiles still say one"
237plant "$PARSER" \
238 '(or several? (null? root-nodes))' \
239 '(or #t (null? root-nodes))'
240run_gate "$work/leg03.log" && die "LEG 03: the gate passed with multiplicity unenforced"
241leg_red 03 "$work/leg03.log" "arm 04: not every case matched its expected verdict"
242rg -qF 'want=refuse got=accept MISMATCH' "$work/leg03.log" \
243 || { rg -n 'SD_P8_LANG' "$work/leg03.log" >&2
244 die "LEG 03: the gate reddened, but no required-refusal case flipped to accept"; }
245restore_all || die "restore failed; the worktree is still sabotaged"
247# ---------------------------------------------------------------- LEG 04
248# PRODUCT. Put the dropped kind back.
249echo "=== LEG 04 -- the model accepts kind book again"
250plant "$MODEL" \
251 "(memq kind '(article chapter))" \
252 "(memq kind '(article book chapter))"
253run_gate "$work/leg04.log" && die "LEG 04: the gate passed with kind book restored"
254leg_red 04 "$work/leg04.log" "arm 02: make-document did not refuse the dropped kind by name"
255restore_all || die "restore failed; the worktree is still sabotaged"
257# ---------------------------------------------------------------- LEG 05
258# PRODUCT, AND THE ONE THE DIFFERENTIAL EXISTS FOR.
260# Change a migrated manuscript in a way that is NOT a header shift. Arm 3 has so
261# far only ever seen the difference it expects; a differential that has only seen
262# the expected difference has not been shown to detect an unexpected one. The
263# edit is one word of prose in a manuscript whose header this phase moved, so it
264# is precisely the change the header normalization must NOT absorb.
265echo "=== LEG 05 -- a migrated manuscript gains a word of prose"
266plant "$FIXTURE" 'A small @em{useful} document' 'A small @em{useful} sabotaged document'
267run_gate "$work/leg05.log" && die "LEG 05: the gate passed with a migrated manuscript's content changed"
268leg_red 05 "$work/leg05.log" "manuscripts changed for a reason this phase did not intend"
269rg -qF 'UNEXPECTED minimal.sgd' "$work/leg05.log" \
270 || { rg -n 'UNEXPECTED|IDENTICAL minimal' "$work/leg05.log" >&2
271 die "LEG 05: arm 5 reddened, but did not name minimal.sgd as the manuscript that moved"; }
272restore_all || die "restore failed; the worktree is still sabotaged"
274# ---------------------------------------------------------------- LEG 06
275# INSTRUMENT. Remove the differential's `make-srcloc` normalization.
277# The normalization is what makes arm 5's equality claim possible, and an
278# instrument gets none of the scrutiny the product does. Without it, every
279# manuscript whose header moved differs in its embedded position DATA -- so the
280# gate must report a wall of UNEXPECTED rather than quietly passing.
281echo "=== LEG 06 -- the differential stops normalizing embedded positions"
282plant "$DUMP" \
283 '(and (pair? value)' \
284 '(and #f (pair? value)'
285run_gate "$work/leg06.log" && die "LEG 06: the gate passed with the position normalization disabled"
286# EXPECTED AT A DIFFERENT ASSERTION THAN LEG 05, and that is the distinguishing
287# evidence rather than a nuisance.
289# The first version of this leg predicted the same string LEG 05 produces
290# ("manuscripts changed for a reason this phase did not intend"), which would
291# have made the two legs indistinguishable by their own output. Run, it reddened
292# somewhere else entirely -- and the somewhere else is better:
294# LEG 05 changes ONE manuscript's prose. 16 of 17 still lower identically, so
295# the identical-floor holds and the arm dies on `unexpected != 0`,
296# naming minimal.sgd.
297# LEG 06 disables the normalization. EVERY header-migrated manuscript's
298# embedded positions go absolute at once, so ZERO lower identically and
299# the arm dies on its FLOOR first, before the unexpected check is
300# reached.
302# So the two legs fail at two different assertions in the same arm, which is a
303# stronger separation than two counts of the same one. An arm reddening
304# somewhere unpredicted is a finding either way
305# (topics/an-expectation-names-which-check-must-fail); this is that rule paying
306# out on the leg written to satisfy it.
307leg_red 06 "$work/leg06.log" "manuscripts lower identically; the migration moved more than the header"
308# CORROBORATION: the wall, not a row. Under this sabotage the header shift leaks
309# into every migrated manuscript at once.
310unexpected_rows=$( { rg -c '^UNEXPECTED' "$work/leg06.log" || echo 0; } )
311test "${unexpected_rows:-0}" -ge 10 \
312 || { rg '^UNEXPECTED|^IDENTICAL' "$work/leg06.log" >&2
313 die "LEG 06: disabling the normalization produced only ${unexpected_rows:-0} unexpected manuscripts; a header-shift leak should move every migrated one, so this red is probably not the one this leg is about"; }
314rg -qF 'UNEXPECTED minimal.sgd' "$work/leg06.log" \
315 || die "LEG 06: the wall of unexpected manuscripts does not include minimal.sgd"
316# ...and the ROOT-ONLY rows must SURVIVE, which is what proves the sabotage hit
317# the position normalization specifically and not the comparison as a whole.
318rg -qF 'ROOT-ONLY sd-p3-wrapped.sgd' "$work/leg06.log" \
319 || die "LEG 06: the root-only classification collapsed too, so this sabotage is broader than the normalization it claims to disable"
320printf 'SD_P8_SABOTAGE_06 CORROBORATED %s manuscripts unexpected and the root-only rows intact, at the identical-floor rather than leg 5 unexpected-check\n' "$unexpected_rows"
321restore_all || die "restore failed; the worktree is still sabotaged"
323# ---------------------------------------------------------------- LEG 07
324# COVERAGE, not correctness. Delete cases from the language probe.
326# Every assertion in arm 4 is about an INDIVIDUAL case, so a probe whose cases
327# were deleted satisfies every one of them and prints `matched N mismatched 0`.
328# Only the denominator can see that, and this is the leg that proves the
329# denominator is real.
330echo "=== LEG 07 -- one of the probe's cases is deleted"
331plant "$PROBE" \
332 '(list "book-refuses-article"' \
333 ';; (list "book-refuses-article"'
334run_gate "$work/leg07.log" && die "LEG 07: the gate passed with a probe case deleted"
335leg_red 07 "$work/leg07.log" "arm 04: the probe declares only 5 cases"
336restore_all || die "restore failed; the worktree is still sabotaged"
338# ---------------------------------------------------------------- LEG 08
339# PRODUCT, AND IT IS THE ONE THE BRIEF ASKED FOR BY NAME.
341# SD-P8 rewrote pinned diagnostic literals. The risk the brief named is a
342# migration that updates a pin to match new output, leaving pin and product
343# moved together and neither tested against the other. Leg 1 attacks that from
344# the PROFILE side; this leg attacks it from the DERIVATION side, which is where
345# a reword would actually come from -- someone tidying the message formatter.
347# Arm 1 is blind to it (both profiles still report the right name, roots and
348# multiplicity) and so is arm 5 (the lowering of a legal manuscript does not
349# change). Only the pinned literals can see it, which is the whole point.
350echo "=== LEG 08 -- the derived misplaced-root message is reworded"
351plant "$PARSER" \
352 '": top-level author forms must precede "' \
353 '": top-level author forms must come before "'
354run_gate "$work/leg08.log" && die "LEG 08: the gate passed with a shipped diagnostic reworded"
355leg_red 08 "$work/leg08.log" "arm 04: the base language did not refuse @chapter with its own diagnostic"
356restore_all || die "restore failed; the worktree is still sabotaged"
357# ...and the SUITE's pins see it too, which is what makes them pins rather than
358# decoration. Run separately because the gate stops at its first failing arm.
359plant "$PARSER" \
360 '": top-level author forms must precede "' \
361 '": top-level author forms must come before "'
362run_suite "$work/leg08-suite.log" && die "LEG 08: the suite passed with a shipped diagnostic reworded"
363rg -qF "the base profile is D8's policy and D8's diagnostics, exactly" "$work/leg08-suite.log" \
364 || { tail -25 "$work/leg08-suite.log" >&2; die "LEG 08: the suite reddened somewhere other than the base pins"; }
365rg -qF "the book genre language is a sibling profile with its own diagnostics" "$work/leg08-suite.log" \
366 || { tail -25 "$work/leg08-suite.log" >&2; die "LEG 08: the GENRE language's pins did not see the reword"; }
367printf 'SD_P8_SABOTAGE_08 RED both languages pinned literals saw a reworded derivation\n'
368restore_all || die "restore failed; the worktree is still sabotaged"
370# ---------------------------------------------------------------- LEG 09
371# THE POSITIVE CONTROL, and it is not a formality.
373# Eight reds prove the gate can fire. They say nothing about whether it can
374# STAND DOWN -- a gate wedged red by leg 1's restore having gone wrong would
375# score every leg above as a success. This is also what proves the tree came
376# back: `restore_all` checks hashes, and this checks behaviour.
377echo "=== LEG 09 -- the restored tree is green"
378restore_all || die "restore failed; the worktree is still sabotaged"
379# THE FULL GATE, with SD_P8_SKIP_REGRESSION deliberately UNSET. Every leg above
380# ran with it set; if this control ran the same way, nothing here would ever have
381# executed the regression arm and "green" would be a claim about five arms
382# described as six.
383set +e
384bash test/gate-sd-p8.sh >"$work/leg09.log" 2>&1
385control_rc=$?
386set -e
387sed -i 's/\x1b\[[0-9;]*m//g' "$work/leg09.log"
388test "$control_rc" -eq 0 || {
389 tail -40 "$work/leg09.log" >&2
390 die "LEG 09: the restored tree is NOT green; every red above is uninterpretable"; }
391rg -q '^SD_P8_GATE_GREEN$' "$work/leg09.log" \
392 || { tail -20 "$work/leg09.log" >&2; die "LEG 09: the gate exited 0 without printing its green marker"; }
393rg -q 'SD_P8_GATE_PARTIAL' "$work/leg09.log" \
394 && die "LEG 09: the control run skipped an arm; it does not establish a full green"
395for n in 01 02 03 04 05 06 07; do
396 rg -q "^SD_P8_GATE_$n " "$work/leg09.log" \
397 || die "LEG 09: arm $n printed no marker in the control run"
398done
399printf 'SD_P8_SABOTAGE_09 GREEN the restored tree passes all seven arms\n'
401echo "SD_P8_SABOTAGE_COMPLETE 8 reds (6 product, 1 instrument, 1 coverage) and 1 positive control"