AtlatestRepositorysigil-doc

sigil-doc / tree / testsabotage-sd-p1.sh

1#!/usr/bin/env bash
2# SD-P1 sabotage driver.
3#
4# Six product-layer legs, one assertion-layer leg, and one leg that asks
5# whether a check of this gate can fail at all. For each: plant the defect the
6# arm exists to catch, prove the arm goes RED, restore, and prove the
7# restoration by CONTENT HASH rather than by the exit code of the restore.
8#
9# LEG 1 IS THE PHASE'S FALSIFICATION. The §0 row names it, and it replaces the
10# withdrawn "run the new suite against the pre-SD-P1 parser" criterion: for a
11# phase that ADDS BINDINGS, that run fails at COMPILE time on absent names,
12# which is a green-for-the-wrong-reason rather than evidence about behaviour.
13# Leg 1 keeps every binding in place and removes only the BEHAVIOUR.
15# Three properties this driver asserts about ITSELF, because a sabotage run that
16# quietly did nothing is indistinguishable from a passing one:
17# * the edit must actually change the file (hash before != hash after), or the
18# leg is vacuous and the run aborts;
19# * the red must be an ASSERTION failure from a suite that compiled and ran --
20# the driver requires the runner's own "N passed, M failed" line, so a
21# compile error cannot be mistaken for a demonstrated red;
22# * restoration is verified by hash against a pristine copy taken up front.
23set -euo pipefail
24repo_dir="$(cd "$(dirname "$0")/.." && pwd)"
25cd "$repo_dir"
26sigil="${SIGIL_D1_COMPILER:-$repo_dir/../task-sigil-language-readers/build/dev/bin/sigil}"
27export SIGIL_REDIRECTS="${SIGIL_D1_REDIRECTS:-$repo_dir/dev-redirects.sgl}"
28mkdir -p "$repo_dir/build"
29work=$(mktemp -d "$repo_dir/build/sd-p1-sabotage.XXXXXX")
30export XDG_CACHE_HOME="$work/cache"
32command -v python3 >/dev/null || { echo "python3 is required by this driver" >&2; exit 1; }
33command -v rg >/dev/null || { echo "rg is required by this driver" >&2; exit 1; }
35parser=src/sigil/doc/private/parser.sgl
36reader=src/sigil/doc/reader.sgl
37suite=test/test-root-profile.sgl
38before=test/transcripts/sd-p1/classification-before.txt
40pristine="$work/pristine"
41mkdir -p "$pristine"
42for f in "$parser" "$reader" "$suite" "$before"; do
43 mkdir -p "$pristine/$(dirname "$f")"
44 cp "$f" "$pristine/$f"
45done
47hash_of() { sha256sum "$1" | cut -d' ' -f1; }
49restore() {
50 for f in "$parser" "$reader" "$suite" "$before"; do
51 cp "$pristine/$f" "$f"
52 test "$(hash_of "$f")" = "$(hash_of "$pristine/$f")" || {
53 echo "RESTORE FAILED for $f" >&2; exit 1; }
54 done
57# LEG 8 deliberately corrupts a COMMITTED artifact (the BEFORE transcript), and
58# an abort between the forge and the restore leaves that corruption in the
59# worktree. It happened: a leftover variable from a renumbering tripped `set -u`
60# at exactly that point. Restoration is therefore unconditional -- on success,
61# on failure, and on interrupt -- not a line at the end of each leg.
62trap restore EXIT INT TERM
64# apply <file> <old> <new> -- exact single-occurrence replacement, aborting if
65# the target is absent, ambiguous, or the edit no-ops.
66apply() {
67 local file="$1" old="$2" new="$3" before_hash after_hash
68 before_hash=$(hash_of "$file")
69 python3 - "$file" "$old" "$new" <<'PY'
70import sys
71path, old, new = sys.argv[1], sys.argv[2], sys.argv[3]
72text = open(path).read()
73n = text.count(old)
74if n != 1:
75 sys.exit("sabotage target appears %d times in %s; expected exactly 1" % (n, path))
76open(path, "w").write(text.replace(old, new))
77PY
78 after_hash=$(hash_of "$file")
79 test "$before_hash" != "$after_hash" || { echo "SABOTAGE DID NOT CHANGE $file" >&2; exit 1; }
82# The COLLECTED test count, measured from the leg runs below rather than
83# written in. The closing green check at the end of this driver needs a
84# denominator; a literal one goes stale the first time somebody adds a test,
85# and a stale denominator in a check is how a gate stops describing its
86# subject and becomes a second thing to keep in sync.
87collected_total=0
89# expect_suite_red <label> [<expected failing test name fragment>]
90expect_suite_red() {
91 local label="$1" expect="${2:-}" status log
92 # SPLIT DELIBERATELY. `local a="$1" b="$(... $a ...)"` expands EVERY
93 # assignment before `local` runs, so `$a` is still UNBOUND at that point.
94 # Under `set -u` this printed `label: unbound variable` and evaluated to
95 # `$work/.log` for every leg, so all seven legs overwrote one another's
96 # log and the per-leg evidence this driver is built around did not exist.
97 # It is in the committed transcript seven times
98 # (test/transcripts/sd-p1/sabotage.txt, reported at "line 84").
99 log="$work/$(printf '%s' "$label" | tr ' /' '__').log"
100 set +e
101 setsid -w "$sigil" -L src -L test test "$suite" >"$log" 2>&1
102 status=$?
103 set -e
104 sed -e 's/\x1b\[[0-9;]*m//g' "$log" >"$log.clean"
105 echo "--- $label: exit=$status"
106 # A red must come from a suite that COMPILED AND RAN. Without this the leg
107 # would accept a compile error as a demonstrated failure.
108 rg -q '[0-9]+ passed, [0-9]+ failed' "$log.clean" || {
109 echo "$label: no runner summary -- the suite did not run; this is NOT a demonstrated red" >&2
110 tail -40 "$log.clean" >&2; exit 1; }
111 # Record this run's collected count (passed + failed) for the closing check.
112 local summary p f
113 summary=$(rg -o '[0-9]+ passed, [0-9]+ failed' "$log.clean" | head -1)
114 p=${summary%% passed*}
115 f=${summary#*, }; f=${f%% failed*}
116 if [ $((p + f)) -gt "$collected_total" ]; then collected_total=$((p + f)); fi
117 test "$status" -ne 0 || { echo "$label: SABOTAGE DID NOT TURN THE SUITE RED" >&2; exit 1; }
118 rg -o '[0-9]+ passed, [0-9]+ failed' "$log.clean" | head -1
119 rg -o '✗ .*' "$log.clean" | head -6 || true
120 if [ -n "$expect" ]; then
121 # Anchor to the runner's FAILURE marker. Matching the test name anywhere in
122 # the log would also match the line that reports the same test PASSING, so
123 # this check would have confirmed "red at the expected arm" for a leg whose
124 # red came from somewhere else entirely.
125 rg -q "✗.*$expect" "$log.clean" || {
126 echo "$label: red, but NOT at the expected arm ($expect)" >&2
127 tail -40 "$log.clean" >&2; exit 1; }
128 fi
131echo "=============================================================="
132echo "LEG 1 (product) -- THE NAMED SABOTAGE: the root allowlist is ignored."
133echo "Every binding, every message, every location stays in place; only the"
134echo "membership test is removed. This is the falsification for the phase."
135apply "$parser" \
136 "(memq (car (car xs)) roots-allowed)" \
137 "(begin roots-allowed #t)"
138expect_suite_red "leg1-allowlist-ignored" "allowlist"
139restore
141echo "=============================================================="
142echo "LEG 2 (product) -- multiplicity is ignored; every profile accepts any"
143echo "number of roots."
144apply "$parser" \
145 "(or several? (null? root-nodes))" \
146 "(begin several? #t)"
147expect_suite_red "leg2-multiplicity-ignored" "multiplicity"
148restore
150echo "=============================================================="
151echo "LEG 3 (product) -- the DEFAULT profile is silently widened, so"
152echo "sigil/doc would start accepting a folio root."
153apply "$parser" \
154 "(make-document-profile \"sigil/doc\" '(chapter) 'one)" \
155 "(make-document-profile \"sigil/doc\" '(chapter note) 'one)"
156expect_suite_red "leg3-default-widened" "default profile"
157restore
159echo "=============================================================="
160echo "LEG 4 (product) -- spans are measured in SCALARS instead of UTF-8 BYTES."
161echo "This is the exact bug the byte-vs-scalar precondition on the location arm"
162echo "exists to make detectable."
163apply "$parser" \
164 "(- (position-at state end) start-byte)" \
165 "(- end start)"
166expect_suite_red "leg4-scalar-spans" "exact locations"
167restore
169echo "=============================================================="
170echo "LEG 5 (product) -- top-level classification reverts to the HEAD SYMBOL,"
171echo "so an author's @text{} command is mistaken for the parser's own prose"
172echo "pseudo-node again."
173apply "$parser" \
174 "(not (and (pseudo-kind? node 'text)" \
175 "(not (and (eq? (car node) 'text)"
176expect_suite_red "leg5-head-symbol-classification" "dispatched as author commands"
177restore
179echo "=============================================================="
180echo "LEG 6 (product) -- trim-body-edges reverts to the HEAD SYMBOL."
181echo "This is the site the first version of this phase MISSED, and the most"
182echo "dangerous of the family: it does not merely misread a @text{} command,"
183echo "it REBUILDS it as a five-slot prose node, so every later stage then"
184echo "behaves correctly on the wrong node. Fixing lower-block alone left the"
185echo "block-position case completely unchanged and looked like success."
186apply "$parser" \
187 " (define (left xs)
188 (cond
189 ((null? xs) '())
190 ((not (pseudo-kind? (car xs) 'text)) xs)" \
191 " (define (left xs)
192 (cond
193 ((null? xs) '())
194 ((not (eq? (caar xs) 'text)) xs)"
195expect_suite_red "leg6-trim-body-edges-head-symbol" "dispatched as author commands"
196restore
198echo "=============================================================="
199echo "LEG 7 (ASSERTION layer) -- one expected byte offset in the suite is"
200echo "perturbed by one. This proves the location assertions actually execute"
201echo "rather than sitting in a branch nothing reaches."
202apply "$suite" \
203 "(assert-equal (byte-offset source start) (srcloc-position loc))" \
204 "(assert-equal (+ 1 (byte-offset source start)) (srcloc-position loc))"
205expect_suite_red "leg7-assertion-layer" "exact locations"
206restore
208echo "=============================================================="
209echo "LEG 8 (instrument validity) -- CAN GATE ARM 2 FAIL AT ALL?"
210echo "Arm 2 asserts the classification change is confined: at least one"
211echo "colliding spelling moved, and every positive control is identical. Its"
212echo "INPUT is the committed BEFORE transcript. Here that input alone is"
213echo "neutered -- replaced with the CURRENT behaviour, control marker kept --"
214echo "while the gate's scoring and exit logic are left untouched. If arm 2"
215echo "still passes, it was passing for free."
216after_now="$work/after-now.txt"
217setsid -w "$sigil" -L src -L test eval -f test/sd-p1-classification-probe.sgl >"$after_now" 2>&1
218# DERIVED, not hardcoded. This exact guard shipped with a stale "8 cases" and
219# aborted leg 8 on a probe that had run all 9 correctly -- the fourth instance
220# of this hazard in one session. It failed SAFE (it refused to forge with input
221# it could not vouch for), but the fix is to read the count from the artifact.
222probe_cases=$(rg -c '^\(report "' test/sd-p1-classification-probe.sgl || echo 0)
223test "$probe_cases" -ge 1 || { echo "could not count the probe's cases" >&2; exit 1; }
224rg -q "SD_P1_CLASSIFICATION_PROBE_COMPLETE $probe_cases cases" "$after_now" || {
225 echo "could not produce a current transcript to neuter with" >&2
226 cat "$after_now" >&2; exit 1; }
227{ echo "PRE_SD_P1_CONTROL_UNBOUND (forged for leg 8 -- this transcript is NOT pre-SD-P1)"
228 cat "$after_now"; } >"$before"
229test "$(hash_of "$before")" != "$(hash_of "$pristine/$before")" || {
230 echo "leg 8 did not change the BEFORE transcript" >&2; exit 1; }
231set +e
232SD_P1_SKIP_REGRESSION=1 test/gate-sd-p1.sh >"$work/leg8-gate.log" 2>&1
233leg8_status=$?
234set -e
235echo "--- leg8-before-transcript-neutered: exit=$leg8_status"
236tail -5 "$work/leg8-gate.log"
237test "$leg8_status" -ne 0 || {
238 echo "LEG 8: arm 2 PASSED against a BEFORE transcript identical to the AFTER." >&2
239 echo "It cannot detect an unconfined change and is not a gate." >&2; exit 1; }
240rg -q 'the classification fix is inert' "$work/leg8-gate.log" || {
241 echo "LEG 8: the gate failed, but not at arm 2's confinement check" >&2
242 tail -30 "$work/leg8-gate.log" >&2; exit 1; }
243# Arm 1 must have passed before arm 2 failed, or the red came from somewhere else.
244rg -q 'SD_P1_GATE_01' "$work/leg8-gate.log" || {
245 echo "LEG 8: arm 1 did not pass, so the red is not attributable to arm 2" >&2; exit 1; }
246restore
248echo "=============================================================="
249echo "RESTORED -- every file back to its pristine hash. Re-running the suite"
250echo "green closes the loop: sabotage proves green->red, this proves red->green."
251for f in "$parser" "$reader" "$suite" "$before"; do
252 printf ' %s %s\n' "$(hash_of "$f")" "$f"
253done
255# THE CLOSING RUN ASSERTED NOTHING ABOUT ITS OUTPUT UNTIL 2026-08-22, AND IT IS
256# THE ONLY THING IN THIS DRIVER THAT PROVES RED->GREEN.
258# It was `setsid "$sigil" ... | sed ... | tail -8`, with no marker check of any
259# kind, and `SD_P1_SABOTAGE_COMPLETE 8 legs` printed on the next line.
261# WHAT WAS AND WAS NOT WRONG WITH IT. The first version of this comment blamed a
262# `setsid` race, a leader acted on that, and it was wrong -- so the correction
263# lives here rather than being quietly dropped.
265# NOT wrong: the exit status. `set -o pipefail` is on, so a suite exiting
266# non-zero would have aborted the script. And a PIPELINE blocks even when
267# `setsid` takes its fork branch, because the shell waits for EOF on the pipe
268# and the grandchild holds the write end. Measured on the forced fork branch:
269# 3020 ms for the piped form against a 3-second child, versus 10 ms for the
270# same command redirected to a file. This line was never racy. The racy shape
271# is redirect-to-a-file, which is a different set of call sites.
273# Wrong, and still disqualifying: NOTHING ABOUT THE CONTENT WAS CHECKED. A
274# runner that exited 0 having collected zero tests, or having printed nothing
275# at all, satisfied every condition this block imposed, and the completion
276# marker went out regardless. A step whose whole job is to prove red->green has
277# to read what the runner actually said.
279# It now asserts the RUNNER'S OWN WORDS from a log on disk, requires the run to
280# exit 0, and -- because an assertion that has never been shown to fire cannot
281# certify anything -- is armed against BOTH a truncated log and a known-good
282# summary before the real run happens.
283closing_log="$work/closing-green.log"
285# assert_suite_green <plain-log> <minimum collected count>
286# 0 = the log shows a complete green run; 1 = refuse, with the reason.
287assert_suite_green() {
288 local log="$1" floor="$2" passed total
289 rg -q 'All tests passed' "$log" || { echo " closing check: no 'All tests passed' banner" >&2; return 1; }
290 passed=$(rg -o 'Tests:[[:space:]]+[0-9]+ passed' "$log" | head -1 | grep -oE '[0-9]+' | head -1)
291 total=$(rg -o '\([0-9]+ total\)' "$log" | head -1 | grep -oE '[0-9]+' | head -1)
292 test -n "$passed" -a -n "$total" || { echo " closing check: the runner printed no summary line" >&2; return 1; }
293 # Self-consistency: every collected test must be a passing one. This needs no
294 # literal and cannot go stale.
295 test "$passed" = "$total" || { echo " closing check: $passed passed of $total collected" >&2; return 1; }
296 # ...and a FLOOR, because "0 passed (0 total)" is self-consistent too. A bound
297 # satisfied by a suite that collected nothing is not a bound.
298 #
299 # WHAT THIS FLOOR IS AND IS NOT. It is derived from the same suite it then
300 # bounds -- the maximum collected count observed across the legs above -- so it
301 # cannot go stale when a test is added, and it cannot become unsatisfiable
302 # (sabotage reduces or preserves collection, never increases it). It also
303 # cannot detect a suite that shrank: gut the file from 11 tests to 1 and every
304 # leg still reddens on the survivor, the floor becomes 1, and this reports
305 # "green over 1 collected tests". It catches total collapse to zero and nothing
306 # weaker. `test/gate-sd-p1.sh` arm 1 is what pins the absolute count.
307 test "$total" -ge "$floor" || {
308 echo " closing check: collected $total, but the legs above collected $floor" >&2; return 1; }
309 if rg -q '[1-9][0-9]* failed' "$log"; then
310 echo " closing check: the runner reported failures" >&2; return 1; fi
311 return 0
314test "$collected_total" -ge 1 || {
315 echo "no leg recorded a collected test count; the closing check has no denominator" >&2; exit 1; }
317# POSITIVE CONTROL 1 -- it must REFUSE the exact artifact the fork branch
318# produces. `Running tests...` and nothing else is the real 18-byte log from the
319# 2026-08-11 measurement.
320printf 'Running tests...\n' >"$closing_log.truncated"
321if assert_suite_green "$closing_log.truncated" "$collected_total" 2>/dev/null; then
322 echo "CLOSING CHECK CANNOT FAIL: it accepted a truncated log" >&2; exit 1; fi
323# POSITIVE CONTROL 2 -- and it must ACCEPT a known-good summary, or control 1
324# passes for free: a check that refuses everything refuses truncation too.
325printf 'Tests: %s passed (%s total)\nAll tests passed\n' "$collected_total" "$collected_total" \
326 >"$closing_log.control"
327assert_suite_green "$closing_log.control" "$collected_total" || {
328 echo "CLOSING CHECK REFUSES A KNOWN-GOOD SUMMARY -- it is a wall, not a check" >&2; exit 1; }
329# POSITIVE CONTROL 3 -- the floor must bite. A self-consistent empty run is the
330# shape a stalled or mis-filtered runner produces, and it must not be accepted.
331printf 'Tests: 0 passed (0 total)\nAll tests passed\n' >"$closing_log.empty"
332if assert_suite_green "$closing_log.empty" "$collected_total" 2>/dev/null; then
333 echo "CLOSING CHECK CANNOT FAIL: it accepted a run that collected 0 tests" >&2; exit 1; fi
334# POSITIVE CONTROL 4 -- the `passed == total` arm must bite on its own. Controls
335# 1-3 are all refused by the banner or the floor before this arm is reached, so
336# without this one the arm could be deleted and every control would still pass.
337printf 'Tests: %s passed (%s total)\nAll tests passed\n' \
338 "$((collected_total - 1))" "$collected_total" >"$closing_log.short"
339if assert_suite_green "$closing_log.short" "$collected_total" 2>/dev/null; then
340 echo "CLOSING CHECK CANNOT FAIL: it accepted a run where not every collected test passed" >&2
341 exit 1; fi
342# POSITIVE CONTROL 5 -- and so must the `N failed` arm, which controls 1-4 also
343# leave untouched: a log can be self-consistent, banner-carrying, at or above the
344# floor, AND still report failures.
345printf 'Tests: %s passed (%s total)\nAll tests passed\n2 failed\n' \
346 "$collected_total" "$collected_total" >"$closing_log.failing"
347if assert_suite_green "$closing_log.failing" "$collected_total" 2>/dev/null; then
348 echo "CLOSING CHECK CANNOT FAIL: it accepted a log that reported failures" >&2
349 exit 1; fi
350echo "closing check armed: refuses a truncated log, an empty run, a partial pass and a reported failure; accepts a green summary"
352set +e
353setsid -w "$sigil" -L src -L test test "$suite" >"$closing_log" 2>&1
354closing_status=$?
355set -e
356sed -e 's/\x1b\[[0-9;]*m//g' "$closing_log" >"$closing_log.clean"
357tail -8 "$closing_log.clean"
358test "$closing_status" -eq 0 || {
359 echo "CLOSING RUN EXITED $closing_status on RESTORED sources -- red->green is NOT demonstrated" >&2
360 tail -40 "$closing_log.clean" >&2; exit 1; }
361assert_suite_green "$closing_log.clean" "$collected_total" || {
362 echo "CLOSING RUN did not demonstrate red->green on restored sources" >&2
363 tail -40 "$closing_log.clean" >&2; exit 1; }
365echo "SD_P1_SABOTAGE_COMPLETE 8 legs, closing suite green over $collected_total collected tests"