AtlatestRepositoryfolio

folio / tree / testsabotage-f1b.sh

1#!/usr/bin/env bash
2#
3# F1b -- the sabotage driver. A gate that cannot demonstrate it fails is not a
4# gate; it has only been shown to be quiet.
5#
6# LEGS ARE COUNTED BY WHICH ARM THEY REDDEN, NOT BY HOW MANY THERE ARE.
7#
8# SD-P7 shipped a gate with four arms and twelve sabotage legs, which reads like
9# thorough coverage. Every one of those legs reddened arm 1. Its 42 KB
10# differential -- the arm that existed precisely to cover what the suite did not
11# -- had nothing establishing it could fail at all, and a large equality that
12# cannot go red is the most convincing vacuous gate there is: the bigger and more
13# precise the comparison, the more it reads as overwhelming evidence and the less
14# anyone asks whether it can move.
16# So this driver states coverage BY ARM, every leg names the arm it targets, and
17# every leg ASSERTS IT REACHED THAT ARM before accepting a red. A leg that
18# reddens the wrong arm is worse than no leg, because it looks like coverage in
19# the transcript.
21# EVERY LEG REQUIRES A NAMED SIGNATURE. A non-zero exit is NOT evidence of a
22# red: exit 127 (not found), 126 (not executable) and a usage error all end the
23# process badly while executing zero assertions, and all three have produced
24# convincing false reds in this estate.
26# RESTORE IS BY HASH, NEVER BY EXIT CODE, and by copy-aside rather than by
27# `git checkout --`. That command restores from the last COMMIT, so if any work
28# were uncommitted it would discard it -- silently, with exit 0. A restore
29# verified by exit status reports success on a wipe.
31# BOTH LAYERS ARE SABOTAGED. Breaking a test ASSERTION proves the harness
32# notices when the harness changes; breaking the PRODUCT proves the gate executes
33# the loaded product source rather than a mirror of itself. Only the second
34# establishes the property that makes a gate a gate -- a stale module, a fixture
35# that reimplements the behaviour, and a wrong load path are all invisible to
36# assertion-layer sabotage, and all three have occurred here.
37set -uo pipefail
39repo_dir="$(cd "$(dirname "$0")/.." && pwd)"
40cd "$repo_dir"
42sigil="${SIGIL_F1B_COMPILER:-sigil}"
43export SIGIL_REDIRECTS="${SIGIL_F1B_REDIRECTS:-$repo_dir/dev-redirects.sgl}"
44export XDG_CACHE_HOME="${XDG_CACHE_HOME:-$(mktemp -d)}"
46work=$(mktemp -d)
47backup="$work/backup"
48mkdir -p "$backup"
49legs_run=0
50legs_red=0
51failures=""
53say() { echo "$*"; }
55# ---- save / restore, verified by CONTENT -----------------------------------
56saved_files=""
57save_file() { # $1 path
58 local p="$1" flat
59 flat=$(echo "$p" | tr '/' '_')
60 cp "$p" "$backup/$flat"
61 saved_files="$saved_files $p"
63restore_all() {
64 local p flat before after
65 for p in $saved_files; do
66 flat=$(echo "$p" | tr '/' '_')
67 cp "$backup/$flat" "$p"
68 before=$(sha256sum "$backup/$flat" | cut -d' ' -f1)
69 after=$(sha256sum "$p" | cut -d' ' -f1)
70 if [ "$before" != "$after" ]; then
71 echo "F1B SABOTAGE FATAL: restore of $p did not match by hash" >&2
72 exit 2
73 fi
74 done
75 saved_files=""
77trap 'restore_all' EXIT
79# ---- line substitution that cannot silently become a no-op -----------------
80# The anchor must be present and UNIQUE. A leg whose anchor has drifted must
81# fail loudly, not plant nothing and then report the gate stayed green.
82replace_once() { # $1 file, $2 literal anchor substring, $3 replacement line
83 local file="$1" anchor="$2" new="$3" n before after
84 n=$(grep -cF "$anchor" "$file")
85 if [ "$n" != "1" ]; then
86 echo "F1B SABOTAGE FATAL: anchor appears $n times in $file (need exactly 1): $anchor" >&2
87 exit 2
88 fi
89 before=$(sha256sum "$file" | cut -d' ' -f1)
90 # THE ANCHOR IS PASSED THROUGH A FILE, NOT THROUGH `awk -v`.
91 #
92 # `awk -v var=value` INTERPRETS ESCAPE SEQUENCES in the assignment. An anchor
93 # containing `\n` -- e.g. `@(display "F1B-WITNESS-EXPR-RAN\n")` -- becomes a
94 # real newline inside awk, so `index()` searches for a two-line string that
95 # cannot occur in any single line. `grep -cF` finds the anchor, awk does not,
96 # and THE SUBSTITUTION SILENTLY DOES NOTHING.
97 #
98 # Measured, not theorised: that is exactly what happened to leg S10, which then
99 # reported a verdict about an arm it had never tested. SD-P7 met the same
100 # `awk -v` limitation and it failed LOUDLY there; here it failed silently,
101 # which is worse.
102 printf '%s' "$anchor" > "$file.anchor"
103 printf '%s' "$new" > "$file.repl"
104 awk 'BEGIN {
105 getline anchor < (ARGV[1]); getline repl < (ARGV[2]); ARGV[1]=""; ARGV[2]=""
106 }
107 index($0, anchor) { print repl; next } { print }
108 ' "$file.anchor" "$file.repl" "$file" > "$file.tmp" && mv "$file.tmp" "$file"
109 rm -f "$file.anchor" "$file.repl"
110 # AND THE FILE MUST HAVE CHANGED. This is the guard the earlier version
111 # CLAIMED to provide and did not: it verified that the anchor was FINDABLE and
112 # never that anything MOVED. A check verified by "I could locate the target"
113 # rather than "the target changed" is the same shape as a restore verified by
114 # exit code instead of by content.
115 after=$(sha256sum "$file" | cut -d' ' -f1)
116 if [ "$before" = "$after" ]; then
117 echo "F1B SABOTAGE FATAL: substitution in $file was a NO-OP (file unchanged by sha256)." >&2
118 echo " anchor: $anchor" >&2
119 exit 2
120 fi
123# ---- run one leg -----------------------------------------------------------
124# $1 leg id, $2 arm, $3 layer, $4 expected signature, $5 description
125run_leg() {
126 local id="$1" arm="$2" layer="$3" signature="$4" desc="$5"
127 legs_run=$((legs_run + 1))
128 local log="$work/leg-$id.log"
129 F1B_GATE_ONLY_ARM="$arm" bash test/gate-f1b.sh > "$log" 2>&1
130 local status=$?
131 # (a) IT MUST HAVE REACHED THE NAMED ARM. Without this a leg whose defect was
132 # caught by an earlier arm would report a red and certify coverage of an
133 # arm it never exercised.
134 if ! grep -q -- "--- arm $arm" "$log"; then
135 failures="$failures\n $id: never reached arm $arm (the defect was caught earlier, or the gate died in setup)"
136 say " $id [arm $arm, $layer] DID NOT REACH ARM $arm -- $desc"
137 return
138 fi
139 # (b) IT MUST NOT HAVE PRINTED THE ARM'S GREEN MARKER.
140 if grep -q "F1B_GATE_ARM_${arm}_GREEN" "$log"; then
141 failures="$failures\n $id: arm $arm stayed GREEN with the defect planted"
142 say " $id [arm $arm, $layer] DID NOT GO RED -- $desc"
143 return
144 fi
145 # (c) IT MUST HAVE FAILED WITH THE NAMED SIGNATURE, not merely exited badly.
146 if ! grep -qF "$signature" "$log"; then
147 failures="$failures\n $id: went red WITHOUT its signature (exit $status). Expected: $signature"
148 say " $id [arm $arm, $layer] RED FOR THE WRONG REASON -- $desc"
149 tail -3 "$log" | sed 's/^/ /'
150 return
151 fi
152 legs_red=$((legs_red + 1))
153 say " $id [arm $arm, $layer] RED as predicted -- $desc"
156echo "=== F1b sabotage ==========================================="
157echo "repo : $repo_dir @ $(git rev-parse HEAD)"
158echo "work : $work"
159echo "============================================================"
161# ===========================================================================
162# ARM 1 -- the build
163# ===========================================================================
164save_file src/folio/doc/author.sgl
165# AN UNBALANCED PAREN, not a missing import.
167# The first version deleted `(folio doc profile)` from the import list, and arm 1
168# stayed GREEN -- correctly. The module still COMPILES; the unbound
169# `folio-content-profile` fails at LOAD, not at compile, and arm 1's claim is
170# only that the three modules are in the build graph, asserted against their
171# `.sgb` artifacts. A leg has to break the thing the arm actually claims.
173# Same lesson as S14, in the opposite direction: there, a substitution applied
174# and the defect did not bite; here, the defect bit somewhere the arm does not
175# look. Only the arm going red distinguishes either case from coverage.
176replace_once src/folio/doc/author.sgl \
177 " (define (children command values)" \
178 " (define (children command values"
179run_leg S1 1 product "build exited non-zero" \
180 "an unbalanced paren in author.sgl; the build cannot produce the module"
181restore_all
183# ===========================================================================
184# ARM 2 -- the reader suite
185# ===========================================================================
186save_file src/folio/doc/author.sgl
187# The uniqueness check returns its input unchanged: ids are collected, compared,
188# and the verdict discarded. The shape a real regression takes.
189replace_once src/folio/doc/author.sgl \
190 " (error (string-append \"folio/doc: id \" (duplicate-report folio-entries id)))))" \
191 " (begin id (string-append \"folio/doc: id \" (duplicate-report folio-entries id)))))"
192run_leg S2 2 product "reader suite exited non-zero" \
193 "the duplicate-id refusal computes its verdict and throws it away"
194restore_all
196save_file test/test-folio-doc-reader.sgl
197replace_once test/test-folio-doc-reader.sgl \
198 " (assert-equal 'one-or-more (document-profile-multiplicity folio-document-profile)))" \
199 " (assert-equal 'one (document-profile-multiplicity folio-document-profile)))"
200run_leg S3 2 assertion "reader suite exited non-zero" \
201 "an expected value in the suite is altered -- the harness notices harness changes"
202restore_all
204# ARM 2's DERIVED-COUNT CLAUSE HAS NO LEG, AND THE REASON IS STRUCTURAL.
206# An adversarial review correctly noted the clause was uncovered: S2 and S3 both
207# redden arm 2 through `reader suite exited non-zero`, which fires before the
208# count comparison is evaluated. A leg (S16) was written to close it -- comment
209# out one suite arm so the suite PASSES with a lower count -- and it DID NOT GO
210# RED, for a reason worth recording rather than working around.
212# `expected_tests` is derived by `grep -cE '^\(test "' "$suite"`. Commenting the
213# arm out removes it from BOTH sides at once: the file declares 29 and the runner
214# reports 29. That is the clause behaving correctly, not failing -- the
215# expectation is SUPPOSED to track the file, which is what stops it going stale.
217# What the clause actually guards is a RUNNER that reports fewer cases than the
218# file declares: a suite half of which failed to load, or a harness that silently
219# skipped arms. Those are defects in `sigil test`, not in anything this driver
220# can plant with a one-line substitution, and inventing a product change to make
221# the leg possible would mean building a worse product to satisfy a test -- the
222# same call SD-P7's leader ruling made when it DELETED its leg S-b rather than
223# keep one that could not fail.
225# So the leg is deleted and the gap is named. A leg that always passes is worse
226# than no leg, because it reads as coverage in the transcript.
227# ===========================================================================
228# ARM 3 -- exact membership of the refused witnesses
229# ===========================================================================
230save_file src/folio/doc/profile.sgl
231# The DECLARATION is deleted while the predicate is untouched, so the profile
232# still CONSTRUCTS -- only the membership assertion can catch this. That is the
233# whole point of obligation (a): silence must be impossible.
234replace_once src/folio/doc/profile.sgl \
235 " (make-phase \"p-witness\" \"a refused container\" 'planned 1 '() witness-location)))" \
236 " ))"
237run_leg S4 3 product "is not exactly {milestone, note, phase, project}" \
238 "the phase refused-witness declaration is deleted; the predicate still refuses it"
239restore_all
241# ===========================================================================
242# ARM 4 -- delegation rather than re-enumeration
243# ===========================================================================
244save_file src/folio/doc/profile.sgl
245# BEHAVIOUR IS IDENTICAL. This is the leg that proves arm 4's TEXTUAL half is
246# not decorative: every runtime assertion in the gate stays green, because the
247# expanded predicate admits and refuses exactly what the delegating one did.
248replace_once src/folio/doc/profile.sgl \
249 " (or (block-node? x) (folio-work-item? x) (folio-relational-node? x)))" \
250 " (or (block-node? x) (section? x) (paragraph? x) (code-block? x) (item-list? x) (table? x) (video? x) (term? x) (glossary? x) (admonition? x) (exercise? x) (example? x) (project-check? x) (anchor? x) (folio-work-item? x) (folio-relational-node? x)))"
251# The `block-node?` call is KEPT and the disjuncts are added BESIDE it. The first
252# version replaced the whole line, deleting the delegating call, so the arm's
253# "must call block-node?" check fired first and the RE-ENUMERATION DETECTOR --
254# the thing this leg exists to exercise, and the thing that was silently vacuous
255# under ERE -- was never reached. A leg that reddens the wrong check inside the
256# right arm is still a leg that proves nothing about the check it names.
257run_leg S5 4 product "re-enumerates core kinds instead of delegating to block-node?" \
258 "the 13 disjuncts are added BESIDE the delegating call; behaviour UNCHANGED"
259restore_all
261save_file src/folio/doc/publication.sgl
262replace_once src/folio/doc/publication.sgl \
263 " ((section? child)" \
264 " ((and #f (section? child))"
265run_leg S6 4 product "lower-child does not descend into: section" \
266 "publication stops descending into a core section -- F1a's reachable defect returns"
267restore_all
269# ===========================================================================
270# ARM 5 -- delegation parity
271# ===========================================================================
272save_file test/fixtures/f1b-parity-folio.sgd
273replace_once test/fixtures/f1b-parity-folio.sgd \
274 '@note["topics/f1b-parity" title: "Delegation parity"]{' \
275 '@note["topics/f1b-parity" title: "Delegation parityX"]{'
276run_leg S7 5 product "the shared body no longer starts at the same offset" \
277 "the folio root line grows one byte; positions stop being comparable"
278restore_all
280save_file src/folio/doc/author.sgl
281# A LOCATION-ONLY defect: the tree's SHAPE stays identical and only the ROOT's
282# srcloc moves. It is the only evidence that arm 5 compares LOCATIONS rather than
283# structure, which is the whole reason the parity claim is stronger than a shape
284# comparison.
286# The first version added `(set! body (map (lambda (c) c) body))`, which changes
287# nothing observable, and gave it the signature `F1B_PARITY_PROBE` -- a string
288# that appears in GREEN output too. Two independent defects in one leg, either of
289# which alone would have made it unable to fail.
291# TWO substitutions, and the second is why: the parity fixture's shared body
292# contains ONLY sigil-doc constructs, so none of folio's eleven commands runs
293# during that comparison. The single thing folio code contributes to those nodes
294# is the LOCATION forwarded by the `author-at` wrapper -- which makes that
295# wrapper the only place a location-only defect can be planted, and makes this
296# leg a sharper test of arm 5 than perturbing a folio command would have been.
297# `make-srcloc` lives in `(sigil syntax)`, which this module does not import, so
298# the import is planted too; without it the defect fails to BUILD and reddens
299# arm 1 instead of arm 5.
300replace_once src/folio/doc/author.sgl \
301 " (sigil doc model)" \
302 " (sigil doc model) (sigil syntax)"
303replace_once src/folio/doc/author.sgl \
304 " (lambda () (sd:author-at loc thunk))))" \
305 " (lambda () (sd:author-at (make-srcloc \"synthesized\" 1 1 0 0) thunk))))"
306run_leg S8 5 product "BYTE-IDENTICAL canonical datum" \
307 "every node's location is synthesized; SHAPE identical, positions moved"
308restore_all
310# ===========================================================================
311# ARM 6 -- the no-execution witness
312# ===========================================================================
313save_file src/folio/doc/reader.sgl
314replace_once src/folio/doc/reader.sgl \
315 " (if (pair? author-forms)" \
316 " (if #f"
317run_leg S9 6 product "NOT by the author-form refusal" \
318 "the author-form refusal is disabled; the manuscript executes"
319restore_all
321save_file test/fixtures/f1b-witness-sigil.sgd
322save_file test/fixtures/f1b-witness-folio.sgd
323# THE TRIPWIRE IS REMOVED FROM BOTH FIXTURES, NOT JUST THE CONTROL.
325# Editing the control alone made arm 6's fixture-PAIR assertion fire first (the
326# pair went from 4 differing lines to 6), so the leg reddened the wrong clause --
327# it proved the pair is pinned, which S-nothing needed, and left the
328# control-degradation clause uncovered. The two assertions genuinely conflict:
329# pinning the pair means no single-sided edit is possible.
331# Removing the tripwire from the SHARED body keeps the pair intact and still
332# degrades the control, which is the property under test: with no
333# `F1B-WITNESS-EXPR-RAN` anywhere, channel 1 cannot be shown able to fire, and
334# an absence proven by a control that cannot fire proves nothing.
336# ANCHOR WITH NO BACKSLASH ESCAPE. The original anchor was the whole line,
337# `@(display "F1B-WITNESS-EXPR-RAN\n")`, and the `\n` in it was interpreted by
338# `awk -v` into a real newline -- so `grep -cF` found the anchor, `index()` never
339# did, and the substitution silently did nothing while the guard reported
340# success. A prefix of the same line carries the identity without the escape.
341# The replacement is a blank line rather than `;; removed`, because `;;` is
342# PROSE in a `.sgd` manuscript and not a comment.
343replace_once test/fixtures/f1b-witness-sigil.sgd \
344 '@(display "F1B-WITNESS-EXPR-RAN' \
345 ''
346replace_once test/fixtures/f1b-witness-folio.sgd \
347 '@(display "F1B-WITNESS-EXPR-RAN' \
348 ''
349run_leg S10 6 product "channel 1 cannot fire" \
350 "the CONTROL loses a tripwire -- an absence proven by a broken control means nothing"
351restore_all
353# ===========================================================================
354# ARM 7 -- publication fallback
355# ===========================================================================
356save_file src/folio/doc/unresolved.sgl
357# The marker keeps its PREFIX and loses its IDENTITY -- exactly the defect
358# `make-query` exists to prevent, one layer down: a marker that satisfies every
359# "the marker is present" assertion while naming nothing.
361# The first version replaced the prefix instead, and arm 7 stayed GREEN: the
362# prefix assertion was satisfied by the sibling TRANSCLUSION marker and the query
363# id survived elsewhere in the rendering. Nothing the probe checked actually
364# depended on the query marker existing. Both the leg and the probe were fixed --
365# the probe now asserts the WHOLE marker text, derived from the same function the
366# renderer uses.
367replace_once src/folio/doc/unresolved.sgl \
368 ' (if id (string-append " " id) "")' \
369 ' ""'
370run_leg S11 7 product "names the unresolved query by id" \
371 "the query marker keeps its prefix and stops naming its query"
372restore_all
374# ===========================================================================
375# ARM 8 -- the probe harnesses can go red
376# ===========================================================================
377save_file test/f1b-parity-probe.sgl
378replace_once test/f1b-parity-probe.sgl \
379 '(if (getenv "F1B_PROBE_SELFTEST")' \
380 '(if #f'
381run_leg S12 8 assertion "self-test EXITED 0 with a failure injected" \
382 "a probe loses its self-test hook -- its green stops meaning anything"
383restore_all
385# ===========================================================================
386# ARM 9 -- reader-tier passivity
387# ===========================================================================
388save_file src/folio/doc/reader.sgl
389replace_once src/folio/doc/reader.sgl \
390 " (import (sigil core) (sigil syntax) (sigil doc reader))" \
391 " (import (sigil core) (sigil syntax) (sigil doc reader) (sigil process))"
392run_leg S13 9 product "which is outside the reader-tier allowlist" \
393 "the reader imports a subprocess runner"
394restore_all
396# ===========================================================================
397# ARM 10 -- reader/inspector layering
398# ===========================================================================
399save_file src/folio/doc/author.sgl
400# The identity COLLECTION is neutered, so the scan runs over nothing and finds
401# nothing. The first version prepended a no-op expression to the definition line
402# and left the original body in place, so the check still ran and still raised --
403# a substitution that APPLIED (the file changed) and was behaviourally inert.
404# The file-changed guard cannot catch that; only a leg whose defect is chosen for
405# its EFFECT rather than for its edit can.
406replace_once src/folio/doc/author.sgl \
407 " (let ((folio-entries (collect-identities roots folio-identity-of))" \
408 " (let ((folio-entries '())"
409run_leg S14 10 product "the READER did not refuse the duplicate document" \
410 "identity collection is neutered; the reader stops refusing duplicates"
411restore_all
413# ARM 10's INSPECTOR HALF, which S14 cannot reach: S14 reddens the reader-side
414# assertion, which runs FIRST, so the three inspector clauses -- the half the
415# arm's own comment calls "the half that would silently rot if the two layers
416# were ever harmonised" -- had nothing showing they can fire. Named by an
417# adversarial review.
419# The duplicate fixture is edited instead of the product, so the reader-side
420# assertion still fires normally and the inspector genuinely sees a document
421# with no duplicate.
422save_file test/fixtures/f1b-duplicate.sgd
423replace_once test/fixtures/f1b-duplicate.sgd \
424 '@task["t-dup" title: "The second declaration, further down" status: '"'"'active]{' \
425 '@task["t-not-dup" title: "The second declaration, further down" status: '"'"'active]{'
426run_leg S17 10 product "the inspector reported" \
427 "the duplicate is removed; the inspector must stop reporting one"
428restore_all
430# ===========================================================================
431# ARM 7 again -- the DEEPER descent, which the first section-only fix missed
432# ===========================================================================
433save_file src/folio/doc/publication.sgl
434# Descend into `section` only, which is what the code did until an adversarial
435# review showed the completeness argument was a non-sequitur. Reachable by an
436# ordinary manuscript: `@aside{ @section{ @task } }`.
437replace_once src/folio/doc/publication.sgl \
438 " ((admonition? child)" \
439 " ((and #f (admonition? child))"
440run_leg S18 7 product "publication probe exited non-zero" \
441 "the lowering stops descending into an ADMONITION; folio nodes under one are dropped"
442restore_all
444# ===========================================================================
445# ARM 11 -- suite arithmetic
446# ===========================================================================
447save_file test/gate-f1b.sh
448replace_once test/gate-f1b.sh \
449 "baseline_tests=351" \
450 "baseline_tests=350"
451run_leg S15 11 assertion "suite arithmetic does not close" \
452 "the recorded baseline is altered -- the arithmetic must not be self-satisfying"
453restore_all
455# ---------------------------------------------------------------------------
456echo
457echo "COVERAGE BY ARM -- the number that matters. A suite that only ever"
458echo "reddens one instrument has not tested the others."
459echo " arm 1 build <- S1"
460echo " arm 2 reader suite <- S2 (product), S3 (assertion)"
461echo " ...its DERIVED-COUNT clause is DELIBERATELY UNCOVERED. It guards"
462echo " the RUNNER reporting fewer cases than the file declares; the"
463echo " expectation tracks the file BY DESIGN, so no substitution here"
464echo " can separate the two. Leg S16 was written, measured unable to go"
465echo " red, and DELETED rather than kept as false coverage."
466echo " arm 3 exact membership <- S4"
467echo " arm 4 delegation spelling <- S5 (behaviour-identical), S6"
468echo " arm 5 parity <- S7, S8"
469echo " arm 6 no-execution witness <- S9 (product), S10 (control)"
470echo " arm 7 publication fallback <- S11, S18 (the deeper container descent)"
471echo " arm 8 harness self-tests <- S12 (assertion)"
472echo " arm 9 reader-tier passivity <- S13"
473echo " arm 10 reader/inspector <- S14 (reader half), S17 (inspector half)"
474echo " arm 11 suite arithmetic <- S15 (assertion)"
475echo "Every arm has a row. S5 is the load-bearing one: it is BEHAVIOUR-IDENTICAL,"
476echo "so it is the only evidence that arm 4's textual half is not decorative."
477echo
478if [ -n "$failures" ]; then
479 echo "F1B_SABOTAGE legs=$legs_run red=$legs_red -- FAILURES:"
480 printf "%b\n" "$failures"
481 exit 1
482fi
483echo "F1B_SABOTAGE legs=$legs_run red=$legs_red"
484echo "F1B_SABOTAGE_COMPLETE all legs red at their named arm, all targets restored by sha256"