AtlatestRepositorysigil-doc

sigil-doc / tree / testsabotage-sd-p4-counts.sh

1#!/usr/bin/env bash
2# Sabotage driver for gate-sd-p4.sh ARM 01's DERIVED test count.
3#
4# ARM 01 used to assert a hardcoded `Tests: 17 passed (17 total)`. It now derives
5# the declared count from test/test-passive-parse.sgl and makes TWO assertions:
6# declared == collected, and collected == passed. A derived count never shown to
7# DISCRIMINATE is a literal with extra steps, so this drives a red for each of
8# them, plus one for the derivation itself.
9#
10# LEG 1 product layer. A `(test "` form is planted at column 0 INSIDE a
11# function that is never called, so the derivation counts it and the
12# runner never collects it -- the shape of "a test silently stopped
13# running", which is what assertion one exists for.
14# LEG 2 product layer. A real toplevel test is planted that FAILS. It goes
15# red, but through the gate's pre-existing exit-status check rather
16# than assertion two, because the runner exits non-zero. Asserted on
17# that message, and reported as what it is rather than dressed up.
18# LEG 2b assertion layer. The gate's summary assertions are SLICED OUT OF THE
19# GATE FILE by its sentinel comments and run against synthetic
20# summaries under the same `set -euo pipefail`. Slicing rather than
21# retyping is the point: a retyped copy agrees with the gate by
22# construction, and the shell options differ, which is how the first
23# version of this leg certified semantics the gate did not have.
24# Includes an ANSI-coloured summary, because the real path reaches
25# these assertions only through the gate's colour strip.
26# LEG 3 assertion layer. The derivation's anchor is defeated while the suite
27# still passes -- the product is untouched -- and the gate must blame
28# the derivation rather than accuse the suite.
29# LEG 4 assertion layer. ONE form is indented, which the floor cannot see: the
30# derivation drops by one while the runner still collects them all, so
31# without the completeness check the gate accuses a healthy suite.
33# Attribution anchors on output the PASSING path cannot produce: the gate prints
34# `SD_P4_GATE_01` only when arm 1 passes, so EVERY leg requires its specific
35# error text AND the absence of that marker. A bare non-zero exit proves
36# nothing; 126 and 127 produce one too.
38# COST: the legs are cheap (each dies inside arm 1), but the POSITIVE CONTROL at
39# the end runs the gate to completion, including arm 4's book rebuild. Budget
40# roughly fifteen minutes for the run as a whole.
42# The suite is restored from a copy and verified BY HASH under a trap that fires
43# on abort as well as exit. A sibling of this script has no trap, and an abort
44# between the damage and the repair left that corruption in the worktree.
46# EVERY `|| true` here is load-bearing for the same reason it is in the gate:
47# under `set -e` a command substitution whose command fails aborts the script,
48# and `grep -c`/`rg` exit 1 on zero matches. Without them a plant that failed to
49# take would kill this driver at the assignment, and its SETUP-FAILED message --
50# the one that exists to say the leg is vacuous -- would never print. Three of
51# them were missing in the first version of this file, found by adversarial
52# review rather than by running it, because the plants all happened to work.
53set -euo pipefail
54repo_dir="$(cd "$(dirname "$0")/.." && pwd)"
55cd "$repo_dir"
56suite=test/test-passive-parse.sgl
57gate=test/gate-sd-p4.sh
59command -v rg >/dev/null || { echo "SETUP-FAILED: rg is required" >&2; exit 2; }
60test -f "$suite" || { echo "SETUP-FAILED: $suite is missing" >&2; exit 2; }
61test -x "$gate" || { echo "SETUP-FAILED: $gate is missing or not executable" >&2; exit 2; }
63work=$(mktemp -d "${TMPDIR:-/tmp}/sd-p4-counts-sabotage.XXXXXX")
64pristine="$work/suite.pristine"
65cp "$suite" "$pristine"
66base_sha=$(sha256sum "$suite" | awk '{print $1}')
67echo "baseline $suite sha256=$base_sha"
69restore() {
70 cp "$pristine" "$suite"
71 local now; now=$(sha256sum "$suite" | awk '{print $1}')
72 if [ "$now" = "$base_sha" ]; then
73 echo "RESTORED $suite (sha256 matches baseline)"
74 else
75 echo "RESTORE FAILED: $suite is $now, baseline was $base_sha" >&2
76 fi
78# Restore FIRST, then clear scratch. Never the other way round: $pristine lives
79# in $work.
80cleanup() { restore; rm -rf "$work"; }
81trap cleanup EXIT INT TERM
83declared_now() { grep -cE '^\(test "' "$suite" || true; }
84base_declared=$(declared_now)
85test "$base_declared" -ge 1 || {
86 echo "SETUP-FAILED: the pristine suite derives $base_declared tests; nothing to sabotage" >&2
87 exit 2; }
88echo "baseline derivation: $base_declared declared tests"
89echo
91# Runs the gate and reports arm 1's verdict.
92expect_arm1_red() {
93 local name="$1" expect="$2" log="$work/$1.log" rc=0
94 "$gate" >"$log" 2>&1 || rc=$?
95 echo "--- $name: derivation now reads $(declared_now); gate exit=$rc"
96 if [ "$rc" -eq 0 ]; then
97 echo " NOT RED: the gate passed. $name did not discriminate." >&2; return 1; fi
98 if rg -q 'SD_P4_GATE_01' "$log"; then
99 echo " MIS-ATTRIBUTED: arm 1 printed its pass marker, so the gate died later." >&2
100 tail -5 "$log" >&2; return 1; fi
101 if rg -qF "$expect" "$log"; then
102 echo " RED, attributed to arm 1, and SD_P4_GATE_01 absent:"
103 rg -NF "$expect" "$log" | head -2 | sed 's/^/ /' || true
104 else
105 echo " RED but with the WRONG message. Expected: $expect" >&2
106 tail -12 "$log" >&2; return 1; fi
109# ---------------------------------------------------------------------- LEG 1
110echo "=== LEG 1 (product): declared > collected -- a test that stopped running ==="
111restore
112cat >>"$suite" <<'PLANT'
114;; --- sabotage-sd-p4-counts LEG 1: declared at column 0, never collected ---
115(define (sd-p4-counts-sabotage-never-called)
116(test "sd-p4 counts sabotage -- declared, never collected"
117 (assert-true #t)))
118PLANT
119planted=$(declared_now)
120test "$planted" -eq $(( base_declared + 1 )) || {
121 echo "SETUP-FAILED: the plant did not move the derivation ($base_declared -> $planted)" >&2
122 exit 2; }
123echo "plant verified: derivation moved $base_declared -> $planted"
124expect_arm1_red LEG1 "declares $planted tests but the runner collected $base_declared"
125echo
127# ---------------------------------------------------------------------- LEG 2
128echo "=== LEG 2 (product): a collected test FAILS ==="
129restore
130cat >>"$suite" <<'PLANT'
132;; --- sabotage-sd-p4-counts LEG 2: collected and failing ---
133(test "sd-p4 counts sabotage -- collected and failing"
134 (assert-true #f))
135PLANT
136leg2_declared=$(declared_now)
137test "$leg2_declared" -eq $(( base_declared + 1 )) || {
138 echo "SETUP-FAILED: the LEG 2 plant did not reach the suite" >&2; exit 2; }
139expect_arm1_red LEG2 "passive suite exited non-zero"
140echo " NOTE: the runner exits non-zero on a failing test, so the gate's"
141echo " pre-existing exit-status check fires before assertion two is reached."
142echo " Assertion two therefore guards a runner that reports a failure while"
143echo " exiting clean. LEG 2b exercises it on exactly that input."
144echo
146# --------------------------------------------------------------------- LEG 2b
147echo "=== LEG 2b (assertion layer): the gate's OWN summary assertions, sliced ==="
148restore
149arm1="$work/arm1-assertions.sh"
151 echo 'set -euo pipefail'
152 sed -n '/^# >>> ARM1_SUMMARY_ASSERTIONS BEGIN$/,/^# <<< ARM1_SUMMARY_ASSERTIONS END$/p' "$gate"
153 echo 'echo "green ($reported/$collected/$expected_tests)"'
154} >"$arm1"
155# The slice must actually contain the assertions, or this leg silently tests an
156# empty script and passes.
157for needle in 'reported=' 'collected=' 'test "$collected" = "$expected_tests"' 'test "$reported" = "$collected"'; do
158 grep -qF -- "$needle" "$arm1" || {
159 echo "SETUP-FAILED: the sentinel slice from $gate is missing: $needle" >&2
160 echo " (did the ARM1_SUMMARY_ASSERTIONS markers move?)" >&2; exit 2; }
161done
162echo "sliced $(grep -cvE '^\s*#|^\s*$' "$arm1") code lines out of $gate between its sentinels"
164# Synthetic summaries, ANSI-coloured and then put through the GATE'S OWN strip,
165# because that is the only way the real path reaches these assertions.
166strip() { sed -e 's/\x1b\[[0-9;]*m//g' "$1" >"$2"; }
167n="$base_declared"; m=$(( base_declared - 1 ))
168printf ' \033[32mTests:\033[0m %d passed, 1 failed \033[2m(%d total)\033[0m\n' "$m" "$n" >"$work/buried.raw"
169printf ' \033[32mTests:\033[0m %d passed \033[2m(%d total)\033[0m\n' "$n" "$n" >"$work/healthy.raw"
170printf ' \033[2mno summary here at all\033[0m\n' >"$work/nosummary.raw"
171for f in buried healthy nosummary; do strip "$work/$f.raw" "$work/$f.clean"; done
172grep -q $'\033' "$work/buried.raw" || { echo "SETUP-FAILED: the coloured fixture carries no ANSI" >&2; exit 2; }
173grep -q $'\033' "$work/buried.clean" && { echo "SETUP-FAILED: the strip left ANSI behind" >&2; exit 2; }
175run_arm1() { # $1=clean-log $2=expected_tests -> the sliced block's WHOLE output
176 # NOT `tail -1`. Each failing branch in the gate prints its message and then
177 # dumps `tail -40` of the log, so the last line is log content and the message
178 # is buried above it. Matching on the last line reported LEG 2b as failing when
179 # the assertion had in fact fired correctly -- the leg accused a working gate.
180 suite_log="${1%.clean}" expected_tests="$2" bash "$arm1" 2>&1 || true
182saw() { printf '%s' "$1" | grep -qF -- "$2"; }
183buried=$(run_arm1 "$work/buried.clean" "$n")
184healthy=$(run_arm1 "$work/healthy.clean" "$n")
185nosummary=$(run_arm1 "$work/nosummary.clean" "$n")
186saw "$buried" "reported $m passed of $n collected" && \
187 echo " coloured buried-failure summary -> fired: reported $m passed of $n collected" || {
188 echo " LEG 2b FAILED: assertion two did not fire on a buried failure. It said:" >&2
189 printf '%s\n' "$buried" | head -5 >&2; exit 1; }
190saw "$healthy" "green ($n/$n/$n)" && \
191 echo " coloured healthy summary -> stood down: green ($n/$n/$n)" || {
192 echo " LEG 2b POSITIVE CONTROL FAILED: the assertions fired on a healthy summary:" >&2
193 printf '%s\n' "$healthy" | head -5 >&2; exit 1; }
194saw "$nosummary" "printed no summary line" && \
195 echo " no summary at all -> fired: printed no summary line" || {
196 echo " LEG 2b FAILED: a missing summary did not produce the no-summary message." >&2
197 echo " That is the guard that dies silently without its \`|| true\`. It said:" >&2
198 printf '%s\n' "$nosummary" | head -5 >&2; exit 1; }
199echo " the gate's own assertions fire on a buried failure and on a missing summary,"
200echo " and stand down on a healthy one."
201echo
203# ---------------------------------------------------------------------- LEG 3
204echo "=== LEG 3 (assertion layer): the DERIVATION itself is broken ==="
205restore
206# INDENT every `(test "` by one space. The suite still parses and still passes --
207# the PRODUCT is untouched -- but the derivation's anchor stops matching. The
208# first version of this leg commented the forms out with `;;`, which orphaned
209# their closing parens and broke the suite's LOAD; the gate's exit-status check
210# fired first and the driver correctly reported WRONG-MESSAGE rather than a red.
211sed -i 's|^(test "| (test "|' "$suite"
212zero=$(declared_now)
213test "$zero" -eq 0 || { echo "SETUP-FAILED: derivation still yields $zero, not 0" >&2; exit 2; }
214still=$(grep -cE '^ \(test "' "$suite" || true)
215test "$still" -eq "$base_declared" || {
216 echo "SETUP-FAILED: indenting lost tests ($base_declared -> $still); that is a PRODUCT break," >&2
217 echo " which would make this an assertion-layer leg in name only" >&2; exit 2; }
218echo "plant verified: derivation yields 0 while the suite still declares $still tests"
219# The indent guard fires before the floor, and names the real defect.
220expect_arm1_red LEG3 "INDENTED (test"
221echo
223# ---------------------------------------------------------------------- LEG 4
224echo "=== LEG 4 (assertion layer): a PARTIAL derivation miss -- one test indented ==="
225restore
226# The case the floor cannot see. Indent exactly ONE form: the derivation drops to
227# N-1 while the runner still collects N, so without the completeness check the
228# gate reports "declares 16 tests but the runner collected 17" -- accusing a
229# suite that is perfectly healthy for a defect in the gate's own anchor. That is
230# the same lie the hardcoded literal told, one size smaller, and it is why the
231# completeness check runs before the count.
232sed -i '0,/^(test "/s|^(test "| (test "|' "$suite"
233one_off=$(declared_now)
234test "$one_off" -eq $(( base_declared - 1 )) || {
235 echo "SETUP-FAILED: expected the derivation to drop by exactly one ($base_declared -> $one_off)" >&2
236 exit 2; }
237indented=$(grep -cE '^[[:space:]]+\(test "' "$suite" || true)
238test "$indented" -eq 1 || {
239 echo "SETUP-FAILED: expected exactly 1 indented form, found $indented" >&2; exit 2; }
240echo "plant verified: derivation $base_declared -> $one_off with 1 form indented; the suite still declares $base_declared"
241expect_arm1_red LEG4 "has 1 INDENTED"
242echo
244# --------------------------------------------------------- POSITIVE CONTROL
245echo "=== POSITIVE CONTROL: arm 1 must go GREEN again on the restored suite ==="
246echo " (this one runs the gate to completion; arm 4 rebuilds the book)"
247restore
248ctl="$work/control.log"; ctl_rc=0
249"$gate" >"$ctl" 2>&1 || ctl_rc=$?
250if rg -q 'SD_P4_GATE_01' "$ctl"; then
251 rg -N 'SD_P4_GATE_0' "$ctl" | sed 's/^/ /'
252 if [ "$ctl_rc" -ne 0 ]; then
253 echo " arm 1 GREEN; the gate exited $ctl_rc in a LATER arm, which does not affect"
254 echo " the arm-1 verdicts above. Run the gate on its own for a full green."
255 fi
256else
257 echo "POSITIVE CONTROL FAILED: arm 1 did not pass on the restored suite (exit $ctl_rc)" >&2
258 tail -20 "$ctl" >&2; exit 1
259fi
260echo
261echo "SD_P4_COUNTS_SABOTAGE_COMPLETE"