test/sabotage-f2a.sh

1#!/usr/bin/env bash

2#

3# F2a sabotage -- proof that every arm of gate-f2a.sh can go RED, one NAMED

4# SIGNATURE per leg.

5#

6# A gate reported only green has not been shown to work; it has been shown to be

7# quiet. Each leg plants a REAL DEFECT (in the product, except the two that say

8# otherwise) and requires a SPECIFIC failure signature, never merely a non-zero

9# exit. Exit codes lie: 126 is a script that is not executable, 127 is a binary

10# that is not there, and both look like a convincing red while running no

11# assertions at all.

12#

13# TWO LEGS ATTACK THE INSTRUMENTS THEMSELVES rather than the product (S5, S6),

14# and they are the ones that matter most here. This phase's headline properties

15# are both ABSENCES -- no author code ran, no forbidden module is reachable --

16# and an absence assertion whose detector is broken passes for free. S5 breaks

17# the witness's POSITIVE CONTROL and requires the gate to refuse; S6 stops the

18# probe from reaching the dangerous code path and requires the gate to notice

19# that the channel was never exercised. Without those two, "no execution

20# observed" would be a sentence about nothing.

21#

22# RESTORE IS UNCONDITIONAL, NOT REACHED. Originals are copied aside up front and

23# put back from an `EXIT INT TERM` trap, then verified BY SHA256 rather than by

24# the exit status of `cp`. `git checkout -- <path>` is deliberately not used: it

25# restores from the last COMMIT, so on an uncommitted tree it silently discards

26# real work along with the sabotage, and exits 0 either way.

27#

28# EVERY LEG RUNS THE FULL GATE, which rebuilds. That is slow and it is the only

29# honest option: a product-layer defect in `src/` does not reach the test runner

30# until the module is recompiled, so a leg that skipped the build would be

31# grading the PREVIOUS binary -- the estate's "thing measured was not thing

32# shipped" shape, arriving inside the harness that exists to prevent it.

33set -uo pipefail

35repo_dir="$(cd "$(dirname "$0")/.." && pwd)"

36cd "$repo_dir"

37export XDG_CACHE_HOME="${XDG_CACHE_HOME:-$(mktemp -d)}"

39targets=(

40 src/folio/doc/inspect.sgl

41 test/test-folio-doc-inspect.sgl

42 test/f2a-witness-probe.sgl

43 test/fixtures/f2a-witness.sgd

44)

45backup_dir=$(mktemp -d)

46declare -A original_hash

47for t in "${targets[@]}"; do

48 mkdir -p "$backup_dir/$(dirname "$t")"

49 cp "$t" "$backup_dir/$t"

50 original_hash["$t"]=$(sha256sum "$t" | cut -d' ' -f1)

51done

53restore_all() {

54 local bad=0

55 for t in "${targets[@]}"; do

56 cp "$backup_dir/$t" "$t"

57 local now

58 now=$(sha256sum "$t" | cut -d' ' -f1)

59 if [ "$now" != "${original_hash[$t]}" ]; then

60 echo "RESTORE FAILED for $t (${original_hash[$t]} != $now)" >&2

61 bad=1

62 fi

63 done

64 [ "$bad" = "0" ] || { echo "TREE MAY BE DIRTY -- CHECK BEFORE COMMITTING" >&2; exit 9; }

65}

66trap restore_all EXIT INT TERM

68# Same PATH hazard as the gate: `python3` vanished from PATH mid-session on

69# 2026-08-11, and this driver uses it for exact-string patching. Resolved at the

70# hop, printed below, and refused loudly if absent -- a sabotage driver that

71# could not apply its patches would report the UNSABOTAGED product as red-proof.

72if command -v python3 >/dev/null 2>&1; then

73 PY=(python3); py_source="PATH"

74elif command -v guix >/dev/null 2>&1 && guix shell python -- python3 --version >/dev/null 2>&1; then

75 PY=(guix shell python -- python3); py_source="guix shell python"

76else

77 echo "no python3 available; this driver cannot apply its patches" >&2; exit 1

78fi

80legs_run=0

81legs_red=0

82declare -a leg_report=()

84# Exact-string patcher. `sed` regexes over multi-line Sigil forms are how a

85# sabotage silently fails to apply and the leg then reports the UNSABOTAGED

86# product as red-proof. This refuses instead.

87patch_file() { # $1 file, $2 old, $3 new

88 "${PY[@]}" - "$1" "$2" "$3" <<'PY'

89import sys

90path, old, new = sys.argv[1], sys.argv[2], sys.argv[3]

91s = open(path).read()

92if s.count(old) != 1:

93 sys.stderr.write("PATCH TARGET APPEARS %d TIMES IN %s (need exactly 1)\n" % (s.count(old), path))

94 sys.exit(1)

95open(path, "w").write(s.replace(old, new))

96PY

97}

99# $1 human name, $2 ARM NUMBER it must redden, $3 signature that must appear.

100#

101# THE ARM IS RUN IN ISOLATION (`F2A_GATE_ONLY_ARM`), and that is not a shortcut.

102# The gate's arms overlap deliberately -- the suite and the manifest both assert

103# the duplicate-id contract, and a byte/scalar defect is visible to several of

104# them -- so a full-gate run reddens at whichever arm comes first and says

105# nothing about whether the arm under test can fire. Isolating it is the only way

106# a leg proves what it claims to prove. A selective run cannot print the full

107# gate's marker, so this cannot be used to manufacture a green.

108#

109# EACH LEG ALSO REQUIRES THAT ITS ARM ACTUALLY RAN. A leg that died in the build,

110# or in the header checks, is a non-zero exit that never reached an assertion --

111# 126 and 127 wearing a red coat. The `--- arm N` line is the arm's own evidence

112# that it started.

113expect_red() {

114 local name="$1" arm="$2" signature="$3"

115 legs_run=$((legs_run + 1))

116 local log; log=$(mktemp)

117 F2A_GATE_ONLY_ARM="$arm" bash test/gate-f2a.sh > "$log" 2>&1

118 local status=$?

119 local clean; clean=$(mktemp)

120 sed 's/\x1b\[[0-9;]*m//g' "$log" > "$clean"

121 if ! grep -q -- "--- arm $arm" "$clean"; then

122 echo " [$name] arm $arm NEVER RAN -- the leg failed upstream of the assertion it is testing" >&2

123 tail -15 "$clean" >&2

124 leg_report+=("$name DID-NOT-REACH-ARM-$arm")

125 return 1

126 fi

127 if [ "$status" = "0" ]; then

128 echo " [$name] DID NOT GO RED - arm $arm does not catch this" >&2

129 tail -12 "$clean" >&2

130 leg_report+=("$name GREEN(BAD)")

131 return 1

132 fi

133 if ! grep -qF "$signature" "$clean"; then

134 echo " [$name] went red, but NOT for the expected reason." >&2

135 echo " expected signature: $signature" >&2

136 tail -20 "$clean" >&2

137 leg_report+=("$name RED(WRONG-REASON)")

138 return 1

139 fi

140 legs_red=$((legs_red + 1))

141 echo " [$name] RED as required (arm $arm), signature matched: $signature"

142 leg_report+=("$name RED@arm$arm")

143 return 0

144}

145

146echo "=== F2a sabotage: $(git rev-parse HEAD) ==="

147echo "=== python3: $("${PY[@]}" --version 2>&1 | tail -1) (via $py_source) ==="

148echo "=== sigil-doc: $(git -C "$repo_dir/../sigil-doc" rev-parse HEAD) ==="

149

150# --- S1 PRODUCT: spans computed from CHARACTERS instead of BYTES ------------

151# The plan names this sabotage explicitly. It is invisible to any check written

152# against the descriptor -- both coordinate systems still look self-consistent --

153# and it is caught here only because the verifier re-encodes the prefix itself.

154echo "S1: byte offsets become scalar offsets"

155patch_file src/folio/doc/inspect.sgl \

156 ' byte-start: (index-byte idx start)

157 byte-length: (- (index-byte idx end) (index-byte idx start))' \

158 ' byte-start: start

159 byte-length: (- end start)' || exit 1

160expect_red "S1 characters-not-bytes" "3" "encodes to byte" || true

161restore_all

162

163# --- S2 PRODUCT: a field span widened to its whole node ---------------------

164# "A task's status span is distinct from its body span" is the plan's wording.

165# A field span that silently equals the node's is the failure that makes every

166# per-field mutation in F3 rewrite the entire node.

167echo "S2: every field span becomes its node's span"

168patch_file src/folio/doc/inspect.sgl \

169 "(fields (if command? (arguments->fields idx (tree-node-arguments node)) '()))" \

170 "(fields (if command?

171 (map (lambda (f)

172 (field-record name: (field-record-name f)

173 index: (field-record-index f)

174 value: (field-record-value f)

175 name-span: (field-record-name-span f)

176 span: (span-from-scalars idx start end)))

177 (arguments->fields idx (tree-node-arguments node)))

178 '()))" || exit 1

179expect_red "S2 field span widened" "3" "field span is identical to the whole node's span" || true

180restore_all

181

182# --- S3 PRODUCT: a duplicate id resolves to the first occurrence ------------

183# The exact behaviour the plan forbids: "never pick by scan order".

184echo "S3: duplicate ids stop being ambiguous"

185patch_file src/folio/doc/inspect.sgl \

186 " ((null? (cddr hit)) 'unique)

187 (else 'ambiguous)" \

188 " (else 'unique)" || exit 1

189expect_red "S3 duplicate resolves silently" "2" "inspector suite exited non-zero" || true

190restore_all

191

192# --- S4 PRODUCT: only the FIRST occurrence of a duplicate is reported -------

193# Subtler than S3 and the one a reviewer would miss: resolution is still

194# refused, but the report privileges document order, so a repair tool driven by

195# diagnostics would only ever see one end of the problem.

196echo "S4: only one diagnostic per duplicated id"

197patch_file src/folio/doc/inspect.sgl \

198 " source-name s))

199 spans))" \

200 " source-name s))

201 (list (car spans))))" || exit 1

202expect_red "S4 one diagnostic per duplicate" "6" "duplicate-node-id diagnostics, got 1" || true

203restore_all

204

205# --- S5 INSTRUMENT: the witness's POSITIVE CONTROL is disarmed --------------

206# THE MOST IMPORTANT LEG IN THIS FILE. If a tripwire cannot fire, the passive

207# run trips nothing for a reason that has nothing to do with passivity, and the

208# arm reports a clean witness over a dead one. This is the check that asks

209# whether my own check can fail at all.

210echo "S5: the fixture's stdout tripwire is made a no-op (instrument layer)"

211patch_file test/fixtures/f2a-witness.sgd \

212 '@(display "F2A-WITNESS-EXPR-RAN\n")' \

213 '@(if #f (display "F2A-WITNESS-EXPR-RAN\n"))' || exit 1

214expect_red "S5 disarmed positive control" "4" "positive control: evaluating the fixture did not print" || true

215restore_all

216

217# --- S6 INSTRUMENT: the probe stops reaching the dangerous code path --------

218# Channel 3 exists because per-field extraction is the one way this phase could

219# newly execute author code. A probe that never extracts a field observes no

220# execution for free.

221echo "S6a: the witness probe stops extracting fields at all (instrument layer)"

222patch_file test/f2a-witness-probe.sgl \

223 ' (inspected-node-fields n))' \

224 " '())" || exit 1

225expect_red "S6a probe inspected nothing" "4" "it inspected nothing" || true

226restore_all

227

228# S6b is the one that proves the CHANNEL-3 assertion can fire. S6a above cannot:

229# it trips the coverage guard first, so the gate reddens one check earlier and

230# says nothing about whether "channel 3 was never exercised" is reachable. The

231# probe here still extracts fields -- it just stops capturing THE field whose

232# value is an expression, which is precisely the blind spot channel 3 exists to

233# rule out.

234echo "S6b: the probe extracts fields but skips the argument-position one (instrument layer)"

235patch_file test/f2a-witness-probe.sgl \

236 " (if (and (eq? (inspected-node-name n) 'argument-tripwire-command)" \

237 " (if (and #f (eq? (inspected-node-name n) 'argument-tripwire-command)" || exit 1

238expect_red "S6b channel 3 unexercised" "4" "channel 3 was never exercised" || true

239restore_all

240

241# --- S7 PRODUCT: parse diagnostics lose their location ---------------------

242# The failure this prevents is silent: every diagnostic in the estate degrades to

243# a positionless one and nothing errors.

244echo "S7: parse errors stop carrying a position"

245patch_file src/folio/doc/inspect.sgl \

246 ' (if (not (string-starts-with? message prefix))' \

247 ' (if #t' || exit 1

248expect_red "S7 unlocated diagnostics" "5" "did not produce an error diagnostic with code parse-error" || true

249restore_all

250

251# --- S8 PRODUCT: a node kind drops out of the vocabulary --------------------

252# The plan's fourth named sabotage: "omit one node kind".

253echo "S8: 'question' leaves the folio vocabulary"

254patch_file src/folio/doc/inspect.sgl \

255 '(decision . #t) (question . #t) (update . #t) (related . #f)' \

256 '(decision . #t) (update . #t) (related . #f)' || exit 1

257expect_red "S8 omitted node kind" "7" "disagree" || true

258restore_all

259

260# --- S9 PRODUCT: the passive core imports the filesystem -------------------

261# "Load the author module during inspection" is the plan's first named sabotage;

262# this is its enforceable form -- the allowlist refuses the import, and the

263# closure walker refuses what the import drags in.

264echo "S9: the passive core imports (sigil fs)"

265patch_file src/folio/doc/inspect.sgl \

266 ' (import (sigil core) (sigil struct) (sigil math) (sigil string) (sigil list)

267 (sigil syntax)' \

268 ' (import (sigil core) (sigil struct) (sigil math) (sigil string) (sigil list)

269 (sigil fs) (sigil syntax)' || exit 1

270expect_red "S9 filesystem in the passive core" "8" "outside the passive allowlist" || true

271restore_all

272

273# --- S10 ASSERTION LAYER: the suite's expected value is altered -------------

274# Deliberately the ONLY assertion-layer leg. Assertion sabotage proves the

275# harness notices when the harness changes; it says nothing about whether the

276# suite executes the real product. The eight product-layer legs above are what

277# establish that.

278echo "S10: the expected duplicate-id status is altered (assertion layer)"

279patch_file test/test-folio-doc-inspect.sgl \

280 " (assert-equal 'absent (descriptor-id-status duplicates \"t-does-not-exist\"))" \

281 " (assert-equal 'ambiguous (descriptor-id-status duplicates \"t-does-not-exist\"))" || exit 1

282expect_red "S10 altered expectation" "2" "inspector suite exited non-zero" || true

283restore_all

284

285# --- S13 PRODUCT: end-line/end-column stop describing the end --------------

286# These two columns were emitted by the manifest, parsed by the verifier, and

287# graded by NOTHING until adversarial review found it: a manifest claiming

288# end-line 999 and end-column -4 passed with failed=0. This leg exists so the

289# checks that closed that hole are known to fire.

290echo "S13: end-line/end-column report the START position"

291patch_file src/folio/doc/inspect.sgl \

292 " end-line: (index-line idx end)

293 end-column: (index-column idx end)" \

294 " end-line: (index-line idx start)

295 end-column: (index-column idx start)" || exit 1

296expect_red "S13 end position ungraded" "3" "end-line" || true

297restore_all

298

299# --- S14 PRODUCT: this module's column stops matching the parser's ----------

300# The suite's line/column differential replaced a test that asserted

301# `(integer? line)` and `(>= line 1)` -- both structurally guaranteed, so the

302# only falsifiable clause in it was a corpus-size floor. This leg is what makes

303# the replacement more than a rewrite.

304echo "S14: the column index stops advancing"

305patch_file src/folio/doc/inspect.sgl \

306 " (loop (+ i 1) (+ b (utf8-width c)) line (+ column 1))" \

307 " (loop (+ i 1) (+ b (utf8-width c)) line 1)" || exit 1

308expect_red "S14 column disagrees with the parser" "2" "line and column agree with the parser" || true

309restore_all

310

311# --- S12 PRODUCT: a callable is stored in a field ---------------------------

312# `descriptor-callable-values` exists so that "the descriptor is inert data" is a

313# WALK rather than a claim. Until this leg it had never returned anything, in any

314# run, ever -- which is exactly the state in which nobody would notice it had

315# stopped working.

316echo "S12: a field value becomes a procedure"

317# NESTED, not bare. A bare `(lambda (x) x)` exercises only the top-level check;

318# the walk that matters recurses into list values, because field values come from

319# `syntax->datum` and are usually lists -- which is the shape a real leak takes.

320patch_file src/folio/doc/inspect.sgl \

321 " value: (unquote-datum (syntax->datum (cadr rest)))" \

322 " value: (list 1 (lambda (x) x))" || exit 1

323expect_red "S12 nested callable in the descriptor" "3" "it is not inert data" || true

324restore_all

325

326# --- S15 PRODUCT: an inspector bug must RAISE, not become a diagnostic ------

327# HIGH finding 7 was that a blanket `guard` wrapped the whole pipeline, so an

328# internal error here became a `parse-error` diagnostic saying the MANUSCRIPT was

329# broken -- and at a trust boundary that means F3 marks read-only a document the

330# parser accepted. The guard was narrowed to the parse call. Nothing proved the

331# narrowing, which is the same gap S13 and S14 exist to close for findings 1 and 2.

332#

333# The defect is planted AFTER the parse succeeds. Under the old blanket guard it

334# would have been swallowed into `PARSED no` plus a located parse-error; with the

335# narrow guard it propagates and the probe dies, which is what the signature

336# below requires.

337echo "S15: an internal error inside convert-node"

338patch_file src/folio/doc/inspect.sgl \

339 " (slice (substring source start end))" \

340 " (slice (substring source start (+ end 1)))" || exit 1

341expect_red "S15 inspector bug is not a document diagnostic" "3" "span manifest probe exited non-zero" || true

342restore_all

343

344# --- S11 PRODUCT: the module stops compiling --------------------------------

345# Arm 1 asserts on the ARTIFACT ("Bundle created"), not on an exit code, because

346# this build is usually piped and a pipeline reports its last stage. That is not

347# theoretical here: earlier in this phase a backgrounded build was read as

348# successful from its `real 2m39` timing line while it had in fact died on an

349# unclosed delimiter, and the next command graded the PREVIOUS binary. Arm 1 is

350# what caught it.

351echo "S11: the passive core stops compiling"

352patch_file src/folio/doc/inspect.sgl \

353 '(define (folio-vocabulary-names)' \

354 '(define (folio-vocabulary-names' || exit 1

355expect_red "S11 broken build" "1" "build exited non-zero" || true

356restore_all

357

358echo

359for r in "${leg_report[@]}"; do echo " $r"; done

360echo "F2A_SABOTAGE legs=$legs_run red=$legs_red"

361if [ "$legs_run" != "$legs_red" ]; then

362 echo "SABOTAGE INCOMPLETE: $((legs_run - legs_red)) leg(s) failed to go red" >&2

363 exit 1

364fi

365echo "F2A_SABOTAGE_COMPLETE all legs red"