AtlatestRepositorysigil-doc
1
#!/usr/bin/env bash2
# SD-P1 sabotage driver.3
#4
# Six product-layer legs, one assertion-layer leg, and one leg that asks5
# whether a check of this gate can fail at all. For each: plant the defect the6
# arm exists to catch, prove the arm goes RED, restore, and prove the7
# restoration by CONTENT HASH rather than by the exit code of the restore.8
#9
# LEG 1 IS THE PHASE'S FALSIFICATION. The §0 row names it, and it replaces the10
# withdrawn "run the new suite against the pre-SD-P1 parser" criterion: for a11
# phase that ADDS BINDINGS, that run fails at COMPILE time on absent names,12
# which is a green-for-the-wrong-reason rather than evidence about behaviour.13
# Leg 1 keeps every binding in place and removes only the BEHAVIOUR.14
#15
# Three properties this driver asserts about ITSELF, because a sabotage run that16
# quietly did nothing is indistinguishable from a passing one:17
# * the edit must actually change the file (hash before != hash after), or the18
# leg is vacuous and the run aborts;19
# * the red must be an ASSERTION failure from a suite that compiled and ran --20
# the driver requires the runner's own "N passed, M failed" line, so a21
# compile error cannot be mistaken for a demonstrated red;22
# * restoration is verified by hash against a pristine copy taken up front.23
set -euo pipefail24
repo_dir="$(cd "$(dirname "$0")/.." && pwd)"25
cd "$repo_dir"26
sigil="${SIGIL_D1_COMPILER:-$repo_dir/../task-sigil-language-readers/build/dev/bin/sigil}"27
export SIGIL_REDIRECTS="${SIGIL_D1_REDIRECTS:-$repo_dir/dev-redirects.sgl}"28
mkdir -p "$repo_dir/build"29
work=$(mktemp -d "$repo_dir/build/sd-p1-sabotage.XXXXXX")30
export XDG_CACHE_HOME="$work/cache"32
command -v python3 >/dev/null || { echo "python3 is required by this driver" >&2; exit 1; }33
command -v rg >/dev/null || { echo "rg is required by this driver" >&2; exit 1; }35
parser=src/sigil/doc/private/parser.sgl36
reader=src/sigil/doc/reader.sgl37
suite=test/test-root-profile.sgl38
before=test/transcripts/sd-p1/classification-before.txt40
pristine="$work/pristine"41
mkdir -p "$pristine"42
for f in "$parser" "$reader" "$suite" "$before"; do43
mkdir -p "$pristine/$(dirname "$f")"44
cp "$f" "$pristine/$f"45
done47
hash_of() { sha256sum "$1" | cut -d' ' -f1; }49
restore() {50
for f in "$parser" "$reader" "$suite" "$before"; do51
cp "$pristine/$f" "$f"52
test "$(hash_of "$f")" = "$(hash_of "$pristine/$f")" || {53
echo "RESTORE FAILED for $f" >&2; exit 1; }54
done55
}57
# LEG 8 deliberately corrupts a COMMITTED artifact (the BEFORE transcript), and58
# an abort between the forge and the restore leaves that corruption in the59
# worktree. It happened: a leftover variable from a renumbering tripped `set -u`60
# at exactly that point. Restoration is therefore unconditional -- on success,61
# on failure, and on interrupt -- not a line at the end of each leg.62
trap restore EXIT INT TERM64
# apply <file> <old> <new> -- exact single-occurrence replacement, aborting if65
# the target is absent, ambiguous, or the edit no-ops.66
apply() {67
local file="$1" old="$2" new="$3" before_hash after_hash68
before_hash=$(hash_of "$file")69
python3 - "$file" "$old" "$new" <<'PY'70
import sys71
path, old, new = sys.argv[1], sys.argv[2], sys.argv[3]72
text = open(path).read()73
n = text.count(old)74
if n != 1:75
sys.exit("sabotage target appears %d times in %s; expected exactly 1" % (n, path))76
open(path, "w").write(text.replace(old, new))77
PY78
after_hash=$(hash_of "$file")79
test "$before_hash" != "$after_hash" || { echo "SABOTAGE DID NOT CHANGE $file" >&2; exit 1; }80
}82
# The COLLECTED test count, measured from the leg runs below rather than83
# written in. The closing green check at the end of this driver needs a84
# denominator; a literal one goes stale the first time somebody adds a test,85
# and a stale denominator in a check is how a gate stops describing its86
# subject and becomes a second thing to keep in sync.87
collected_total=089
# expect_suite_red <label> [<expected failing test name fragment>]90
expect_suite_red() {91
local label="$1" expect="${2:-}" status log92
# SPLIT DELIBERATELY. `local a="$1" b="$(... $a ...)"` expands EVERY93
# assignment before `local` runs, so `$a` is still UNBOUND at that point.94
# Under `set -u` this printed `label: unbound variable` and evaluated to95
# `$work/.log` for every leg, so all seven legs overwrote one another's96
# log and the per-leg evidence this driver is built around did not exist.97
# It is in the committed transcript seven times98
# (test/transcripts/sd-p1/sabotage.txt, reported at "line 84").99
log="$work/$(printf '%s' "$label" | tr ' /' '__').log"100
set +e101
setsid -w "$sigil" -L src -L test test "$suite" >"$log" 2>&1102
status=$?103
set -e104
sed -e 's/\x1b\[[0-9;]*m//g' "$log" >"$log.clean"105
echo "--- $label: exit=$status"106
# A red must come from a suite that COMPILED AND RAN. Without this the leg107
# would accept a compile error as a demonstrated failure.108
rg -q '[0-9]+ passed, [0-9]+ failed' "$log.clean" || {109
echo "$label: no runner summary -- the suite did not run; this is NOT a demonstrated red" >&2110
tail -40 "$log.clean" >&2; exit 1; }111
# Record this run's collected count (passed + failed) for the closing check.112
local summary p f113
summary=$(rg -o '[0-9]+ passed, [0-9]+ failed' "$log.clean" | head -1)114
p=${summary%% passed*}115
f=${summary#*, }; f=${f%% failed*}116
if [ $((p + f)) -gt "$collected_total" ]; then collected_total=$((p + f)); fi117
test "$status" -ne 0 || { echo "$label: SABOTAGE DID NOT TURN THE SUITE RED" >&2; exit 1; }118
rg -o '[0-9]+ passed, [0-9]+ failed' "$log.clean" | head -1119
rg -o '✗ .*' "$log.clean" | head -6 || true120
if [ -n "$expect" ]; then121
# Anchor to the runner's FAILURE marker. Matching the test name anywhere in122
# the log would also match the line that reports the same test PASSING, so123
# this check would have confirmed "red at the expected arm" for a leg whose124
# red came from somewhere else entirely.125
rg -q "✗.*$expect" "$log.clean" || {126
echo "$label: red, but NOT at the expected arm ($expect)" >&2127
tail -40 "$log.clean" >&2; exit 1; }128
fi129
}131
echo "=============================================================="132
echo "LEG 1 (product) -- THE NAMED SABOTAGE: the root allowlist is ignored."133
echo "Every binding, every message, every location stays in place; only the"134
echo "membership test is removed. This is the falsification for the phase."135
apply "$parser" \136
"(memq (car (car xs)) roots-allowed)" \137
"(begin roots-allowed #t)"138
expect_suite_red "leg1-allowlist-ignored" "allowlist"139
restore141
echo "=============================================================="142
echo "LEG 2 (product) -- multiplicity is ignored; every profile accepts any"143
echo "number of roots."144
apply "$parser" \145
"(or several? (null? root-nodes))" \146
"(begin several? #t)"147
expect_suite_red "leg2-multiplicity-ignored" "multiplicity"148
restore150
echo "=============================================================="151
echo "LEG 3 (product) -- the DEFAULT profile is silently widened, so"152
echo "sigil/doc would start accepting a folio root."153
apply "$parser" \154
"(make-document-profile \"sigil/doc\" '(chapter) 'one)" \155
"(make-document-profile \"sigil/doc\" '(chapter note) 'one)"156
expect_suite_red "leg3-default-widened" "default profile"157
restore159
echo "=============================================================="160
echo "LEG 4 (product) -- spans are measured in SCALARS instead of UTF-8 BYTES."161
echo "This is the exact bug the byte-vs-scalar precondition on the location arm"162
echo "exists to make detectable."163
apply "$parser" \164
"(- (position-at state end) start-byte)" \165
"(- end start)"166
expect_suite_red "leg4-scalar-spans" "exact locations"167
restore169
echo "=============================================================="170
echo "LEG 5 (product) -- top-level classification reverts to the HEAD SYMBOL,"171
echo "so an author's @text{} command is mistaken for the parser's own prose"172
echo "pseudo-node again."173
apply "$parser" \174
"(not (and (pseudo-kind? node 'text)" \175
"(not (and (eq? (car node) 'text)"176
expect_suite_red "leg5-head-symbol-classification" "dispatched as author commands"177
restore179
echo "=============================================================="180
echo "LEG 6 (product) -- trim-body-edges reverts to the HEAD SYMBOL."181
echo "This is the site the first version of this phase MISSED, and the most"182
echo "dangerous of the family: it does not merely misread a @text{} command,"183
echo "it REBUILDS it as a five-slot prose node, so every later stage then"184
echo "behaves correctly on the wrong node. Fixing lower-block alone left the"185
echo "block-position case completely unchanged and looked like success."186
apply "$parser" \187
" (define (left xs)188
(cond189
((null? xs) '())190
((not (pseudo-kind? (car xs) 'text)) xs)" \191
" (define (left xs)192
(cond193
((null? xs) '())194
((not (eq? (caar xs) 'text)) xs)"195
expect_suite_red "leg6-trim-body-edges-head-symbol" "dispatched as author commands"196
restore198
echo "=============================================================="199
echo "LEG 7 (ASSERTION layer) -- one expected byte offset in the suite is"200
echo "perturbed by one. This proves the location assertions actually execute"201
echo "rather than sitting in a branch nothing reaches."202
apply "$suite" \203
"(assert-equal (byte-offset source start) (srcloc-position loc))" \204
"(assert-equal (+ 1 (byte-offset source start)) (srcloc-position loc))"205
expect_suite_red "leg7-assertion-layer" "exact locations"206
restore208
echo "=============================================================="209
echo "LEG 8 (instrument validity) -- CAN GATE ARM 2 FAIL AT ALL?"210
echo "Arm 2 asserts the classification change is confined: at least one"211
echo "colliding spelling moved, and every positive control is identical. Its"212
echo "INPUT is the committed BEFORE transcript. Here that input alone is"213
echo "neutered -- replaced with the CURRENT behaviour, control marker kept --"214
echo "while the gate's scoring and exit logic are left untouched. If arm 2"215
echo "still passes, it was passing for free."216
after_now="$work/after-now.txt"217
setsid -w "$sigil" -L src -L test eval -f test/sd-p1-classification-probe.sgl >"$after_now" 2>&1218
# DERIVED, not hardcoded. This exact guard shipped with a stale "8 cases" and219
# aborted leg 8 on a probe that had run all 9 correctly -- the fourth instance220
# of this hazard in one session. It failed SAFE (it refused to forge with input221
# it could not vouch for), but the fix is to read the count from the artifact.222
probe_cases=$(rg -c '^\(report "' test/sd-p1-classification-probe.sgl || echo 0)223
test "$probe_cases" -ge 1 || { echo "could not count the probe's cases" >&2; exit 1; }224
rg -q "SD_P1_CLASSIFICATION_PROBE_COMPLETE $probe_cases cases" "$after_now" || {225
echo "could not produce a current transcript to neuter with" >&2226
cat "$after_now" >&2; exit 1; }227
{ echo "PRE_SD_P1_CONTROL_UNBOUND (forged for leg 8 -- this transcript is NOT pre-SD-P1)"228
cat "$after_now"; } >"$before"229
test "$(hash_of "$before")" != "$(hash_of "$pristine/$before")" || {230
echo "leg 8 did not change the BEFORE transcript" >&2; exit 1; }231
set +e232
SD_P1_SKIP_REGRESSION=1 test/gate-sd-p1.sh >"$work/leg8-gate.log" 2>&1233
leg8_status=$?234
set -e235
echo "--- leg8-before-transcript-neutered: exit=$leg8_status"236
tail -5 "$work/leg8-gate.log"237
test "$leg8_status" -ne 0 || {238
echo "LEG 8: arm 2 PASSED against a BEFORE transcript identical to the AFTER." >&2239
echo "It cannot detect an unconfined change and is not a gate." >&2; exit 1; }240
rg -q 'the classification fix is inert' "$work/leg8-gate.log" || {241
echo "LEG 8: the gate failed, but not at arm 2's confinement check" >&2242
tail -30 "$work/leg8-gate.log" >&2; exit 1; }243
# Arm 1 must have passed before arm 2 failed, or the red came from somewhere else.244
rg -q 'SD_P1_GATE_01' "$work/leg8-gate.log" || {245
echo "LEG 8: arm 1 did not pass, so the red is not attributable to arm 2" >&2; exit 1; }246
restore248
echo "=============================================================="249
echo "RESTORED -- every file back to its pristine hash. Re-running the suite"250
echo "green closes the loop: sabotage proves green->red, this proves red->green."251
for f in "$parser" "$reader" "$suite" "$before"; do252
printf ' %s %s\n' "$(hash_of "$f")" "$f"253
done255
# THE CLOSING RUN ASSERTED NOTHING ABOUT ITS OUTPUT UNTIL 2026-08-22, AND IT IS256
# THE ONLY THING IN THIS DRIVER THAT PROVES RED->GREEN.257
#258
# It was `setsid "$sigil" ... | sed ... | tail -8`, with no marker check of any259
# kind, and `SD_P1_SABOTAGE_COMPLETE 8 legs` printed on the next line.260
#261
# WHAT WAS AND WAS NOT WRONG WITH IT. The first version of this comment blamed a262
# `setsid` race, a leader acted on that, and it was wrong -- so the correction263
# lives here rather than being quietly dropped.264
#265
# NOT wrong: the exit status. `set -o pipefail` is on, so a suite exiting266
# non-zero would have aborted the script. And a PIPELINE blocks even when267
# `setsid` takes its fork branch, because the shell waits for EOF on the pipe268
# and the grandchild holds the write end. Measured on the forced fork branch:269
# 3020 ms for the piped form against a 3-second child, versus 10 ms for the270
# same command redirected to a file. This line was never racy. The racy shape271
# is redirect-to-a-file, which is a different set of call sites.272
#273
# Wrong, and still disqualifying: NOTHING ABOUT THE CONTENT WAS CHECKED. A274
# runner that exited 0 having collected zero tests, or having printed nothing275
# at all, satisfied every condition this block imposed, and the completion276
# marker went out regardless. A step whose whole job is to prove red->green has277
# to read what the runner actually said.278
#279
# It now asserts the RUNNER'S OWN WORDS from a log on disk, requires the run to280
# exit 0, and -- because an assertion that has never been shown to fire cannot281
# certify anything -- is armed against BOTH a truncated log and a known-good282
# summary before the real run happens.283
closing_log="$work/closing-green.log"285
# assert_suite_green <plain-log> <minimum collected count>286
# 0 = the log shows a complete green run; 1 = refuse, with the reason.287
assert_suite_green() {288
local log="$1" floor="$2" passed total289
rg -q 'All tests passed' "$log" || { echo " closing check: no 'All tests passed' banner" >&2; return 1; }290
passed=$(rg -o 'Tests:[[:space:]]+[0-9]+ passed' "$log" | head -1 | grep -oE '[0-9]+' | head -1)291
total=$(rg -o '\([0-9]+ total\)' "$log" | head -1 | grep -oE '[0-9]+' | head -1)292
test -n "$passed" -a -n "$total" || { echo " closing check: the runner printed no summary line" >&2; return 1; }293
# Self-consistency: every collected test must be a passing one. This needs no294
# literal and cannot go stale.295
test "$passed" = "$total" || { echo " closing check: $passed passed of $total collected" >&2; return 1; }296
# ...and a FLOOR, because "0 passed (0 total)" is self-consistent too. A bound297
# satisfied by a suite that collected nothing is not a bound.298
#299
# WHAT THIS FLOOR IS AND IS NOT. It is derived from the same suite it then300
# bounds -- the maximum collected count observed across the legs above -- so it301
# cannot go stale when a test is added, and it cannot become unsatisfiable302
# (sabotage reduces or preserves collection, never increases it). It also303
# cannot detect a suite that shrank: gut the file from 11 tests to 1 and every304
# leg still reddens on the survivor, the floor becomes 1, and this reports305
# "green over 1 collected tests". It catches total collapse to zero and nothing306
# weaker. `test/gate-sd-p1.sh` arm 1 is what pins the absolute count.307
test "$total" -ge "$floor" || {308
echo " closing check: collected $total, but the legs above collected $floor" >&2; return 1; }309
if rg -q '[1-9][0-9]* failed' "$log"; then310
echo " closing check: the runner reported failures" >&2; return 1; fi311
return 0312
}314
test "$collected_total" -ge 1 || {315
echo "no leg recorded a collected test count; the closing check has no denominator" >&2; exit 1; }317
# POSITIVE CONTROL 1 -- it must REFUSE the exact artifact the fork branch318
# produces. `Running tests...` and nothing else is the real 18-byte log from the319
# 2026-08-11 measurement.320
printf 'Running tests...\n' >"$closing_log.truncated"321
if assert_suite_green "$closing_log.truncated" "$collected_total" 2>/dev/null; then322
echo "CLOSING CHECK CANNOT FAIL: it accepted a truncated log" >&2; exit 1; fi323
# POSITIVE CONTROL 2 -- and it must ACCEPT a known-good summary, or control 1324
# passes for free: a check that refuses everything refuses truncation too.325
printf 'Tests: %s passed (%s total)\nAll tests passed\n' "$collected_total" "$collected_total" \326
>"$closing_log.control"327
assert_suite_green "$closing_log.control" "$collected_total" || {328
echo "CLOSING CHECK REFUSES A KNOWN-GOOD SUMMARY -- it is a wall, not a check" >&2; exit 1; }329
# POSITIVE CONTROL 3 -- the floor must bite. A self-consistent empty run is the330
# shape a stalled or mis-filtered runner produces, and it must not be accepted.331
printf 'Tests: 0 passed (0 total)\nAll tests passed\n' >"$closing_log.empty"332
if assert_suite_green "$closing_log.empty" "$collected_total" 2>/dev/null; then333
echo "CLOSING CHECK CANNOT FAIL: it accepted a run that collected 0 tests" >&2; exit 1; fi334
# POSITIVE CONTROL 4 -- the `passed == total` arm must bite on its own. Controls335
# 1-3 are all refused by the banner or the floor before this arm is reached, so336
# without this one the arm could be deleted and every control would still pass.337
printf 'Tests: %s passed (%s total)\nAll tests passed\n' \338
"$((collected_total - 1))" "$collected_total" >"$closing_log.short"339
if assert_suite_green "$closing_log.short" "$collected_total" 2>/dev/null; then340
echo "CLOSING CHECK CANNOT FAIL: it accepted a run where not every collected test passed" >&2341
exit 1; fi342
# POSITIVE CONTROL 5 -- and so must the `N failed` arm, which controls 1-4 also343
# leave untouched: a log can be self-consistent, banner-carrying, at or above the344
# floor, AND still report failures.345
printf 'Tests: %s passed (%s total)\nAll tests passed\n2 failed\n' \346
"$collected_total" "$collected_total" >"$closing_log.failing"347
if assert_suite_green "$closing_log.failing" "$collected_total" 2>/dev/null; then348
echo "CLOSING CHECK CANNOT FAIL: it accepted a log that reported failures" >&2349
exit 1; fi350
echo "closing check armed: refuses a truncated log, an empty run, a partial pass and a reported failure; accepts a green summary"352
set +e353
setsid -w "$sigil" -L src -L test test "$suite" >"$closing_log" 2>&1354
closing_status=$?355
set -e356
sed -e 's/\x1b\[[0-9;]*m//g' "$closing_log" >"$closing_log.clean"357
tail -8 "$closing_log.clean"358
test "$closing_status" -eq 0 || {359
echo "CLOSING RUN EXITED $closing_status on RESTORED sources -- red->green is NOT demonstrated" >&2360
tail -40 "$closing_log.clean" >&2; exit 1; }361
assert_suite_green "$closing_log.clean" "$collected_total" || {362
echo "CLOSING RUN did not demonstrate red->green on restored sources" >&2363
tail -40 "$closing_log.clean" >&2; exit 1; }365
echo "SD_P1_SABOTAGE_COMPLETE 8 legs, closing suite green over $collected_total collected tests"