AtlatestRepositoryfolio
1
#!/usr/bin/env bash2
#3
# F1b -- the sabotage driver. A gate that cannot demonstrate it fails is not a4
# gate; it has only been shown to be quiet.5
#6
# LEGS ARE COUNTED BY WHICH ARM THEY REDDEN, NOT BY HOW MANY THERE ARE.7
#8
# SD-P7 shipped a gate with four arms and twelve sabotage legs, which reads like9
# thorough coverage. Every one of those legs reddened arm 1. Its 42 KB10
# differential -- the arm that existed precisely to cover what the suite did not11
# -- had nothing establishing it could fail at all, and a large equality that12
# cannot go red is the most convincing vacuous gate there is: the bigger and more13
# precise the comparison, the more it reads as overwhelming evidence and the less14
# anyone asks whether it can move.15
#16
# So this driver states coverage BY ARM, every leg names the arm it targets, and17
# every leg ASSERTS IT REACHED THAT ARM before accepting a red. A leg that18
# reddens the wrong arm is worse than no leg, because it looks like coverage in19
# the transcript.20
#21
# EVERY LEG REQUIRES A NAMED SIGNATURE. A non-zero exit is NOT evidence of a22
# red: exit 127 (not found), 126 (not executable) and a usage error all end the23
# process badly while executing zero assertions, and all three have produced24
# convincing false reds in this estate.25
#26
# RESTORE IS BY HASH, NEVER BY EXIT CODE, and by copy-aside rather than by27
# `git checkout --`. That command restores from the last COMMIT, so if any work28
# were uncommitted it would discard it -- silently, with exit 0. A restore29
# verified by exit status reports success on a wipe.30
#31
# BOTH LAYERS ARE SABOTAGED. Breaking a test ASSERTION proves the harness32
# notices when the harness changes; breaking the PRODUCT proves the gate executes33
# the loaded product source rather than a mirror of itself. Only the second34
# establishes the property that makes a gate a gate -- a stale module, a fixture35
# that reimplements the behaviour, and a wrong load path are all invisible to36
# assertion-layer sabotage, and all three have occurred here.37
set -uo pipefail39
repo_dir="$(cd "$(dirname "$0")/.." && pwd)"40
cd "$repo_dir"42
sigil="${SIGIL_F1B_COMPILER:-sigil}"43
export SIGIL_REDIRECTS="${SIGIL_F1B_REDIRECTS:-$repo_dir/dev-redirects.sgl}"44
export XDG_CACHE_HOME="${XDG_CACHE_HOME:-$(mktemp -d)}"46
work=$(mktemp -d)47
backup="$work/backup"48
mkdir -p "$backup"49
legs_run=050
legs_red=051
failures=""53
say() { echo "$*"; }55
# ---- save / restore, verified by CONTENT -----------------------------------56
saved_files=""57
save_file() { # $1 path58
local p="$1" flat59
flat=$(echo "$p" | tr '/' '_')60
cp "$p" "$backup/$flat"61
saved_files="$saved_files $p"62
}63
restore_all() {64
local p flat before after65
for p in $saved_files; do66
flat=$(echo "$p" | tr '/' '_')67
cp "$backup/$flat" "$p"68
before=$(sha256sum "$backup/$flat" | cut -d' ' -f1)69
after=$(sha256sum "$p" | cut -d' ' -f1)70
if [ "$before" != "$after" ]; then71
echo "F1B SABOTAGE FATAL: restore of $p did not match by hash" >&272
exit 273
fi74
done75
saved_files=""76
}77
trap 'restore_all' EXIT79
# ---- line substitution that cannot silently become a no-op -----------------80
# The anchor must be present and UNIQUE. A leg whose anchor has drifted must81
# fail loudly, not plant nothing and then report the gate stayed green.82
replace_once() { # $1 file, $2 literal anchor substring, $3 replacement line83
local file="$1" anchor="$2" new="$3" n before after84
n=$(grep -cF "$anchor" "$file")85
if [ "$n" != "1" ]; then86
echo "F1B SABOTAGE FATAL: anchor appears $n times in $file (need exactly 1): $anchor" >&287
exit 288
fi89
before=$(sha256sum "$file" | cut -d' ' -f1)90
# THE ANCHOR IS PASSED THROUGH A FILE, NOT THROUGH `awk -v`.91
#92
# `awk -v var=value` INTERPRETS ESCAPE SEQUENCES in the assignment. An anchor93
# containing `\n` -- e.g. `@(display "F1B-WITNESS-EXPR-RAN\n")` -- becomes a94
# real newline inside awk, so `index()` searches for a two-line string that95
# cannot occur in any single line. `grep -cF` finds the anchor, awk does not,96
# and THE SUBSTITUTION SILENTLY DOES NOTHING.97
#98
# Measured, not theorised: that is exactly what happened to leg S10, which then99
# reported a verdict about an arm it had never tested. SD-P7 met the same100
# `awk -v` limitation and it failed LOUDLY there; here it failed silently,101
# which is worse.102
printf '%s' "$anchor" > "$file.anchor"103
printf '%s' "$new" > "$file.repl"104
awk 'BEGIN {105
getline anchor < (ARGV[1]); getline repl < (ARGV[2]); ARGV[1]=""; ARGV[2]=""106
}107
index($0, anchor) { print repl; next } { print }108
' "$file.anchor" "$file.repl" "$file" > "$file.tmp" && mv "$file.tmp" "$file"109
rm -f "$file.anchor" "$file.repl"110
# AND THE FILE MUST HAVE CHANGED. This is the guard the earlier version111
# CLAIMED to provide and did not: it verified that the anchor was FINDABLE and112
# never that anything MOVED. A check verified by "I could locate the target"113
# rather than "the target changed" is the same shape as a restore verified by114
# exit code instead of by content.115
after=$(sha256sum "$file" | cut -d' ' -f1)116
if [ "$before" = "$after" ]; then117
echo "F1B SABOTAGE FATAL: substitution in $file was a NO-OP (file unchanged by sha256)." >&2118
echo " anchor: $anchor" >&2119
exit 2120
fi121
}123
# ---- run one leg -----------------------------------------------------------124
# $1 leg id, $2 arm, $3 layer, $4 expected signature, $5 description125
run_leg() {126
local id="$1" arm="$2" layer="$3" signature="$4" desc="$5"127
legs_run=$((legs_run + 1))128
local log="$work/leg-$id.log"129
F1B_GATE_ONLY_ARM="$arm" bash test/gate-f1b.sh > "$log" 2>&1130
local status=$?131
# (a) IT MUST HAVE REACHED THE NAMED ARM. Without this a leg whose defect was132
# caught by an earlier arm would report a red and certify coverage of an133
# arm it never exercised.134
if ! grep -q -- "--- arm $arm" "$log"; then135
failures="$failures\n $id: never reached arm $arm (the defect was caught earlier, or the gate died in setup)"136
say " $id [arm $arm, $layer] DID NOT REACH ARM $arm -- $desc"137
return138
fi139
# (b) IT MUST NOT HAVE PRINTED THE ARM'S GREEN MARKER.140
if grep -q "F1B_GATE_ARM_${arm}_GREEN" "$log"; then141
failures="$failures\n $id: arm $arm stayed GREEN with the defect planted"142
say " $id [arm $arm, $layer] DID NOT GO RED -- $desc"143
return144
fi145
# (c) IT MUST HAVE FAILED WITH THE NAMED SIGNATURE, not merely exited badly.146
if ! grep -qF "$signature" "$log"; then147
failures="$failures\n $id: went red WITHOUT its signature (exit $status). Expected: $signature"148
say " $id [arm $arm, $layer] RED FOR THE WRONG REASON -- $desc"149
tail -3 "$log" | sed 's/^/ /'150
return151
fi152
legs_red=$((legs_red + 1))153
say " $id [arm $arm, $layer] RED as predicted -- $desc"154
}156
echo "=== F1b sabotage ==========================================="157
echo "repo : $repo_dir @ $(git rev-parse HEAD)"158
echo "work : $work"159
echo "============================================================"161
# ===========================================================================162
# ARM 1 -- the build163
# ===========================================================================164
save_file src/folio/doc/author.sgl165
# AN UNBALANCED PAREN, not a missing import.166
#167
# The first version deleted `(folio doc profile)` from the import list, and arm 1168
# stayed GREEN -- correctly. The module still COMPILES; the unbound169
# `folio-content-profile` fails at LOAD, not at compile, and arm 1's claim is170
# only that the three modules are in the build graph, asserted against their171
# `.sgb` artifacts. A leg has to break the thing the arm actually claims.172
#173
# Same lesson as S14, in the opposite direction: there, a substitution applied174
# and the defect did not bite; here, the defect bit somewhere the arm does not175
# look. Only the arm going red distinguishes either case from coverage.176
replace_once src/folio/doc/author.sgl \177
" (define (children command values)" \178
" (define (children command values"179
run_leg S1 1 product "build exited non-zero" \180
"an unbalanced paren in author.sgl; the build cannot produce the module"181
restore_all183
# ===========================================================================184
# ARM 2 -- the reader suite185
# ===========================================================================186
save_file src/folio/doc/author.sgl187
# The uniqueness check returns its input unchanged: ids are collected, compared,188
# and the verdict discarded. The shape a real regression takes.189
replace_once src/folio/doc/author.sgl \190
" (error (string-append \"folio/doc: id \" (duplicate-report folio-entries id)))))" \191
" (begin id (string-append \"folio/doc: id \" (duplicate-report folio-entries id)))))"192
run_leg S2 2 product "reader suite exited non-zero" \193
"the duplicate-id refusal computes its verdict and throws it away"194
restore_all196
save_file test/test-folio-doc-reader.sgl197
replace_once test/test-folio-doc-reader.sgl \198
" (assert-equal 'one-or-more (document-profile-multiplicity folio-document-profile)))" \199
" (assert-equal 'one (document-profile-multiplicity folio-document-profile)))"200
run_leg S3 2 assertion "reader suite exited non-zero" \201
"an expected value in the suite is altered -- the harness notices harness changes"202
restore_all204
# ARM 2's DERIVED-COUNT CLAUSE HAS NO LEG, AND THE REASON IS STRUCTURAL.205
#206
# An adversarial review correctly noted the clause was uncovered: S2 and S3 both207
# redden arm 2 through `reader suite exited non-zero`, which fires before the208
# count comparison is evaluated. A leg (S16) was written to close it -- comment209
# out one suite arm so the suite PASSES with a lower count -- and it DID NOT GO210
# RED, for a reason worth recording rather than working around.211
#212
# `expected_tests` is derived by `grep -cE '^\(test "' "$suite"`. Commenting the213
# arm out removes it from BOTH sides at once: the file declares 29 and the runner214
# reports 29. That is the clause behaving correctly, not failing -- the215
# expectation is SUPPOSED to track the file, which is what stops it going stale.216
#217
# What the clause actually guards is a RUNNER that reports fewer cases than the218
# file declares: a suite half of which failed to load, or a harness that silently219
# skipped arms. Those are defects in `sigil test`, not in anything this driver220
# can plant with a one-line substitution, and inventing a product change to make221
# the leg possible would mean building a worse product to satisfy a test -- the222
# same call SD-P7's leader ruling made when it DELETED its leg S-b rather than223
# keep one that could not fail.224
#225
# So the leg is deleted and the gap is named. A leg that always passes is worse226
# than no leg, because it reads as coverage in the transcript.227
# ===========================================================================228
# ARM 3 -- exact membership of the refused witnesses229
# ===========================================================================230
save_file src/folio/doc/profile.sgl231
# The DECLARATION is deleted while the predicate is untouched, so the profile232
# still CONSTRUCTS -- only the membership assertion can catch this. That is the233
# whole point of obligation (a): silence must be impossible.234
replace_once src/folio/doc/profile.sgl \235
" (make-phase \"p-witness\" \"a refused container\" 'planned 1 '() witness-location)))" \236
" ))"237
run_leg S4 3 product "is not exactly {milestone, note, phase, project}" \238
"the phase refused-witness declaration is deleted; the predicate still refuses it"239
restore_all241
# ===========================================================================242
# ARM 4 -- delegation rather than re-enumeration243
# ===========================================================================244
save_file src/folio/doc/profile.sgl245
# BEHAVIOUR IS IDENTICAL. This is the leg that proves arm 4's TEXTUAL half is246
# not decorative: every runtime assertion in the gate stays green, because the247
# expanded predicate admits and refuses exactly what the delegating one did.248
replace_once src/folio/doc/profile.sgl \249
" (or (block-node? x) (folio-work-item? x) (folio-relational-node? x)))" \250
" (or (block-node? x) (section? x) (paragraph? x) (code-block? x) (item-list? x) (table? x) (video? x) (term? x) (glossary? x) (admonition? x) (exercise? x) (example? x) (project-check? x) (anchor? x) (folio-work-item? x) (folio-relational-node? x)))"251
# The `block-node?` call is KEPT and the disjuncts are added BESIDE it. The first252
# version replaced the whole line, deleting the delegating call, so the arm's253
# "must call block-node?" check fired first and the RE-ENUMERATION DETECTOR --254
# the thing this leg exists to exercise, and the thing that was silently vacuous255
# under ERE -- was never reached. A leg that reddens the wrong check inside the256
# right arm is still a leg that proves nothing about the check it names.257
run_leg S5 4 product "re-enumerates core kinds instead of delegating to block-node?" \258
"the 13 disjuncts are added BESIDE the delegating call; behaviour UNCHANGED"259
restore_all261
save_file src/folio/doc/publication.sgl262
replace_once src/folio/doc/publication.sgl \263
" ((section? child)" \264
" ((and #f (section? child))"265
run_leg S6 4 product "lower-child does not descend into: section" \266
"publication stops descending into a core section -- F1a's reachable defect returns"267
restore_all269
# ===========================================================================270
# ARM 5 -- delegation parity271
# ===========================================================================272
save_file test/fixtures/f1b-parity-folio.sgd273
replace_once test/fixtures/f1b-parity-folio.sgd \274
'@note["topics/f1b-parity" title: "Delegation parity"]{' \275
'@note["topics/f1b-parity" title: "Delegation parityX"]{'276
run_leg S7 5 product "the shared body no longer starts at the same offset" \277
"the folio root line grows one byte; positions stop being comparable"278
restore_all280
save_file src/folio/doc/author.sgl281
# A LOCATION-ONLY defect: the tree's SHAPE stays identical and only the ROOT's282
# srcloc moves. It is the only evidence that arm 5 compares LOCATIONS rather than283
# structure, which is the whole reason the parity claim is stronger than a shape284
# comparison.285
#286
# The first version added `(set! body (map (lambda (c) c) body))`, which changes287
# nothing observable, and gave it the signature `F1B_PARITY_PROBE` -- a string288
# that appears in GREEN output too. Two independent defects in one leg, either of289
# which alone would have made it unable to fail.290
#291
# TWO substitutions, and the second is why: the parity fixture's shared body292
# contains ONLY sigil-doc constructs, so none of folio's eleven commands runs293
# during that comparison. The single thing folio code contributes to those nodes294
# is the LOCATION forwarded by the `author-at` wrapper -- which makes that295
# wrapper the only place a location-only defect can be planted, and makes this296
# leg a sharper test of arm 5 than perturbing a folio command would have been.297
# `make-srcloc` lives in `(sigil syntax)`, which this module does not import, so298
# the import is planted too; without it the defect fails to BUILD and reddens299
# arm 1 instead of arm 5.300
replace_once src/folio/doc/author.sgl \301
" (sigil doc model)" \302
" (sigil doc model) (sigil syntax)"303
replace_once src/folio/doc/author.sgl \304
" (lambda () (sd:author-at loc thunk))))" \305
" (lambda () (sd:author-at (make-srcloc \"synthesized\" 1 1 0 0) thunk))))"306
run_leg S8 5 product "BYTE-IDENTICAL canonical datum" \307
"every node's location is synthesized; SHAPE identical, positions moved"308
restore_all310
# ===========================================================================311
# ARM 6 -- the no-execution witness312
# ===========================================================================313
save_file src/folio/doc/reader.sgl314
replace_once src/folio/doc/reader.sgl \315
" (if (pair? author-forms)" \316
" (if #f"317
run_leg S9 6 product "NOT by the author-form refusal" \318
"the author-form refusal is disabled; the manuscript executes"319
restore_all321
save_file test/fixtures/f1b-witness-sigil.sgd322
save_file test/fixtures/f1b-witness-folio.sgd323
# THE TRIPWIRE IS REMOVED FROM BOTH FIXTURES, NOT JUST THE CONTROL.324
#325
# Editing the control alone made arm 6's fixture-PAIR assertion fire first (the326
# pair went from 4 differing lines to 6), so the leg reddened the wrong clause --327
# it proved the pair is pinned, which S-nothing needed, and left the328
# control-degradation clause uncovered. The two assertions genuinely conflict:329
# pinning the pair means no single-sided edit is possible.330
#331
# Removing the tripwire from the SHARED body keeps the pair intact and still332
# degrades the control, which is the property under test: with no333
# `F1B-WITNESS-EXPR-RAN` anywhere, channel 1 cannot be shown able to fire, and334
# an absence proven by a control that cannot fire proves nothing.335
#336
# ANCHOR WITH NO BACKSLASH ESCAPE. The original anchor was the whole line,337
# `@(display "F1B-WITNESS-EXPR-RAN\n")`, and the `\n` in it was interpreted by338
# `awk -v` into a real newline -- so `grep -cF` found the anchor, `index()` never339
# did, and the substitution silently did nothing while the guard reported340
# success. A prefix of the same line carries the identity without the escape.341
# The replacement is a blank line rather than `;; removed`, because `;;` is342
# PROSE in a `.sgd` manuscript and not a comment.343
replace_once test/fixtures/f1b-witness-sigil.sgd \344
'@(display "F1B-WITNESS-EXPR-RAN' \345
''346
replace_once test/fixtures/f1b-witness-folio.sgd \347
'@(display "F1B-WITNESS-EXPR-RAN' \348
''349
run_leg S10 6 product "channel 1 cannot fire" \350
"the CONTROL loses a tripwire -- an absence proven by a broken control means nothing"351
restore_all353
# ===========================================================================354
# ARM 7 -- publication fallback355
# ===========================================================================356
save_file src/folio/doc/unresolved.sgl357
# The marker keeps its PREFIX and loses its IDENTITY -- exactly the defect358
# `make-query` exists to prevent, one layer down: a marker that satisfies every359
# "the marker is present" assertion while naming nothing.360
#361
# The first version replaced the prefix instead, and arm 7 stayed GREEN: the362
# prefix assertion was satisfied by the sibling TRANSCLUSION marker and the query363
# id survived elsewhere in the rendering. Nothing the probe checked actually364
# depended on the query marker existing. Both the leg and the probe were fixed --365
# the probe now asserts the WHOLE marker text, derived from the same function the366
# renderer uses.367
replace_once src/folio/doc/unresolved.sgl \368
' (if id (string-append " " id) "")' \369
' ""'370
run_leg S11 7 product "names the unresolved query by id" \371
"the query marker keeps its prefix and stops naming its query"372
restore_all374
# ===========================================================================375
# ARM 8 -- the probe harnesses can go red376
# ===========================================================================377
save_file test/f1b-parity-probe.sgl378
replace_once test/f1b-parity-probe.sgl \379
'(if (getenv "F1B_PROBE_SELFTEST")' \380
'(if #f'381
run_leg S12 8 assertion "self-test EXITED 0 with a failure injected" \382
"a probe loses its self-test hook -- its green stops meaning anything"383
restore_all385
# ===========================================================================386
# ARM 9 -- reader-tier passivity387
# ===========================================================================388
save_file src/folio/doc/reader.sgl389
replace_once src/folio/doc/reader.sgl \390
" (import (sigil core) (sigil syntax) (sigil doc reader))" \391
" (import (sigil core) (sigil syntax) (sigil doc reader) (sigil process))"392
run_leg S13 9 product "which is outside the reader-tier allowlist" \393
"the reader imports a subprocess runner"394
restore_all396
# ===========================================================================397
# ARM 10 -- reader/inspector layering398
# ===========================================================================399
save_file src/folio/doc/author.sgl400
# The identity COLLECTION is neutered, so the scan runs over nothing and finds401
# nothing. The first version prepended a no-op expression to the definition line402
# and left the original body in place, so the check still ran and still raised --403
# a substitution that APPLIED (the file changed) and was behaviourally inert.404
# The file-changed guard cannot catch that; only a leg whose defect is chosen for405
# its EFFECT rather than for its edit can.406
replace_once src/folio/doc/author.sgl \407
" (let ((folio-entries (collect-identities roots folio-identity-of))" \408
" (let ((folio-entries '())"409
run_leg S14 10 product "the READER did not refuse the duplicate document" \410
"identity collection is neutered; the reader stops refusing duplicates"411
restore_all413
# ARM 10's INSPECTOR HALF, which S14 cannot reach: S14 reddens the reader-side414
# assertion, which runs FIRST, so the three inspector clauses -- the half the415
# arm's own comment calls "the half that would silently rot if the two layers416
# were ever harmonised" -- had nothing showing they can fire. Named by an417
# adversarial review.418
#419
# The duplicate fixture is edited instead of the product, so the reader-side420
# assertion still fires normally and the inspector genuinely sees a document421
# with no duplicate.422
save_file test/fixtures/f1b-duplicate.sgd423
replace_once test/fixtures/f1b-duplicate.sgd \424
'@task["t-dup" title: "The second declaration, further down" status: '"'"'active]{' \425
'@task["t-not-dup" title: "The second declaration, further down" status: '"'"'active]{'426
run_leg S17 10 product "the inspector reported" \427
"the duplicate is removed; the inspector must stop reporting one"428
restore_all430
# ===========================================================================431
# ARM 7 again -- the DEEPER descent, which the first section-only fix missed432
# ===========================================================================433
save_file src/folio/doc/publication.sgl434
# Descend into `section` only, which is what the code did until an adversarial435
# review showed the completeness argument was a non-sequitur. Reachable by an436
# ordinary manuscript: `@aside{ @section{ @task } }`.437
replace_once src/folio/doc/publication.sgl \438
" ((admonition? child)" \439
" ((and #f (admonition? child))"440
run_leg S18 7 product "publication probe exited non-zero" \441
"the lowering stops descending into an ADMONITION; folio nodes under one are dropped"442
restore_all444
# ===========================================================================445
# ARM 11 -- suite arithmetic446
# ===========================================================================447
save_file test/gate-f1b.sh448
replace_once test/gate-f1b.sh \449
"baseline_tests=351" \450
"baseline_tests=350"451
run_leg S15 11 assertion "suite arithmetic does not close" \452
"the recorded baseline is altered -- the arithmetic must not be self-satisfying"453
restore_all455
# ---------------------------------------------------------------------------456
echo457
echo "COVERAGE BY ARM -- the number that matters. A suite that only ever"458
echo "reddens one instrument has not tested the others."459
echo " arm 1 build <- S1"460
echo " arm 2 reader suite <- S2 (product), S3 (assertion)"461
echo " ...its DERIVED-COUNT clause is DELIBERATELY UNCOVERED. It guards"462
echo " the RUNNER reporting fewer cases than the file declares; the"463
echo " expectation tracks the file BY DESIGN, so no substitution here"464
echo " can separate the two. Leg S16 was written, measured unable to go"465
echo " red, and DELETED rather than kept as false coverage."466
echo " arm 3 exact membership <- S4"467
echo " arm 4 delegation spelling <- S5 (behaviour-identical), S6"468
echo " arm 5 parity <- S7, S8"469
echo " arm 6 no-execution witness <- S9 (product), S10 (control)"470
echo " arm 7 publication fallback <- S11, S18 (the deeper container descent)"471
echo " arm 8 harness self-tests <- S12 (assertion)"472
echo " arm 9 reader-tier passivity <- S13"473
echo " arm 10 reader/inspector <- S14 (reader half), S17 (inspector half)"474
echo " arm 11 suite arithmetic <- S15 (assertion)"475
echo "Every arm has a row. S5 is the load-bearing one: it is BEHAVIOUR-IDENTICAL,"476
echo "so it is the only evidence that arm 4's textual half is not decorative."477
echo478
if [ -n "$failures" ]; then479
echo "F1B_SABOTAGE legs=$legs_run red=$legs_red -- FAILURES:"480
printf "%b\n" "$failures"481
exit 1482
fi483
echo "F1B_SABOTAGE legs=$legs_run red=$legs_red"484
echo "F1B_SABOTAGE_COMPLETE all legs red at their named arm, all targets restored by sha256"