AtlatestRepositorysigil-doc

sigil-doc / tree / testsabotage-sd-p2.sh

1#!/usr/bin/env bash
2# SD-P2 sabotage harness.
3#
4# A gate that has never been shown to fail is not a gate. Each leg plants ONE
5# specific defect, runs the arm meant to catch it, and requires that arm to go
6# red WITH A NAMED SIGNATURE - not merely to exit non-zero, which a compile
7# error, a missing binary (127) or a non-executable script (126) would also do.
8#
9# EVERY INPUT IS BROKEN SEPARATELY. Breaking the scheme allowlist does not
10# exercise the control-byte check, and neither exercises the authority check;
11# breaking the manuscript spelling does not exercise the model-kind spelling.
12# A red on one says nothing about the others, so each gets its own leg.
14# LAYERS ARE MIXED DELIBERATELY. PRODUCT legs break the shipped behaviour the
15# gate describes; ASSERTION legs break only the harness; the INSTRUMENT leg
16# breaks the gate's own searcher. A suite sabotaged at the assertion layer
17# alone is consistent with a stale module, a fixture that reimplements the
18# behaviour, and a wrong load path - all three of which have happened here.
20# ONE HONEST GAP, stated rather than papered over: there is no PRODUCT-layer
21# sabotage for HTML escaping, because the escaping lives in (sigil sxml), which
22# this branch must not modify. The escaping leg is therefore ASSERTION-layer
23# and is labelled as such. What it does prove is that the exact-escaped-string
24# assertion is load-bearing rather than decorative.
26# Restoration is verified by CONTENT HASH, never by the exit code of the restore.
28# THE SIGNATURE IS SEARCHED IN THE ARM'S OUTPUT FILE ONLY, NEVER IN THE COMBINED
29# TRANSCRIPT. The first version of this harness wrote "expected red signature:
30# <sig>" into the transcript and then grepped that same transcript for <sig>, so
31# the check MATCHED ITS OWN HEADER and could never fail. All twelve legs
32# reported a verified named signature while the signature half was inert - and it
33# was caught only because one leg's predicted count was wrong and the harness
34# cheerfully accepted the real, different one. Same failure family as a grep
35# whose anchor matches the script's own prose.
36set -euo pipefail
38repo_dir="$(cd "$(dirname "$0")/.." && pwd)"
39cd "$repo_dir"
40leg="${1:?usage: sabotage-sd-p2.sh <leg>|all}"
41out_dir="${SD_P2_SABOTAGE_OUT:-$repo_dir/build/sabotage}"
42mkdir -p "$out_dir"
44sigil="${SIGIL_D1_COMPILER:-$repo_dir/../task-sigil-language-readers/build/dev/bin/sigil}"
45sigil_sxml_root="${SIGIL_SXML_ROOT:-$repo_dir/../sigil-sxml}"
46export SIGIL_REDIRECTS="${SIGIL_D1_REDIRECTS:-$repo_dir/dev-redirects.sgl}"
48arm_url_link() {
49 XDG_CACHE_HOME="$(mktemp -d)" "$sigil" -L "$sigil_sxml_root/src" -L src -L test \
50 test test/test-url-link.sgl --report
52arm_video() {
53 XDG_CACHE_HOME="$(mktemp -d)" "$sigil" -L "$sigil_sxml_root/src" -L src -L test \
54 test test/test-video-destination.sgl --report
56arm_rename() { bash test/gate-sd-p2-rename.sh; }
57arm_backend() { bash test/gate-sd-p2-backend-refusal.sh; }
59ALL_LEGS="scheme-allowlist control-bytes authority-check html-backend-check
60 typst-backend-check escaping-assertion expected-target
61 rename-manuscript rename-model-kind rename-searcher
62 video-source-check video-destination-check"
64describe_leg() {
65 case "$1" in
66 scheme-allowlist)
67 layer=PRODUCT; target=src/sigil/doc/model.sgl
68 expr='s|(if (not (string-member? scheme allowed))|(if #f|'
69 signature='8 passed, 1 failed'; arm=arm_url_link ;;
70 control-bytes)
71 layer=PRODUCT; target=src/sigil/doc/model.sgl
72 expr='s|(if (not (every printable-target-char? (string->list value)))|(if #f|'
73 signature='8 passed, 1 failed'; arm=arm_url_link ;;
74 authority-check)
75 layer=PRODUCT; target=src/sigil/doc/model.sgl
76 expr='s|^ (define (require-authority who value scheme)$| (define (require-authority who value scheme) value)\n (define (neutered-require-authority who value scheme)|'
77 signature='8 passed, 1 failed'; arm=arm_url_link ;;
78 html-backend-check)
79 layer=PRODUCT; target=src/sigil/doc/render/html.sgl
80 expr='s|(let ((url (safe-external-url (url-link-href node)))|(let ((url (url-link-href node))|'
81 signature='HTML backend accepted'; arm=arm_backend ;;
82 typst-backend-check)
83 layer=PRODUCT; target=src/sigil/doc/render/typst.sgl
84 expr='s|(typst-string (safe-link-destination (cadr node)))|(typst-string (cadr node))|'
85 signature='print backend accepted'; arm=arm_backend ;;
86 escaping-assertion)
87 layer=ASSERTION; target=test/test-url-link.sgl
88 expr='s|a=1\&amp;b=\&lt;2\&gt;|a=1\&b=<2>|'
89 signature='8 passed, 1 failed'; arm=arm_url_link ;;
90 expected-target)
91 layer=ASSERTION; target=test/test-url-link.sgl
92 expr='s|html_node/Ports.html|html_node/WRONG.html|'
93 # TWO tests consume `course-targets` - the reader test and the
94 # three-backend render test - so flipping one target flips both. The
95 # observed count is the signature; guessing 8/1 here is what exposed the
96 # unfailable-signature defect in this harness.
97 signature='7 passed, 2 failed'; arm=arm_url_link ;;
98 rename-manuscript)
99 layer=PRODUCT; target=book/practical-sigil/01-useful-first-script.sgd
100 expr='s|^@aside$|@note|'
101 # The arm now reports the manuscript and the code paths separately, so the
102 # signature names WHICH assertion fired rather than a phrase common to both.
103 signature='a manuscript still uses @note as an admonition'; arm=arm_rename ;;
104 rename-model-kind)
105 layer=PRODUCT; target=test/test-glossary.sgl
106 expr="s|make-admonition 'aside|make-admonition 'note|"
107 signature='the admonition kind or binding is still spelled note'; arm=arm_rename ;;
108 rename-searcher)
109 layer=INSTRUMENT; target=test/gate-sd-p2-rename.sh
110 # Break the arm's own searcher. It must refuse to report an absence it
111 # can no longer detect, rather than passing because it found nothing.
112 expr="s|^code_spelling=|code_spelling='NOTHINGMATCHESTHIS' # |"
113 signature='cannot be trusted to report an absence'; arm=arm_rename ;;
114 video-source-check)
115 layer=PRODUCT; target=src/sigil/doc/model.sgl
116 expr="s|(if (eq? provider 'url) (require-link-target|(if #f (require-link-target|"
117 signature='4 passed, 1 failed'; arm=arm_video ;;
118 video-destination-check)
119 layer=PRODUCT; target=src/sigil/doc/render/typst.sgl
120 expr='s|(safe-link-destination (video-destination (cadr node) (caddr node)))|(video-destination (cadr node) (caddr node))|'
121 # TWO tests rest on this check - the refusal test and the interpolation
122 # test both require hostile ids to raise - so removing it flips both.
123 # That the check is load-bearing in two independent places is a property
124 # worth keeping, and the signature records the observed count, not a guess.
125 signature='3 passed, 2 failed'; arm=arm_video ;;
126 *) echo "unknown leg: $1" >&2; exit 2 ;;
127 esac
130run_leg() {
131 local leg="$1"
132 describe_leg "$leg"
133 local backup="$out_dir/$leg.original"
134 cp "$target" "$backup"
135 local before after restored status transcript
136 before="$(sha256sum "$target" | cut -d' ' -f1)"
138 sed -i "$expr" "$target"
139 after="$(sha256sum "$target" | cut -d' ' -f1)"
140 if [ "$before" = "$after" ]; then
141 cp "$backup" "$target"
142 echo "SABOTAGE $leg IS VACUOUS: the edit changed nothing in $target" >&2
143 return 1
144 fi
146 transcript="$out_dir/$leg-red.txt"
147 # The arm's output goes to its OWN file, and the signature is searched only
148 # there. Searching the combined transcript would match the header line this
149 # script writes naming the expected signature - the check would then be
150 # unfailable, which is exactly the defect it exists to catch elsewhere.
151 local arm_output="$out_dir/$leg-arm-output.txt"
152 {
153 echo "leg=$leg layer=$layer"
154 echo "target=$target"
155 echo "hash original=$before"
156 echo "hash sabotaged=$after"
157 echo "required red signature (searched in $leg-arm-output.txt only): $signature"
158 echo "--- planted defect ---"
159 diff "$backup" "$target" || true
160 } >"$transcript"
162 status=0
163 "$arm" >"$arm_output" 2>&1 || status=$?
164 {
165 echo "--- arm output ---"
166 cat "$arm_output"
167 } >>"$transcript"
169 cp "$backup" "$target"
170 restored="$(sha256sum "$target" | cut -d' ' -f1)"
171 if [ "$restored" != "$before" ]; then
172 echo "RESTORE FAILED for $target: $restored != $before" >&2
173 return 1
174 fi
175 echo "--- restored, hash=$restored, matches original ---" >>"$transcript"
177 if [ "$status" -eq 0 ]; then
178 echo "SABOTAGE $leg DID NOT GO RED - the gate does not catch this" | tee -a "$transcript" >&2
179 return 1
180 fi
181 if ! rg -q -- "$signature" "$arm_output"; then
182 echo "SABOTAGE $leg went red for an UNNAMED reason; expected: $signature" \
183 | tee -a "$transcript" >&2
184 return 1
185 fi
186 echo "SABOTAGE_RED_AS_EXPECTED leg=$leg layer=$layer signature=\"$signature\" restored=ok" \
187 | tee -a "$transcript"
190if [ "$leg" = "all" ]; then
191 for one in $ALL_LEGS; do run_leg "$one"; done
192 echo "SD_P2_SABOTAGE_ALL_LEGS_RED"
193else
194 run_leg "$leg"
195fi