AtlatestRepositorysigil-doc

sigil-doc / tree / testsabotage-sd-p4.sh

1#!/usr/bin/env bash
2# SD-P4 sabotage driver.
3#
4# Seven product-layer legs and one assertion-layer leg. For each: plant the defect
5# the arm exists to catch, prove the arm goes RED, restore, and prove the
6# restoration by CONTENT HASH rather than by the exit code of the restore.
7#
8# Two properties this driver asserts about ITSELF, because a sabotage run that
9# quietly did nothing is indistinguishable from a passing one:
10# * the edit must actually change the file (hash before != hash after), or the
11# leg is vacuous and the run aborts;
12# * the red must be an ASSERTION failure from a suite that compiled and ran --
13# the driver requires the test runner's own "N passed, M failed" line, so a
14# compile error cannot be mistaken for a demonstrated red.
15set -euo pipefail
16repo_dir="$(cd "$(dirname "$0")/.." && pwd)"
17cd "$repo_dir"
18sigil="${SIGIL_D1_COMPILER:-$repo_dir/../task-sigil-language-readers/build/dev/bin/sigil}"
19export SIGIL_REDIRECTS="${SIGIL_D1_REDIRECTS:-$repo_dir/dev-redirects.sgl}"
20work=$(mktemp -d "$repo_dir/build/sd-p4-sabotage.XXXXXX")
21export XDG_CACHE_HOME="$work/cache"
23parser=src/sigil/doc/private/parser.sgl
24inspect=src/sigil/doc/inspect.sgl
25suite=test/test-passive-parse.sgl
26reader=test/test-reader.sgl
28pristine="$work/pristine"
29mkdir -p "$pristine"
30for f in "$parser" "$inspect" "$suite" "$reader"; do
31 mkdir -p "$pristine/$(dirname "$f")"
32 cp "$f" "$pristine/$f"
33done
35hash_of() { sha256sum "$1" | cut -d' ' -f1; }
37restore() {
38 for f in "$parser" "$inspect" "$suite" "$reader"; do
39 cp "$pristine/$f" "$f"
40 test "$(hash_of "$f")" = "$(hash_of "$pristine/$f")" || {
41 echo "RESTORE FAILED for $f" >&2; exit 1; }
42 done
45# apply <file> <old> <new> -- exact string replacement, aborting if it no-ops.
46apply() {
47 local file="$1" old="$2" new="$3" before after
48 before=$(hash_of "$file")
49 python3 - "$file" "$old" "$new" <<'PY'
50import sys
51path, old, new = sys.argv[1], sys.argv[2], sys.argv[3]
52text = open(path).read()
53if text.count(old) != 1:
54 sys.exit("sabotage target appears %d times in %s; expected exactly 1" % (text.count(old), path))
55open(path, "w").write(text.replace(old, new))
56PY
57 after=$(hash_of "$file")
58 test "$before" != "$after" || { echo "SABOTAGE DID NOT CHANGE $file" >&2; exit 1; }
61# expect_suite_red <label> <suite> <log>
62expect_suite_red() {
63 local label="$1" target="$2" log="$3"
64 set +e
65 setsid -w "$sigil" -L src -L test test "$target" >"$log" 2>&1
66 local status=$?
67 set -e
68 echo "--- $label: exit=$status"
69 # A red must come from a suite that COMPILED AND RAN. Without this the leg
70 # would accept a compile error as a demonstrated failure.
71 rg -q '\d+ passed, \d+ failed' "$log" || {
72 echo "$label: no runner summary -- the suite did not run, this is NOT a demonstrated red" >&2
73 tail -30 "$log" >&2; exit 1; }
74 test "$status" -ne 0 || { echo "$label: SABOTAGE DID NOT TURN THE GATE RED" >&2; exit 1; }
75 rg -o '\d+ passed, \d+ failed' "$log" | head -1
76 rg -o '^\s+name: .*' "$log" 2>/dev/null | head -5 || true
79echo "=============================================================="
80echo "LEG 1 (product) -- the inspector evaluates author expressions."
81echo "The §0-named sabotage: author code runs during inspection."
82echo "=============================================================="
83apply "$inspect" \
84 "expression: (if (and (not command?) (eq? head 'expr)) (caddr node) #f)" \
85 "expression: (if (and (not command?) (eq? head 'expr)) (begin (eval (syntax->datum (caddr node))) (caddr node)) #f)"
86witness_log="$work/leg1-witness.log"
87set +e
88setsid -w "$sigil" -L src -L test eval -f test/sd-p4-witness-probe.sgl >"$witness_log" 2>&1
89set -e
90if rg -q 'SDP4-WITNESS-' "$witness_log"; then
91 echo "LEG 1 RED (correct): author markers appeared during passive inspection:"
92 rg -o 'SDP4-WITNESS-[A-Z-]*' "$witness_log" | sort -u
93else
94 echo "LEG 1 FAILED TO GO RED -- the witness cannot see execution" >&2
95 cat "$witness_log" >&2; exit 1
96fi
97restore
99echo "=============================================================="
100echo "LEG 2 (product) -- a node's end span is off by one."
101echo "=============================================================="
102apply "$inspect" "(define (node-end node) (list-ref node 4))" \
103 "(define (node-end node) (- (list-ref node 4) 1))"
104expect_suite_red "LEG 2" "$suite" "$work/leg2.log"
105restore
107echo "=============================================================="
108echo "LEG 3 (product) -- the location index counts SCALARS, not UTF-8"
109echo "bytes. The round-trip arm stays green under this; only the"
110echo "byte/scalar arm can see it. That is why they are separate arms."
111echo "=============================================================="
112apply "$parser" \
113 " (cond ((<= n #x7f) 1) ((<= n #x7ff) 2) ((<= n #xffff) 3) (else 4))))" \
114 " (cond ((<= n #x7f) 1) ((<= n #x7ff) 1) ((<= n #xffff) 1) (else 1))))"
115expect_suite_red "LEG 3" "$suite" "$work/leg3.log"
116restore
118echo "=============================================================="
119echo "LEG 4 (product) -- restore the swallowed delegated diagnostic."
120echo "=============================================================="
121apply "$parser" \
122 " (if delegated-failure
123 (string-append
124 reason
125 \"; while delimiting this argument group the standard reader reported: \"
126 (exception-message delegated-failure)
127 \" (a '(' named there is this parser's substitution for the opening '[')\")
128 reason)))" \
129 " reason))"
130expect_suite_red "LEG 4" "$suite" "$work/leg4.log"
131restore
133echo "=============================================================="
134echo "LEG 5 (product) -- reinstate MODULE-LEVEL parser state. The"
135echo "exact defect I3 names: a second parse observes the first."
136echo "=============================================================="
137apply "$parser" \
138 " (define (parse-document-nodes source source-name header-end)
139 (let* ((state (make-parser-state source source-name))" \
140 " (define cached-parser-state #f)
141 (define (parse-document-nodes source source-name header-end)
142 (let* ((state (or cached-parser-state
143 (let ((fresh (make-parser-state source source-name)))
144 (set! cached-parser-state fresh)
145 fresh)))"
146expect_suite_red "LEG 5" "$suite" "$work/leg5.log"
147# The grep arm must independently see the reinstated module-level assignment.
148rg -q '\(set! ' "$parser" && echo "LEG 5: gate ARM 03's set! probe also fires (correct)"
149restore
151echo "=============================================================="
152echo "LEG 6 (product) -- perturb the per-parse step count, so the"
153echo "assertion MOVED off the old module-level telemetry global is"
154echo "shown to be able to fail in its new location."
155echo "=============================================================="
156apply "$parser" "scalars: scalars byte-length: byte steps: visited)" \
157 "scalars: scalars byte-length: byte steps: (+ visited 1))"
158expect_suite_red "LEG 6a (passive suite)" "$suite" "$work/leg6a.log"
159expect_suite_red "LEG 6b (re-pointed D1 assertion)" "$reader" "$work/leg6b.log"
160restore
162echo "=============================================================="
163echo "LEG 7 (assertion layer) -- invert the byte-span comparison in"
164echo "the harness itself, proving the harness notices the harness."
165echo "=============================================================="
166apply "$suite" \
167 " (assert-equal (byte-offset source start) (srcloc-position loc))" \
168 " (assert-equal (+ 1 (byte-offset source start)) (srcloc-position loc))"
169expect_suite_red "LEG 7" "$suite" "$work/leg7.log"
170restore
172echo "=============================================================="
173echo "LEG 8 (product) -- restore a MODULE-LEVEL SHARED LOCATION INDEX"
174echo "spelled WITHOUT set!, exactly as the adversarial review did to"
175echo "defeat the first version of these gates. Every other arm stayed"
176echo "green under this; the discriminating test must go red."
177echo "=============================================================="
178apply "$parser" \
179 " (define (position-at state end) (vector-ref (parser-state-bytes state) end))
180 (define (cursor-at state end)
181 (list (vector-ref (parser-state-lines state) end)
182 (vector-ref (parser-state-columns state) end)))" \
183 " (define shared-index-cell (make-vector 1 #f))
184 (define (shared-index state) (or (vector-ref shared-index-cell 0) state))
185 (define (position-at state end) (vector-ref (parser-state-bytes (shared-index state)) end))
186 (define (cursor-at state end)
187 (list (vector-ref (parser-state-lines (shared-index state)) end)
188 (vector-ref (parser-state-columns (shared-index state)) end)))"
189apply "$parser" \
190 " (parser-state source: source source-name: source-name
191 bytes: bytes lines: lines columns: columns
192 scalars: scalars byte-length: byte steps: visited)" \
193 " (let ((fresh (parser-state source: source source-name: source-name
194 bytes: bytes lines: lines columns: columns
195 scalars: scalars byte-length: byte steps: visited)))
196 (vector-set! shared-index-cell 0 fresh)
197 fresh)"
198# The `set!` grep is deliberately expected to stay BLIND here -- that is the
199# review's point, and the reason the behavioural test above had to exist.
200if rg -q '\(set!' "$parser"; then
201 echo "LEG 8: unexpected -- this sabotage was supposed to contain no set!" >&2; exit 1
202fi
203echo "LEG 8: gate ARM 03's set! grep is BLIND to this defect (0 matches), as expected."
204expect_suite_red "LEG 8" "$suite" "$work/leg8.log"
205restore
207echo
208echo "All files restored; hashes verified against the pristine copies."
209for f in "$parser" "$inspect" "$suite" "$reader"; do
210 printf '%s %s\n' "$(hash_of "$f")" "$f"
211done
212echo "SD_P4_SABOTAGE_COMPLETE"