AtlatestRepositorysigil-doc

sigil-doc / tree / testsd-p8-lowering-dump.sgl

1;;; SD-P8 -- canonical dump of LOWERED output, normalized so that the `#lang`
2;;; header's LENGTH is invisible and nothing else is.
3;;;
4;;; WHY THIS EXISTS AND WHY IT IS NOT `test/sd-p3-lowering-dump.sgl`.
5;;;
6;;; SD-P3's dump is the AFTER half of a differential whose BEFORE half is a
7;;; committed table of sha256s taken from a pre-SD-P3 tree. D8 migrates the
8;;; `#lang` line of 17 manuscripts, which moves every byte offset in each
9;;; of them by the header's length delta, so every one of those hashes changes.
10;;; The BEFORE half stores hashes and not sections, so no normalization can be
11;;; applied to it retroactively -- the differential cannot be rescued, only
12;;; retired. It is retired in `test/gate-sd-p3.sh` arms 3 and 4, under the
13;;; standing pattern Fable's 2026-08-11 ruling recorded for exactly this case,
14;;; and this file is the replacement that pattern asks for: SD-P8's own
15;;; differential, scoped to SD-P8's own base.
16;;;
17;;; TWO PROPERTIES THIS INSTRUMENT IS BUILT AROUND.
18;;;
19;;; * BOTH SIDES RUN THE SAME FILE. `test/gate-sd-p8.sh` copies this script
20;;; into a detached worktree at the base SHA and runs it there, then runs
21;;; it here. An instrument that differs between the two sides of a
22;;; differential measures itself.
23;;;
24;;; * EVERY LOCATION IS EMITTED RELATIVE TO THE MANUSCRIPT'S HEADER END, so
25;;; `#lang sigil/doc` (16 bytes) and `#lang sigil/doc/book` (21 bytes)
26;;; produce identical offsets for identical prose. That is the ONE
27;;; difference this phase intends, and making it invisible is what leaves
28;;; the comparison sensitive to every difference it does NOT intend.
29;;; Generated locations (`<sigil/doc-generated>`, byte=0) are NOT shifted:
30;;; they are not manuscript positions and subtracting from them would be
31;;; arithmetic on a value that means nothing.
32;;;
33;;; The profile comes from the manuscript's own `#lang` id rather than from the
34;;; three-argument default-profile entry point. In the base tree `sigil/doc`
35;;; resolves to that tree's `sigil-doc-profile` (roots `(chapter)`); here it
36;;; resolves to this tree's (roots `(article)`). Both are the tree's own answer
37;;; to "what does this language admit", which is what makes the same file run
38;;; correctly on both sides. `sigil/doc/book` is constructed inline rather than
39;;; imported from `(sigil doc book reader)`, because that module does not exist
40;;; in the base tree and an import of it would make this script unrunnable
41;;; there -- which would silently reduce the differential to one side.
42;;;
43;;; Nothing is evaluated: lowering is a pure source-to-syntax transformation.
44(import (sigil core) (sigil fs) (sigil language)
45 (sigil doc private parser))
47;; ------------------------------------------------------------------ output
49(define header-end 0)
51;; A manuscript location is emitted header-relative; a generated one is not a
52;; manuscript position and is emitted as it stands. The discriminator is the
53;; file name the location carries, so this cannot accidentally shift a location
54;; that has no manuscript behind it.
55(define (emit-location loc depth)
56 (if (or (not loc) (> depth 40))
57 (display "#f")
58 (let* ((file (srcloc-file loc))
59 (generated? (string=? file "<sigil/doc-generated>"))
60 (position (srcloc-position loc)))
61 (display "[") (write file)
62 (display " ") (display (srcloc-line loc))
63 (display ":") (display (srcloc-column loc))
64 (display (if generated? " byte=" " rel="))
65 (display (if generated? position (- position header-end)))
66 (display " span=") (display (srcloc-span loc))
67 (let ((origin (srcloc-origin loc)))
68 (if origin
69 (begin (display " <-") (emit-location origin (+ depth 1)))
70 #f))
71 (display "]"))))
73;; Walk the improper-list structure explicitly. `write` alone would not show
74;; syntax wrappers or locations, which are the part most likely to move.
75;;
76;; THE `make-srcloc` SPECIAL CASE IS NOT COSMETIC. The lowered forms carry the
77;; manuscript's byte offsets TWICE: once as the syntax objects' own srclocs,
78;; and once as literal integer DATA inside the `(make-srcloc <file> <line>
79;; <col> <pos> <span>)` calls the parser generates for `author-at`. Normalizing
80;; only the first leaves the second absolute, and every migrated manuscript
81;; then differs by the header delta in a place that looks like content. The
82;; discriminator is structural -- head symbol `make-srcloc`, six elements, and
83;; a file argument that is this manuscript rather than the generated marker --
84;; so no ordinary integer in a manuscript can be caught by it.
85(define (syntax-datum-of value)
86 (if (syntax? value) (syntax-datum value) value))
88;; THE GUARD IS THE MANUSCRIPT'S OWN NAME, not merely "not the generated
89;; marker". An earlier version accepted any six-element `make-srcloc` call whose
90;; second element was any string, while its comment claimed a check the code did
91;; not make: an author writing `(make-srcloc "anything.sgd" 1 1 100 5)` inside a
92;; top-level `@(...)` would have had element 4 silently rewritten, hiding a real
93;; change to that literal. Nothing in the corpus does that today -- which is
94;; exactly the condition under which an over-broad predicate goes unnoticed.
95(define current-manuscript "")
97(define (generated-srcloc-call? value)
98 (and (pair? value)
99 (eq? (syntax-datum-of (car value)) 'make-srcloc)
100 (= (length value) 6)
101 (let ((file (syntax-datum-of (cadr value))))
102 (and (string? file) (string=? file current-manuscript)))))
104(define expression-binding-prefix "#%sigil/doc-expression:")
105(define (expression-binding? name)
106 (let ((n (string-length name)) (m (string-length expression-binding-prefix)))
107 (and (> n m) (string=? (substring name 0 m) expression-binding-prefix))))
109(define (emit value)
110 (cond
111 ((syntax? value)
112 (display "#<")
113 (emit-location (syntax-srcloc value) 0)
114 (display " ")
115 (emit (syntax-datum value))
116 (display ">"))
117 ((null? value) (display "()"))
118 ((generated-srcloc-call? value)
119 ;; Same rendering as the general list path, with element 4 -- the byte
120 ;; position -- rewritten header-relative and RELABELLED, so a reader can
121 ;; see which number was normalized rather than having to know.
122 (display "(")
123 (let loop ((rest value) (index 0))
124 (cond
125 ((null? rest) #t)
126 (else
127 (if (= index 0) #t (display " "))
128 (if (= index 4)
129 (begin (display "rel:")
130 (display (- (syntax-datum-of (car rest)) header-end)))
131 (emit (car rest)))
132 (loop (cdr rest) (+ index 1)))))
133 (display ")"))
134 ((pair? value)
135 (display "(")
136 (let loop ((rest value) (first? #t))
137 (cond
138 ((null? rest) #t)
139 ((pair? rest)
140 (if first? #t (display " "))
141 (emit (car rest))
142 (loop (cdr rest) #f))
143 (else (display " . ") (emit rest))))
144 (display ")"))
145 ;; The parser spells a hoisted author expression's private binding
146 ;; `#%sigil/doc-expression:<absolute byte offset>`
147 ;; (`private/parser.sgl:595`). That is the THIRD place a manuscript's byte
148 ;; offsets reach the lowered output -- after the syntax srclocs and the
149 ;; `make-srcloc` data -- and it is the least obvious, because it wears the
150 ;; clothes of a name rather than of a position. It is normalized for the
151 ;; same reason as the other two, and it is worth knowing on its own: a
152 ;; manuscript's generated binding names are NOT stable under an edit to its
153 ;; `#lang` line.
154 ;;
155 ;; A SUFFIX THAT IS NOT A NUMBER IS REPORTED, NOT SUBTRACTED. `string->number`
156 ;; returns `#f` on a name this code did not expect, and `(- #f header-end)`
157 ;; would raise from inside the emitter -- killing the dump and surfacing in
158 ;; the gate as "the dump printed no completion marker", a red about a missing
159 ;; marker for what is actually an unrecognised binding name. Emitting the
160 ;; symbol verbatim instead makes it a DIFFERENCE the differential reports
161 ;; against the other side, which is the correct place for a surprise.
162 ((and (symbol? value) (expression-binding? (symbol->string value)))
163 (let* ((name (symbol->string value))
164 (suffix (substring name
165 (string-length expression-binding-prefix)
166 (string-length name)))
167 (offset (string->number suffix)))
168 (if (and offset (integer? offset))
169 (begin (display "#%sigil/doc-expression:rel:")
170 (display (- offset header-end)))
171 (begin (display "#%sigil/doc-expression:UNRECOGNISED-SUFFIX:")
172 (display suffix)))))
173 (else (write value))))
175;; ----------------------------------------------------------------- profiles
177;; `sigil/doc` is the TREE'S OWN base profile, deliberately: this script is run
178;; in two trees whose base languages differ, and asking each tree what its base
179;; language admits is the only way one file can be correct in both.
180(define book-profile (make-document-profile "sigil/doc/book" '(chapter) 'one))
182(define (profile-for id)
183 (cond ((string=? id "sigil/doc") sigil-doc-profile)
184 ((string=? id "sigil/doc/book") book-profile)
185 (else #f)))
187;; ------------------------------------------------------------------ corpus
189(define (basename path)
190 (let loop ((i (string-length path)))
191 (cond ((= i 0) path)
192 ((char=? (string-ref path (- i 1)) #\/) (substring path i (string-length path)))
193 (else (loop (- i 1))))))
195(define (has-suffix? name suffix)
196 (let ((n (string-length name)) (m (string-length suffix)))
197 (and (> n m) (string=? (substring name (- n m) n) suffix))))
199;; `directory-list` order is filesystem order and is NOT stable across two
200;; checkouts. Sorting is what makes the two sides comparable line by line
201;; rather than only as sets.
202(define (string-before? a b)
203 (let ((na (string-length a)) (nb (string-length b)))
204 (let loop ((i 0))
205 (cond ((= i na) (> nb na))
206 ((= i nb) #f)
207 ((char<? (string-ref a i) (string-ref b i)) #t)
208 ((char<? (string-ref b i) (string-ref a i)) #f)
209 (else (loop (+ i 1)))))))
210(define (sort-strings xs)
211 (let insert-sort ((rest xs) (out '()))
212 (if (null? rest) out
213 (insert-sort (cdr rest)
214 (let place ((ys out))
215 (cond ((null? ys) (list (car rest)))
216 ((string-before? (car rest) (car ys)) (cons (car rest) ys))
217 (else (cons (car ys) (place (cdr ys))))))))))
219(define (sgd-files dir)
220 (if (not (file-exists? dir)) '()
221 (let loop ((entries (sort-strings (directory-list dir))) (out '()))
222 (if (null? entries) (reverse out)
223 (loop (cdr entries)
224 (if (has-suffix? (car entries) ".sgd")
225 (cons (string-append dir "/" (car entries)) out)
226 out))))))
228;; DERIVED, not inherited. SD-P3's corpus was an explicit list carrying its own
229;; instruction to be re-derived whenever the tree gained manuscripts, and it was
230;; not: eleven of the tree's .sgd files are absent from its baseline. A derived
231;; population cannot go stale that way. The two sides may legitimately hold
232;; different files -- this phase adds four -- so the GATE takes the intersection
233;; by name and accounts for the difference; this script's job is to report
234;; everything it finds, with a verdict per manuscript.
235(define manuscripts
236 (append (sgd-files "book/practical-sigil") (sgd-files "test/fixtures")))
238;; The first top-level `@command`, read as TEXT. This is a second, independent
239;; instrument: the gate uses it to predict which manuscripts must differ
240;; (the ones whose root command this phase changed) without asking the lowering
241;; -- the thing under test -- what it thinks the root was.
242(define (first-root-command source)
243 (let ((n (string-length source)))
244 (let loop ((i 0) (line-start #t))
245 (cond
246 ((>= i n) "NONE")
247 ((char=? (string-ref source i) #\newline) (loop (+ i 1) #t))
248 ((and line-start (char=? (string-ref source i) #\@)
249 (< (+ i 1) n)
250 (char-alphabetic? (string-ref source (+ i 1))))
251 (let scan ((j (+ i 1)))
252 (if (and (< j n)
253 (or (char-alphabetic? (string-ref source j))
254 (char=? (string-ref source j) #\-)))
255 (scan (+ j 1))
256 (substring source (+ i 1) j))))
257 (else (loop (+ i 1) #f))))))
259(define compared 0)
260(define skipped 0)
262(define (dump path)
263 (let* ((source (read-file-string path))
264 (name (basename path))
265 (header (inspect-language-source source name))
266 (id (language-header-id header))
267 (profile (profile-for id)))
268 (set! header-end (language-header-end header))
269 (set! current-manuscript name)
270 ;; The MANIFEST line carries everything that legitimately differs between
271 ;; the two trees -- the language id, the header length, the root command --
272 ;; and is deliberately OUTSIDE the compared section. Putting it inside would
273 ;; make every migrated manuscript differ for the one reason this
274 ;; differential exists to look past.
275 (display "--- MANIFEST ") (display name)
276 (display " lang=") (display id)
277 (display " header=") (display header-end)
278 (display " root=@") (display (first-root-command source))
279 (newline)
280 (if (not profile)
281 (begin
282 (set! skipped (+ skipped 1))
283 (display "--- SKIPPED ") (display name)
284 (display " reason=unknown-language:") (display id) (newline))
285 (let ((outcome
286 (guard (exn (else (cons 'raised (exception-message exn))))
287 (cons 'forms (parse-document-forms-with-profile
288 profile source name header-end)))))
289 (if (eq? (car outcome) 'raised)
290 (begin
291 (set! skipped (+ skipped 1))
292 (display "--- SKIPPED ") (display name)
293 (display " reason=raised:") (display (cdr outcome)) (newline))
294 (begin
295 (set! compared (+ compared 1))
296 (display "=== ") (display name)
297 (display " forms=") (display (length (cdr outcome))) (newline)
298 (for-each (lambda (form) (emit form) (newline)) (cdr outcome))))))))
300(for-each dump manuscripts)
301(display "SD_P8_LOWERING_DUMP_COMPLETE found ")
302(display (length manuscripts))
303(display " compared ")
304(display compared)
305(display " skipped ")
306(display skipped)
307(newline)