AtlatestRepositorysigil-doc
1
;;; SD-P8 -- canonical dump of LOWERED output, normalized so that the `#lang`2
;;; header's LENGTH is invisible and nothing else is.3
;;;4
;;; WHY THIS EXISTS AND WHY IT IS NOT `test/sd-p3-lowering-dump.sgl`.5
;;;6
;;; SD-P3's dump is the AFTER half of a differential whose BEFORE half is a7
;;; committed table of sha256s taken from a pre-SD-P3 tree. D8 migrates the8
;;; `#lang` line of 17 manuscripts, which moves every byte offset in each9
;;; of them by the header's length delta, so every one of those hashes changes.10
;;; The BEFORE half stores hashes and not sections, so no normalization can be11
;;; applied to it retroactively -- the differential cannot be rescued, only12
;;; retired. It is retired in `test/gate-sd-p3.sh` arms 3 and 4, under the13
;;; standing pattern Fable's 2026-08-11 ruling recorded for exactly this case,14
;;; and this file is the replacement that pattern asks for: SD-P8's own15
;;; differential, scoped to SD-P8's own base.16
;;;17
;;; TWO PROPERTIES THIS INSTRUMENT IS BUILT AROUND.18
;;;19
;;; * BOTH SIDES RUN THE SAME FILE. `test/gate-sd-p8.sh` copies this script20
;;; into a detached worktree at the base SHA and runs it there, then runs21
;;; it here. An instrument that differs between the two sides of a22
;;; differential measures itself.23
;;;24
;;; * EVERY LOCATION IS EMITTED RELATIVE TO THE MANUSCRIPT'S HEADER END, so25
;;; `#lang sigil/doc` (16 bytes) and `#lang sigil/doc/book` (21 bytes)26
;;; produce identical offsets for identical prose. That is the ONE27
;;; difference this phase intends, and making it invisible is what leaves28
;;; the comparison sensitive to every difference it does NOT intend.29
;;; Generated locations (`<sigil/doc-generated>`, byte=0) are NOT shifted:30
;;; they are not manuscript positions and subtracting from them would be31
;;; arithmetic on a value that means nothing.32
;;;33
;;; The profile comes from the manuscript's own `#lang` id rather than from the34
;;; three-argument default-profile entry point. In the base tree `sigil/doc`35
;;; resolves to that tree's `sigil-doc-profile` (roots `(chapter)`); here it36
;;; resolves to this tree's (roots `(article)`). Both are the tree's own answer37
;;; to "what does this language admit", which is what makes the same file run38
;;; correctly on both sides. `sigil/doc/book` is constructed inline rather than39
;;; imported from `(sigil doc book reader)`, because that module does not exist40
;;; in the base tree and an import of it would make this script unrunnable41
;;; there -- which would silently reduce the differential to one side.42
;;;43
;;; Nothing is evaluated: lowering is a pure source-to-syntax transformation.44
(import (sigil core) (sigil fs) (sigil language)45
(sigil doc private parser))47
;; ------------------------------------------------------------------ output49
(define header-end 0)51
;; A manuscript location is emitted header-relative; a generated one is not a52
;; manuscript position and is emitted as it stands. The discriminator is the53
;; file name the location carries, so this cannot accidentally shift a location54
;; that has no manuscript behind it.55
(define (emit-location loc depth)56
(if (or (not loc) (> depth 40))57
(display "#f")58
(let* ((file (srcloc-file loc))59
(generated? (string=? file "<sigil/doc-generated>"))60
(position (srcloc-position loc)))61
(display "[") (write file)62
(display " ") (display (srcloc-line loc))63
(display ":") (display (srcloc-column loc))64
(display (if generated? " byte=" " rel="))65
(display (if generated? position (- position header-end)))66
(display " span=") (display (srcloc-span loc))67
(let ((origin (srcloc-origin loc)))68
(if origin69
(begin (display " <-") (emit-location origin (+ depth 1)))70
#f))71
(display "]"))))73
;; Walk the improper-list structure explicitly. `write` alone would not show74
;; syntax wrappers or locations, which are the part most likely to move.75
;;76
;; THE `make-srcloc` SPECIAL CASE IS NOT COSMETIC. The lowered forms carry the77
;; manuscript's byte offsets TWICE: once as the syntax objects' own srclocs,78
;; and once as literal integer DATA inside the `(make-srcloc <file> <line>79
;; <col> <pos> <span>)` calls the parser generates for `author-at`. Normalizing80
;; only the first leaves the second absolute, and every migrated manuscript81
;; then differs by the header delta in a place that looks like content. The82
;; discriminator is structural -- head symbol `make-srcloc`, six elements, and83
;; a file argument that is this manuscript rather than the generated marker --84
;; so no ordinary integer in a manuscript can be caught by it.85
(define (syntax-datum-of value)86
(if (syntax? value) (syntax-datum value) value))88
;; THE GUARD IS THE MANUSCRIPT'S OWN NAME, not merely "not the generated89
;; marker". An earlier version accepted any six-element `make-srcloc` call whose90
;; second element was any string, while its comment claimed a check the code did91
;; not make: an author writing `(make-srcloc "anything.sgd" 1 1 100 5)` inside a92
;; top-level `@(...)` would have had element 4 silently rewritten, hiding a real93
;; change to that literal. Nothing in the corpus does that today -- which is94
;; exactly the condition under which an over-broad predicate goes unnoticed.95
(define current-manuscript "")97
(define (generated-srcloc-call? value)98
(and (pair? value)99
(eq? (syntax-datum-of (car value)) 'make-srcloc)100
(= (length value) 6)101
(let ((file (syntax-datum-of (cadr value))))102
(and (string? file) (string=? file current-manuscript)))))104
(define expression-binding-prefix "#%sigil/doc-expression:")105
(define (expression-binding? name)106
(let ((n (string-length name)) (m (string-length expression-binding-prefix)))107
(and (> n m) (string=? (substring name 0 m) expression-binding-prefix))))109
(define (emit value)110
(cond111
((syntax? value)112
(display "#<")113
(emit-location (syntax-srcloc value) 0)114
(display " ")115
(emit (syntax-datum value))116
(display ">"))117
((null? value) (display "()"))118
((generated-srcloc-call? value)119
;; Same rendering as the general list path, with element 4 -- the byte120
;; position -- rewritten header-relative and RELABELLED, so a reader can121
;; see which number was normalized rather than having to know.122
(display "(")123
(let loop ((rest value) (index 0))124
(cond125
((null? rest) #t)126
(else127
(if (= index 0) #t (display " "))128
(if (= index 4)129
(begin (display "rel:")130
(display (- (syntax-datum-of (car rest)) header-end)))131
(emit (car rest)))132
(loop (cdr rest) (+ index 1)))))133
(display ")"))134
((pair? value)135
(display "(")136
(let loop ((rest value) (first? #t))137
(cond138
((null? rest) #t)139
((pair? rest)140
(if first? #t (display " "))141
(emit (car rest))142
(loop (cdr rest) #f))143
(else (display " . ") (emit rest))))144
(display ")"))145
;; The parser spells a hoisted author expression's private binding146
;; `#%sigil/doc-expression:<absolute byte offset>`147
;; (`private/parser.sgl:595`). That is the THIRD place a manuscript's byte148
;; offsets reach the lowered output -- after the syntax srclocs and the149
;; `make-srcloc` data -- and it is the least obvious, because it wears the150
;; clothes of a name rather than of a position. It is normalized for the151
;; same reason as the other two, and it is worth knowing on its own: a152
;; manuscript's generated binding names are NOT stable under an edit to its153
;; `#lang` line.154
;;155
;; A SUFFIX THAT IS NOT A NUMBER IS REPORTED, NOT SUBTRACTED. `string->number`156
;; returns `#f` on a name this code did not expect, and `(- #f header-end)`157
;; would raise from inside the emitter -- killing the dump and surfacing in158
;; the gate as "the dump printed no completion marker", a red about a missing159
;; marker for what is actually an unrecognised binding name. Emitting the160
;; symbol verbatim instead makes it a DIFFERENCE the differential reports161
;; against the other side, which is the correct place for a surprise.162
((and (symbol? value) (expression-binding? (symbol->string value)))163
(let* ((name (symbol->string value))164
(suffix (substring name165
(string-length expression-binding-prefix)166
(string-length name)))167
(offset (string->number suffix)))168
(if (and offset (integer? offset))169
(begin (display "#%sigil/doc-expression:rel:")170
(display (- offset header-end)))171
(begin (display "#%sigil/doc-expression:UNRECOGNISED-SUFFIX:")172
(display suffix)))))173
(else (write value))))175
;; ----------------------------------------------------------------- profiles177
;; `sigil/doc` is the TREE'S OWN base profile, deliberately: this script is run178
;; in two trees whose base languages differ, and asking each tree what its base179
;; language admits is the only way one file can be correct in both.180
(define book-profile (make-document-profile "sigil/doc/book" '(chapter) 'one))182
(define (profile-for id)183
(cond ((string=? id "sigil/doc") sigil-doc-profile)184
((string=? id "sigil/doc/book") book-profile)185
(else #f)))187
;; ------------------------------------------------------------------ corpus189
(define (basename path)190
(let loop ((i (string-length path)))191
(cond ((= i 0) path)192
((char=? (string-ref path (- i 1)) #\/) (substring path i (string-length path)))193
(else (loop (- i 1))))))195
(define (has-suffix? name suffix)196
(let ((n (string-length name)) (m (string-length suffix)))197
(and (> n m) (string=? (substring name (- n m) n) suffix))))199
;; `directory-list` order is filesystem order and is NOT stable across two200
;; checkouts. Sorting is what makes the two sides comparable line by line201
;; rather than only as sets.202
(define (string-before? a b)203
(let ((na (string-length a)) (nb (string-length b)))204
(let loop ((i 0))205
(cond ((= i na) (> nb na))206
((= i nb) #f)207
((char<? (string-ref a i) (string-ref b i)) #t)208
((char<? (string-ref b i) (string-ref a i)) #f)209
(else (loop (+ i 1)))))))210
(define (sort-strings xs)211
(let insert-sort ((rest xs) (out '()))212
(if (null? rest) out213
(insert-sort (cdr rest)214
(let place ((ys out))215
(cond ((null? ys) (list (car rest)))216
((string-before? (car rest) (car ys)) (cons (car rest) ys))217
(else (cons (car ys) (place (cdr ys))))))))))219
(define (sgd-files dir)220
(if (not (file-exists? dir)) '()221
(let loop ((entries (sort-strings (directory-list dir))) (out '()))222
(if (null? entries) (reverse out)223
(loop (cdr entries)224
(if (has-suffix? (car entries) ".sgd")225
(cons (string-append dir "/" (car entries)) out)226
out))))))228
;; DERIVED, not inherited. SD-P3's corpus was an explicit list carrying its own229
;; instruction to be re-derived whenever the tree gained manuscripts, and it was230
;; not: eleven of the tree's .sgd files are absent from its baseline. A derived231
;; population cannot go stale that way. The two sides may legitimately hold232
;; different files -- this phase adds four -- so the GATE takes the intersection233
;; by name and accounts for the difference; this script's job is to report234
;; everything it finds, with a verdict per manuscript.235
(define manuscripts236
(append (sgd-files "book/practical-sigil") (sgd-files "test/fixtures")))238
;; The first top-level `@command`, read as TEXT. This is a second, independent239
;; instrument: the gate uses it to predict which manuscripts must differ240
;; (the ones whose root command this phase changed) without asking the lowering241
;; -- the thing under test -- what it thinks the root was.242
(define (first-root-command source)243
(let ((n (string-length source)))244
(let loop ((i 0) (line-start #t))245
(cond246
((>= i n) "NONE")247
((char=? (string-ref source i) #\newline) (loop (+ i 1) #t))248
((and line-start (char=? (string-ref source i) #\@)249
(< (+ i 1) n)250
(char-alphabetic? (string-ref source (+ i 1))))251
(let scan ((j (+ i 1)))252
(if (and (< j n)253
(or (char-alphabetic? (string-ref source j))254
(char=? (string-ref source j) #\-)))255
(scan (+ j 1))256
(substring source (+ i 1) j))))257
(else (loop (+ i 1) #f))))))259
(define compared 0)260
(define skipped 0)262
(define (dump path)263
(let* ((source (read-file-string path))264
(name (basename path))265
(header (inspect-language-source source name))266
(id (language-header-id header))267
(profile (profile-for id)))268
(set! header-end (language-header-end header))269
(set! current-manuscript name)270
;; The MANIFEST line carries everything that legitimately differs between271
;; the two trees -- the language id, the header length, the root command --272
;; and is deliberately OUTSIDE the compared section. Putting it inside would273
;; make every migrated manuscript differ for the one reason this274
;; differential exists to look past.275
(display "--- MANIFEST ") (display name)276
(display " lang=") (display id)277
(display " header=") (display header-end)278
(display " root=@") (display (first-root-command source))279
(newline)280
(if (not profile)281
(begin282
(set! skipped (+ skipped 1))283
(display "--- SKIPPED ") (display name)284
(display " reason=unknown-language:") (display id) (newline))285
(let ((outcome286
(guard (exn (else (cons 'raised (exception-message exn))))287
(cons 'forms (parse-document-forms-with-profile288
profile source name header-end)))))289
(if (eq? (car outcome) 'raised)290
(begin291
(set! skipped (+ skipped 1))292
(display "--- SKIPPED ") (display name)293
(display " reason=raised:") (display (cdr outcome)) (newline))294
(begin295
(set! compared (+ compared 1))296
(display "=== ") (display name)297
(display " forms=") (display (length (cdr outcome))) (newline)298
(for-each (lambda (form) (emit form) (newline)) (cdr outcome))))))))300
(for-each dump manuscripts)301
(display "SD_P8_LOWERING_DUMP_COMPLETE found ")302
(display (length manuscripts))303
(display " compared ")304
(display compared)305
(display " skipped ")306
(display skipped)307
(newline)