AtlatestRepositorysigil-doc
1
(define-library (sigil doc private parser)2
(import (sigil core) (sigil struct) (sigil syntax))3
(export parse-document-source parse-document-forms4
parse-document-nodes5
parse-document-source-with-profile parse-document-forms-with-profile6
parse-document-parts-with-profile7
make-document-profile document-profile? document-profile-name8
document-profile-roots document-profile-multiplicity9
sigil-doc-profile10
parser-state? parser-state-source parser-state-source-name parser-state-steps11
parser-node-location parser-node-cursor)12
(begin13
;; Immutable configuration constants. These are not parser state: nothing14
;; assigns them, and two concurrent parses cannot observe each other through15
;; them. Every value that a parse *writes* lives in `parser-state` below.16
(define max-nesting 64)17
(define max-commands 8192)18
;; Bounds the candidate-']' retry in `read-arguments` (finding I5).19
;;20
;; This is a backstop, deliberately set far above any real manuscript: the21
;; candidates are counted over the rest of the FILE, not over the group, so22
;; a low bound rejects legal documents. At 1024 a chapter title containing23
;; 1024 ']' characters stopped parsing, which is a regression rather than a24
;; protection. It also does NOT bound the cost of a single delegated read;25
;; see docs/passive-parse-api.md on the pre-existing reader pathology that26
;; no bound at this layer can reach.27
(define max-argument-candidates 65536)29
;; ---------------------------------------------------------------- profile30
;;31
;; SD-P1 (review finding B2). "Exactly one `@chapter`" was a constant in32
;; `parse-document-parts` below. It is a LOWERING POLICY, not a fact about33
;; the grammar -- the passive path enforces none of it -- and a delegating34
;; language (`#lang folio/doc`, whose roots are `@note`/`@project`) cannot35
;; be built on a parser that hardcodes it. The policy is now a value.36
;;37
;; A profile is immutable: nothing assigns it, its accessors hand back38
;; copies of the aggregates they own, and two concurrent parses cannot39
;; observe each other through one. It is configuration, like the bounds40
;; above -- NOT parse state, all of which lives in `parser-state`.41
(define-struct document-profile-record (name) (roots) (multiplicity))43
(define (profile-proper-list? xs)44
(let loop ((slow xs) (fast xs))45
(cond ((null? fast) #t) ((not (pair? fast)) #f)46
((null? (cdr fast)) #t) ((not (pair? (cdr fast))) #f)47
((eq? slow (cddr fast)) #f)48
(else (loop (cdr slow) (cddr fast))))))49
(define (profile-copy-list xs)50
(let loop ((rest xs) (out '()))51
(if (null? rest) (reverse out) (loop (cdr rest) (cons (car rest) out)))))52
(define (profile-all-symbols? xs)53
(or (null? xs) (and (symbol? (car xs)) (profile-all-symbols? (cdr xs)))))54
(define (profile-has-duplicate? xs)55
(and (pair? xs) (or (memq (car xs) (cdr xs)) (profile-has-duplicate? (cdr xs)))))57
;; `name` prefixes the two ROOT-POLICY diagnostics only, so a delegating58
;; profile reports `folio/doc:` where the policy genuinely differs.59
;; Grammar-level diagnostics (`fail-at`) keep the `sigil/doc:` prefix60
;; deliberately: the grammar is not what a profile varies, and D1/D2 pin61
;; those exact strings.62
(define (make-document-profile name roots multiplicity)63
(cond64
((or (not (string? name)) (= (string-length name) 0))65
(error "make-document-profile: name must be a nonempty string"))66
((not (profile-proper-list? roots))67
(error "make-document-profile: roots must be a proper list"))68
;; A profile admitting no root rejects every document with a message69
;; naming no command. Refuse it where it is constructed, not where it70
;; would silently look like a manuscript error.71
((null? roots)72
(error "make-document-profile: roots must name at least one command"))73
((not (profile-all-symbols? roots))74
(error "make-document-profile: roots must be command-name symbols"))75
((profile-has-duplicate? roots)76
(error "make-document-profile: roots must be distinct"))77
;; Closed vocabulary. `zero-or-more` is deliberately absent: it would78
;; legalise a rootless document and so retire the diagnostic that79
;; catches an empty or truncated manuscript.80
((not (memq multiplicity '(one one-or-more)))81
(error "make-document-profile: multiplicity must be one or one-or-more"))82
(else83
(document-profile-record name: (string-copy name)84
roots: (profile-copy-list roots)85
multiplicity: multiplicity))))86
(define (document-profile? value) (document-profile-record? value))87
(define (document-profile-name profile)88
(string-copy (document-profile-record-name profile)))89
(define (document-profile-roots profile)90
(profile-copy-list (document-profile-record-roots profile)))91
(define (document-profile-multiplicity profile)92
(document-profile-record-multiplicity profile))94
;; `sigil/doc`'s own profile.95
;;96
;; SD-P8 (D8 ruling, 2026-08-22): the BASE language is GENERIC and GENRES97
;; ARE LANGUAGES. `#lang sigil/doc` admits exactly one `@article` -- the98
;; standalone document: README, design doc, memo, spec, blog post -- and99
;; `@chapter`, which is a PART of a book rather than a document anyone100
;; writes standalone, moved to the sibling genre language101
;; `#lang sigil/doc/book` (`src/sigil/doc/book/reader.sgl`).102
;;103
;; Roots come only from the `#lang` profile: the header line is the import104
;; statement for the root vocabulary, readable without evaluating anything,105
;; which is what keeps a document's TYPE statically inspectable.106
(define sigil-doc-profile (make-document-profile "sigil/doc" '(article) 'one))108
;; The two root-policy diagnostics are DERIVED from the profile rather than109
;; written twice, so a profile change reaches its own error messages with no110
;; second copy to forget. SD-P8 is the first change to exercise that: moving111
;; the base root set from `chapter` to `article` reworded both messages with112
;; no edit here. A test pins both literals for the base AND for113
;; `sigil/doc/book`, so the formatter cannot quietly reword a shipped114
;; diagnostic -- and each pinned literal is shown able to fail.115
(define (profile-root-names roots)116
(let ((n (length roots)))117
(cond118
((= n 1) (string-append "@" (symbol->string (car roots))))119
((= n 2) (string-append "@" (symbol->string (car roots))120
" or @" (symbol->string (cadr roots))))121
(else122
(let loop ((xs roots) (first? #t) (out ""))123
(cond124
((null? (cdr xs))125
(string-append out ", or @" (symbol->string (car xs))))126
(first?127
(loop (cdr xs) #f (string-append "@" (symbol->string (car xs)))))128
(else129
(loop (cdr xs) #f (string-append out ", @" (symbol->string (car xs)))))))))))130
(define (profile-root-requirement profile)131
(string-append132
(if (eq? (document-profile-record-multiplicity profile) 'one)133
"exactly one " "at least one ")134
(profile-root-names (document-profile-record-roots profile))))135
(define (profile-missing-root-message profile)136
(string-append (document-profile-record-name profile)137
": source must contain " (profile-root-requirement profile)))138
(define (profile-misplaced-message profile)139
(string-append (document-profile-record-name profile)140
": top-level author forms must precede "141
(profile-root-requirement profile)))143
;; One value per parse, threaded as an argument. Replaces the module-level144
;; `set!` cells this parser used to carry (review finding I3): a nested parse145
;; -- author code loading another manuscript -- or a future concurrent folio146
;; indexer corrupted locations silently through them. `definitions` is the147
;; only mutable field; the location index is immutable after construction.148
(define-struct parser-state149
(source) (source-name)150
(bytes) (lines) (columns) (scalars) (byte-length)151
(steps)152
(definitions default: '() mutable: #t))154
(define (fail-at source-name line column message)155
(error (string-append source-name ":" (number->string line) ":"156
(number->string column) ": sigil/doc: " message)))157
(define (fail-here state index message)158
(let ((cursor (cursor-at state index)))159
(fail-at (parser-state-source-name state) (car cursor) (cadr cursor) message)))160
(define (utf8-width c)161
(let ((n (char->integer c)))162
(cond ((<= n #x7f) 1) ((<= n #x7ff) 2) ((<= n #xffff) 3) (else 4))))163
;; Single deterministic scalar pass. `steps` records its length so callers can164
;; assert the pass is linear without reaching into module state.165
(define (make-parser-state source source-name)166
(let* ((n (string-length source))167
(bytes (make-vector (+ n 1) 0))168
(lines (make-vector (+ n 1) 1))169
(columns (make-vector (+ n 1) 1))170
(scalars (make-vector (+ (* 4 n) 1) #f)))171
;; `visited` COUNTS the scalars this pass actually touches rather than172
;; being set to the manuscript's length afterwards. An earlier version173
;; stored `n` directly, which made the test asserting "one linear pass"174
;; assert `(string-length source)` against itself -- true under a175
;; quadratic index too.176
(let loop ((i 0) (byte 0) (line 1) (column 1) (visited 0))177
(vector-set! bytes i byte)178
(vector-set! lines i line)179
(vector-set! columns i column)180
(vector-set! scalars byte i)181
(if (= i n)182
(parser-state source: source source-name: source-name183
bytes: bytes lines: lines columns: columns184
scalars: scalars byte-length: byte steps: visited)185
(let ((c (string-ref source i)))186
(if (char=? c #\newline)187
(loop (+ i 1) (+ byte (utf8-width c)) (+ line 1) 1 (+ visited 1))188
(loop (+ i 1) (+ byte (utf8-width c)) line (+ column 1) (+ visited 1))))))))189
(define (position-at state end) (vector-ref (parser-state-bytes state) end))190
(define (cursor-at state end)191
(list (vector-ref (parser-state-lines state) end)192
(vector-ref (parser-state-columns state) end)))193
(define (char-at-byte state byte)194
(let ((scalars (parser-state-scalars state))195
(byte-length (parser-state-byte-length state)))196
(if (and (>= byte 0) (<= byte byte-length) (vector-ref scalars byte))197
(vector-ref scalars byte)198
(error "sigil/doc: standard reader returned a non-scalar boundary"))))199
(define (location state start end)200
(let* ((cursor (cursor-at state start))201
(start-byte (position-at state start)))202
(make-srcloc (parser-state-source-name state) (car cursor) (cadr cursor)203
start-byte (- (position-at state end) start-byte))))204
(define (loc-datum state start end)205
(let ((loc (location state start end)))206
(list 'make-srcloc (srcloc-file loc) (srcloc-line loc) (srcloc-column loc)207
(srcloc-position loc) (srcloc-span loc))))208
(define (generated-location state start end)209
(make-generated-srcloc "<sigil/doc-generated>" 1 1 0 1210
(location state start end)))211
;; Generated authoring forms remain ordinary Sigil syntax, but every212
;; constructor, callee, and literal retains the manuscript command/text as213
;; its authoritative origin. Already-located child commands keep their own214
;; narrower origin instead of being collapsed into their parent.215
(define (generated-tree state start end datum)216
(let ((loc (generated-location state start end)))217
(let recur ((value datum))218
(if (syntax? value) value219
(make-syntax220
(if (pair? value) (map recur value) value)221
loc)))))222
(define (generated-call state start end datum)223
(generated-tree state start end datum))224
(define (author-syntax value)225
(cond226
((syntax? value)227
(make-syntax (author-syntax (syntax-datum value)) (syntax-srcloc value)))228
((pair? value) (cons (author-syntax (car value))229
(author-syntax (cdr value))))230
(else value)))231
(define (argument-syntax state stx)232
(let ((values (syntax-datum stx)))233
(if (list? values) (map author-syntax values)234
(error (string-append (parser-state-source-name state)235
": sigil/doc: argument group must be a proper list")))))237
;; Replacing only the opening '[' with '(' preserves every following byte.238
;; The standard reader therefore owns strings, comments, datum comments and239
;; the matching ']' boundary; document scanning never guesses it.240
;;241
;; Finding I5: the manuscript is copied ONCE and the two candidate bytes are242
;; mutated and restored per attempt, instead of copying the whole source per243
;; candidate; the candidate scan is bounded; and the delegated reader's own244
;; diagnostic is retained rather than swallowed, so a genuine Sigil syntax245
;; error inside `[...]` no longer surfaces as "missing ']'" at the opening246
;; bracket. The widest candidate's failure is the reported one: a real syntax247
;; error persists across every candidate, while closing too early only248
;; defeats the narrow ones.249
(define (read-arguments state opening)250
(let* ((source (parser-state-source state))251
(n (string-length source))252
(delegated (string-copy source))253
(source-name (parser-state-source-name state)))254
(string-set! delegated opening #\()255
(let candidate ((closing (+ opening 1)) (tried 0) (delegated-failure #f))256
;; The delegated reader's message is ADDITIONAL information, not a257
;; replacement: it is reported alongside this parser's own message and258
;; location, never instead of them. Replacing them outright made some259
;; diagnostics worse -- a `]` inside a datum comment moved the reported260
;; location from the argument group into the comment -- and it also261
;; leaks the delegation trick, since the reader sees a '(' this parser262
;; substituted for the author's '['. Both hazards are named in the263
;; text rather than hidden.264
(define (give-up reason)265
(fail-here state opening266
(if delegated-failure267
(string-append268
reason269
"; while delimiting this argument group the standard reader reported: "270
(exception-message delegated-failure)271
" (a '(' named there is this parser's substitution for the opening '[')")272
reason)))273
(cond274
((= closing n) (give-up "unexpected EOF; missing ']'"))275
((not (char=? (string-ref source closing) #\]))276
(candidate (+ closing 1) tried delegated-failure))277
((>= tried max-argument-candidates)278
(give-up (string-append "argument group exceeded "279
(number->string max-argument-candidates)280
" candidate ']' boundaries")))281
(else282
(string-set! delegated closing #\))283
(let ((outcome284
(guard (exn (else (list 'failed exn)))285
(let* ((reader (make-syntax-reader delegated source-name286
(position-at state opening)))287
(stx (syntax-reader-read! reader))288
(end (char-at-byte state (syntax-reader-position reader))))289
(if (= end (+ closing 1)) (list 'read stx end) (list 'short))))))290
(string-set! delegated closing #\])291
(cond292
((eq? (car outcome) 'read)293
(list (argument-syntax state (cadr outcome)) (caddr outcome) (cadr outcome)))294
((eq? (car outcome) 'failed)295
(candidate (+ closing 1) (+ tried 1) (cadr outcome)))296
(else (candidate (+ closing 1) (+ tried 1) delegated-failure)))))))))297
(define (read-author-expression state opening)298
(let* ((reader (make-syntax-reader (parser-state-source state)299
(parser-state-source-name state)300
(position-at state opening)))301
(stx (syntax-reader-read! reader))302
(end (char-at-byte state (syntax-reader-position reader))))303
(list (author-syntax stx) end)))305
(define (command-char? c)306
(or (and (char>=? c #\a) (char<=? c #\z)) (char=? c #\-)))307
(define (horizontal-space? c) (or (char=? c #\space) (char=? c #\tab)))308
(define (skip-horizontal source i)309
(let ((n (string-length source)))310
(let loop ((j i))311
(if (and (< j n) (horizontal-space? (string-ref source j))) (loop (+ j 1)) j))))312
(define (block-boundary? source i)313
(let loop ((j (- i 1)))314
(cond ((< j 0) #t)315
((char=? (string-ref source j) #\newline) #t)316
((horizontal-space? (string-ref source j)) (loop (- j 1)))317
(else #f))))318
(define (line-tail source i)319
(let ((n (string-length source)))320
(let loop ((j i))321
(cond ((= j n) (list #t j))322
((char=? (string-ref source j) #\newline) (list #t (+ j 1)))323
((horizontal-space? (string-ref source j)) (loop (+ j 1)))324
(else (list #f i))))))325
;; Return (name end) for a syntactically complete line-level @end marker.326
(define (named-end source i)327
(let ((n (string-length source)))328
(and (block-boundary? source i)329
(<= (+ i 5) n)330
(string=? (substring source i (+ i 5)) "@end ")331
(let name-loop ((j (+ i 5)))332
(if (and (< j n) (command-char? (string-ref source j)))333
(name-loop (+ j 1))334
(and (> j (+ i 5))335
(let ((tail (line-tail source j)))336
(and (car tail) (list (substring source (+ i 5) j) (cadr tail))))))))))337
(define (fold-prose text)338
;; A single line break is prose whitespace; blank lines are retained as339
;; paragraph separators. Spaces adjacent to a folded break collapse.340
(let loop ((cs (string->list text)) (out '()) (pending-space? #f) (newlines 0))341
(cond342
((null? cs)343
(list->string (reverse (if pending-space? (cons #\space out) out))))344
((char=? (car cs) #\newline)345
(loop (cdr cs) out #t (+ newlines 1)))346
((and pending-space? (char-whitespace? (car cs)))347
(loop (cdr cs) out #t newlines))348
(pending-space?349
(if (>= newlines 2)350
(loop cs (cons #\newline (cons #\newline out)) #f 0)351
(loop cs (if (or (null? out) (char-whitespace? (car out))) out (cons #\space out)) #f 0)))352
(else (loop (cdr cs) (cons (car cs) out) #f 0)))))353
(define (trim text)354
(let* ((n (string-length text))355
(start (let loop ((i 0)) (if (and (< i n) (char-whitespace? (string-ref text i))) (loop (+ i 1)) i)))356
(end (let loop ((i n)) (if (and (> i start) (char-whitespace? (string-ref text (- i 1)))) (loop (- i 1)) i))))357
(substring text start end)))359
;; Whitespace immediately inside a command body is delimiter padding, not360
;; prose. Internal whitespace (including spacing around nested inline361
;; commands) remains untouched.362
(define (trim-text-left state node)363
(let ((source (parser-state-source state))364
(end (fifth node)))365
(let loop ((start (cadddr node)))366
(if (and (< start end) (char-whitespace? (string-ref source start)))367
(loop (+ start 1))368
(and (< start end)369
(list 'text '() (fold-prose (substring source start end)) start end))))))370
(define (trim-text-right state node)371
(let ((source (parser-state-source state))372
(start (cadddr node)))373
(let loop ((end (fifth node)))374
(if (and (> end start) (char-whitespace? (string-ref source (- end 1))))375
(loop (- end 1))376
(and (< start end)377
(list 'text '() (fold-prose (substring source start end)) start end))))))378
;; SLOT COUNT, not head symbol -- and this site is the subtle one. It does379
;; not merely MISREAD a six-slot `@text{}` command: `trim-text-left` rebuilds380
;; whatever it is given as a five-slot text pseudo-node carrying the folded381
;; source slice, so a head-symbol test here silently REWRITES an author's382
;; command INTO prose. Everything downstream then behaves correctly on the383
;; wrong node, which is why fixing `lower-block` alone left `@text{}` at384
;; block position unchanged.385
(define (trim-body-edges state children)386
(define (left xs)387
(cond388
((null? xs) '())389
((not (pseudo-kind? (car xs) 'text)) xs)390
(else391
(let ((first (trim-text-left state (car xs))))392
(if first (cons first (cdr xs)) (left (cdr xs)))))))393
(define (right xs)394
(cond395
((null? xs) '())396
((not (pseudo-kind? (car xs) 'text)) xs)397
(else398
(let ((last (trim-text-right state (car xs))))399
(if last (cons last (cdr xs)) (right (cdr xs)))))))400
(reverse (right (reverse (left children)))))402
;; Internal invocation: (name args children start end body-kind); text and403
;; author expressions are five-slot pseudo-nodes carrying no body kind. The404
;; SLOT COUNT, not the head symbol, is what distinguishes them -- `@text{}`405
;; and `@expr{}` are legal author commands whose head symbol collides with406
;; these pseudo-kinds. `(sigil doc inspect)` classifies on slot count for407
;; that reason; SD-P4 left the lowering path on its historical head-symbol408
;; tests, recording the defect for this phase.409
;;410
;; SD-P1 discharges it. Every site below that asked "is the head symbol411
;; `text`/`expr`?" now asks "is this a six-slot COMMAND node?" first, which412
;; is exact. Three consequences, all intended:413
;;414
;; * `@text{x}` and `@expr{x}` are dispatched as the ordinary author415
;; commands they are. A manuscript that defines `expr` gets its own416
;; command; one that does not gets an honest unbound-variable417
;; diagnostic -- instead of a paragraph containing the literal string418
;; `@text{x}`, or a raw internal node list spliced into generated419
;; syntax.420
;; * `@text{}` at top level no longer reaches the empty-prose filter in421
;; `parse-document-parts-with-profile`, where `caddr` of a COMMAND node422
;; is its child list and `trim` was being handed a list, not a string.423
;; * The root allowlist this phase adds is itself a head-symbol test, so424
;; leaving the collision in place would have let an author's `@text{}`425
;; be classified as parser-generated prose at exactly the point the426
;; phase exists to make configurable.427
;;428
;; Measured before/after: `test/sd-p1-classification-probe.sgl` and the two429
;; transcripts in `test/transcripts/sd-p1/`.430
(define (command-node? node) (= (length node) 6))431
(define (pseudo-kind? node kind)432
(and (not (command-node? node)) (eq? (car node) kind)))433
(define (emit-text state text-start end items)434
(if (= text-start end) items435
(let ((value (fold-prose (substring (parser-state-source state) text-start end))))436
(if (= (string-length value) 0) items437
(cons (list 'text '() value text-start end) items)))))438
(define (raw-result state start end resume count)439
(list (if (= start end) '()440
(list (list 'text '() (substring (parser-state-source state) start end) start end)))441
resume count))442
;; Block-position @code is the one reader-level raw form. Its contents are443
;; acquired without interpreting document escapes, commands, or @(...).444
;; Brace bodies balance literal braces; named bodies stop only at the exact445
;; line-level @end code marker; line bodies stop before the physical newline.446
(define (scan-raw-code state start body-kind count)447
(let* ((source (parser-state-source state))448
(n (string-length source)))449
(cond450
((eq? body-kind 'line)451
(let loop ((i start))452
(if (or (= i n) (char=? (string-ref source i) #\newline))453
(raw-result state start i i count)454
(loop (+ i 1)))))455
((eq? body-kind 'brace)456
(let loop ((i start) (braces 0))457
(cond458
((= i n) (fail-here state i "unexpected EOF; missing '}'"))459
((char=? (string-ref source i) #\{) (loop (+ i 1) (+ braces 1)))460
((char=? (string-ref source i) #\})461
(if (= braces 0)462
(raw-result state start i (+ i 1) count)463
(loop (+ i 1) (- braces 1))))464
(else (loop (+ i 1) braces)))))465
(else466
(let loop ((i start))467
(cond468
((= i n) (fail-here state i "unexpected EOF; expected '@end code'"))469
((and (char=? (string-ref source i) #\@)470
(named-end source i)471
(string=? (car (named-end source i)) "code"))472
(raw-result state start i (cadr (named-end source i)) count))473
(else (loop (+ i 1)))))))))474
(define (scan state start terminator depth count)475
(if (> depth max-nesting) (error "sigil/doc: nesting exceeds 64")476
(scan-loop state (string-length (parser-state-source state))477
terminator depth start start '() count)))478
(define (scan-loop state n terminator depth i text-start items count)479
(let ((source (parser-state-source state)))480
(cond481
((= i n)482
(if (eq? terminator 'line)483
(list (reverse (emit-text state text-start i items)) i count)484
(if terminator485
(fail-here state i486
(if (eq? terminator 'brace) "unexpected EOF; missing '}'"487
(string-append "unexpected EOF; expected '@end " terminator "'")))488
(list (reverse (emit-text state text-start i items)) i count))))489
((and (eq? terminator 'line) (char=? (string-ref source i) #\newline))490
(list (reverse (emit-text state text-start i items)) i count))491
((and (eq? terminator 'brace) (char=? (string-ref source i) #\}))492
(list (reverse (emit-text state text-start i items)) (+ i 1) count))493
((and (string? terminator) (named-end source i))494
(let* ((ending (named-end source i)) (actual (car ending)) (end (cadr ending)))495
(if (string=? actual terminator)496
(list (reverse (emit-text state text-start i items)) end count)497
(fail-here state i498
(string-append "mismatched named block terminator; expected '@end " terminator499
"', got '@end " actual "'")))))500
((char=? (string-ref source i) #\@)501
(let ((items (emit-text state text-start i items)))502
(if (= (+ i 1) n)503
(fail-here state i "dangling '@'")504
(let ((next (string-ref source (+ i 1))))505
(cond506
((memv next '(#\@ #\{ #\} #\[ #\]))507
(let ((escaped (string next)))508
(scan-loop state n terminator depth (+ i 2) (+ i 2)509
(cons (list 'text '() escaped i (+ i 2)) items) count)))510
((char=? next #\()511
(let* ((expression (read-author-expression state (+ i 1)))512
(stx (car expression)) (end (cadr expression)))513
(scan-loop state n terminator depth end end514
(cons (list 'expr '() stx i end) items) (+ count 1))))515
((command-char? next)516
(let name-loop ((j (+ i 1)))517
(if (and (< j n) (command-char? (string-ref source j)))518
(name-loop (+ j 1))519
(let* ((name (substring source (+ i 1) j))520
(after-name (skip-horizontal source j))521
(arg-result (if (and (< after-name n) (char=? (string-ref source after-name) #\[))522
(read-arguments state after-name)523
(list '() j #f)))524
(args (car arg-result))525
(after-args (skip-horizontal source (cadr arg-result))))526
(let* ((tail (line-tail source after-args))527
(named? (and (block-boundary? source i) (car tail)))528
(brace? (and (< after-args n) (char=? (string-ref source after-args) #\{)))529
(line? (and (block-boundary? source i) (< after-args n)530
(char=? (string-ref source after-args) #\:))))531
(if (not (or brace? line? named?))532
(fail-here state after-args533
"command requires '{...}', a line body, or a line-level named body")534
(let* ((body-start (cond (brace? (+ after-args 1))535
(line? (skip-horizontal source (+ after-args 1)))536
(else (cadr tail))))537
(body-end (cond (brace? 'brace) (line? 'line) (else name)))538
(child-result539
(if (and (string=? name "code") (block-boundary? source i))540
(scan-raw-code state body-start541
(cond (brace? 'brace) (line? 'line) (else 'named))542
(+ count 1))543
(scan state body-start body-end (+ depth 1) (+ count 1))))544
(children (trim-body-edges state (car child-result)))545
(end (cadr child-result)) (new-count (caddr child-result)))546
(if (> new-count max-commands) (error "sigil/doc: command count exceeds 8192")547
(scan-loop state n terminator depth end end548
(cons (list (string->symbol name) args children i end549
(cond (brace? 'brace)550
(line? 'line)551
(else 'named)))552
items)553
new-count)))))))))554
(else (fail-here state i "invalid '@' escape or command")))))))555
((and (not terminator) (char=? (string-ref source i) #\}))556
(fail-here state i "unexpected '}'"))557
(else (scan-loop state n terminator depth (+ i 1) text-start items count)))))559
(define (inline-list state nodes)560
(apply append (map (lambda (node) (lower-inline state node)) nodes)))561
(define (fifth value) (car (cdr (cdr (cdr (cdr value))))))562
(define (sixth value) (car (cdr (cdr (cdr (cdr (cdr value)))))))563
(define (raw-children state children)564
(let ((source (parser-state-source state)))565
(apply string-append566
(map (lambda (child) (substring source (cadddr child) (fifth child))) children))))567
;; SLOT COUNT again: `caddr` of a five-slot text node is its STRING, but of568
;; a six-slot command it is the child LIST, so a head-symbol test here hands569
;; `string-append` a list. `@code{@text{x}}` now takes the else branch and570
;; gets this procedure's own diagnostic instead of a type error from inside571
;; `string-append`.572
(define (inline-code-content state start children)573
(let loop ((xs children) (out '()))574
(cond575
((null? xs) (apply string-append (reverse out)))576
((pseudo-kind? (car xs) 'text) (loop (cdr xs) (cons (caddr (car xs)) out)))577
(else578
(fail-here state start579
"inline @code content must be text and document escapes")))))580
(define (author-expression state start end expression name)581
(generated-call state start end582
(list 'author-at (loc-datum state start end)583
(list 'lambda '()584
(list 'author-normalize expression585
(loc-datum state start end)586
(list 'quote name))))))587
(define (command-expression state node lowered-children)588
(let ((kind (car node)) (args (cadr node)) (start (cadddr node)) (end (fifth node)))589
(author-expression state start end590
(cons kind (append args lowered-children)) kind)))591
(define (hoisted-author-expression state start end stx)592
;; Deterministic, reader-private binding spelling: stable compiled output593
;; without occupying an ordinary author identifier such as `expression-1`.594
(let ((name (string->symbol595
(string-append "#%sigil/doc-expression:" (number->string start)))))596
(set-parser-state-definitions! state597
(cons (generated-call state start end598
(list 'define name599
(list 'lambda '()600
(list 'author-normalize stx601
(loc-datum state start end)602
(list 'quote 'expression)))))603
(parser-state-definitions state)))604
(list name)))605
(define (inline-author-expression state start end stx)606
(generated-call state start end607
(hoisted-author-expression state start end stx)))608
(define (code-expression state node)609
(let ((args (cadr node)) (children (caddr node))610
(start (cadddr node)) (end (fifth node)) (body-kind (sixth node)))611
(cond612
((and (null? args) (eq? body-kind 'brace))613
(author-expression state start end614
(list 'code (trim (inline-code-content state start children))) 'code))615
((or (null? args) (= (length args) 1))616
(author-expression state start end617
(list 'code (if (null? args) #f (car args))618
(trim (raw-children state children)))619
'code))620
(else (fail-here state start "invalid arguments for @code")))))621
;; `paragraph-ranges` used to live here. SD-P3 splits its two jobs: the622
;; blank-line scan is `paragraph-segments`, and the value/location shaping is623
;; `lower-prose-run`, which reproduces this procedure's exact output for a624
;; prose-only run.625
(define (lower-inline state node)626
(let ((kind (car node)) (args (cadr node)) (children (caddr node)) (start (cadddr node)) (end (fifth node)))627
(cond628
((pseudo-kind? node 'text)629
(let ((value (caddr node)))630
(if (= (string-length value) 0) '()631
(list (generated-call state start end (list 'make-text value (loc-datum state start end)))))))632
((pseudo-kind? node 'expr)633
(list (inline-author-expression state start end children)))634
((eq? kind 'code)635
(list (code-expression state node)))636
(else (list (command-expression state node637
(inline-list state children)))))))638
;; `lower-block` now lowers only nodes that stand at BLOCK POSITION. Prose639
;; and everything folded into a prose run is `block-list`'s business below,640
;; so the historical `text` branch has moved rather than disappeared.641
(define (lower-block state node)642
(let ((kind (car node)) (children (caddr node)) (start (cadddr node)) (end (fifth node)))643
(cond644
((pseudo-kind? node 'expr)645
(list (inline-author-expression state start end children)))646
((eq? kind 'p)647
(list (command-expression state node648
(inline-list state children))))649
((eq? kind 'code)650
(list (code-expression state node)))651
(else (list (command-expression state node652
(block-list state children)))))))654
;; ---------------------------------------------------- SD-P3: prose runs655
;;656
;; Review finding B4. Before this phase, a text run at block position was657
;; split into paragraphs and ANY command interleaved with it was lifted out658
;; as a sibling BLOCK child -- so `Prose with @code{x} in it.` produced a659
;; paragraph, a stray inline code node, and another paragraph, and the660
;; container rejected the code node with "child belongs to the wrong661
;; semantic category". Every marked-up paragraph therefore had to be662
;; `@p{...}`-wrapped.663
;;664
;; The rule, which is deliberately narrow:665
;;666
;; * a command (or `@(...)`) that is NOT at a block boundary folds into667
;; the enclosing paragraph run;668
;; * anything AT a block boundary keeps exactly its previous semantics;669
;; * blank lines remain the paragraph separator and close the run.670
;;671
;; The boundary case is load-bearing, not timidity: `Intro.` / `@note{...}`672
;; / `More.` on three consecutive lines with no blank line between them is a673
;; real, working pattern, and folding a line-initial command would turn it674
;; into a paragraph containing an admonition. The cost is that a command675
;; that happens to land at the start of a WRAPPED line is still treated as a676
;; block; that residue is recorded in `docs/prose-runs.md` and is a design677
;; question (when is a line break formatting, and when is it a separator?)678
;; rather than something to settle inside a parser.679
;;680
;; What this code does NOT decide is whether a folded run is a paragraph.681
;; It cannot: the parser has no command vocabulary, so block-versus-inline682
;; is only knowable after the command runs. A run that is entirely prose is683
;; emitted as today's paragraph, byte for byte; every other run is handed to684
;; `author-paragraph-run`, which decides once the values exist. See the685
;; commentary there.686
(define (block-position? state node)687
(block-boundary? (parser-state-source state) (cadddr node)))689
;; The same blank-line scan `paragraph-ranges` performs, but returning the690
;; RANGES only and keeping the empty ones: an empty segment still closes a691
;; run even though it contributes no text.692
(define (paragraph-segments state start end)693
(let ((source (parser-state-source state)))694
(let loop ((i start) (part start) (out '()))695
(cond696
((= i end) (reverse (cons (list part end) out)))697
((and (char=? (string-ref source i) #\newline)698
(< (+ i 1) end) (char=? (string-ref source (+ i 1)) #\newline))699
(loop (+ i 2) (+ i 2) (cons (list part i) out)))700
(else (loop (+ i 1) part out))))))702
(define (trimmed-start state start end)703
(let ((source (parser-state-source state)))704
(let loop ((i start))705
(if (and (< i end) (char-whitespace? (string-ref source i))) (loop (+ i 1)) i))))706
(define (trimmed-end state start end)707
(let ((source (parser-state-source state)))708
(let loop ((i end))709
(if (and (> i start) (char-whitespace? (string-ref source (- i 1)))) (loop (- i 1)) i))))711
;; Run items: (text <node> start end) for a prose segment, (node <node>)712
;; for a folded command or author expression. A prose item carries its713
;; originating pseudo-node, not just a range -- see `segment-value`.714
(define (text-item? item) (eq? (car item) 'text))715
(define (item-node item) (cadr item))717
;; A TEXT PSEUDO-NODE'S VALUE IS NOT ALWAYS ITS SOURCE SLICE, and rebuilding718
;; one from the slice is a real defect rather than a shortcut. `scan-loop`719
;; emits a document escape as a text node whose VALUE is the single720
;; unescaped character while its SPAN covers the two source characters:721
;; `@@` -> `@`, `@{` -> `{`, and so on. Re-deriving from the slice turns722
;; `@@` back into two at-signs.723
;;724
;; `lower-inline` -- the path `@p{...}` takes -- reads `caddr`. So bare725
;; prose must read `caddr` too, or the two spellings disagree on exactly the726
;; documents that use an escape outside `@p{}`. Found by adversarial review;727
;; no manuscript in the 74-document corpus writes one, so nothing else here728
;; could have caught it.729
;;730
;; A segment that covers its node EXACTLY uses the node's own value. A731
;; segment that is a proper part of its node came from a blank-line split,732
;; so that node contains a newline and is ordinary prose rather than an733
;; escape; re-deriving from the slice is correct there and is what the734
;; pre-SD-P3 lowering did.735
;;736
;; For ordinary prose the two are the same string by construction:737
;; `emit-text` and `trim-body-edges` both build a text node's value as738
;; `(fold-prose (substring source start end))` over exactly that range.739
(define (segment-value state node start end)740
(if (and (= start (cadddr node)) (= end (fifth node)))741
(caddr node)742
(fold-prose (substring (parser-state-source state) start end))))744
;; A run made only of prose is emitted in EXACTLY the shape this parser has745
;; always emitted for a paragraph -- same constructors, same untrimmed746
;; segment range for both the paragraph and its text, same trimmed folded747
;; value. That is what makes the default lowering byte-identical for every748
;; manuscript containing no non-boundary command.749
(define (lower-prose-run state node start end)750
(let ((value (trim (segment-value state node start end))))751
(if (= (string-length value) 0) '()752
(list (generated-call state start end753
(list 'make-paragraph754
(list 'list (list 'make-text value (loc-datum state start end)))755
(loc-datum state start end)))))))757
;; A mixed run. Each prose segment is trimmed on the side that faces the758
;; run's edge and left alone on the side that faces a command, so the folded759
;; text carries exactly the spacing the `@p{...}` spelling carries; its760
;; location is the trimmed range, which is what `trim-body-edges` gives the761
;; equivalent children inside `@p{...}`.762
;; `item-start`/`item-end` must agree with the range `lower-run-item`763
;; actually used, so all three derive it from one place.764
(define (item-start state item first?)765
(if (text-item? item)766
(if first? (trimmed-start state (caddr item) (cadddr item)) (caddr item))767
(cadddr (item-node item))))768
(define (item-end state item first? last?)769
(if (text-item? item)770
(if last? (trimmed-end state (item-start state item first?) (cadddr item))771
(cadddr item))772
(fifth (item-node item))))773
(define (lower-run-item state item first? last?)774
(if (text-item? item)775
(let ((start (item-start state item first?))776
(end (item-end state item first? last?)))777
(lower-inline state778
(list 'text '() (segment-value state (item-node item) start end)779
start end)))780
(lower-inline state (item-node item))))782
(define (lower-mixed-run state items)783
;; Drop the prose segments that trim away to nothing -- a run may open or784
;; close on whitespace -- BEFORE deciding the run's span, so the span785
;; slices to content rather than to layout.786
(let* ((count (length items))787
(kept (let loop ((xs items) (i 0) (out '()))788
(if (null? xs) (reverse out)789
(let* ((first? (= i 0)) (last? (= i (- count 1)))790
(lowered (lower-run-item state (car xs) first? last?)))791
(loop (cdr xs) (+ i 1)792
(if (null? lowered) out793
(cons (list (item-start state (car xs) first?)794
(item-end state (car xs) first? last?)795
lowered)796
out))))))))797
(if (null? kept) '()798
(let ((start (car (car kept)))799
(end (cadr (car (reverse kept)))))800
(list (generated-call state start end801
(list 'author-paragraph-run802
(cons 'list (apply append (map caddr kept)))803
(loc-datum state start end))))))))805
(define (close-run state run out)806
(if (null? run) out807
(let ((items (reverse run)))808
(append (reverse (if (and (null? (cdr items)) (text-item? (car items)))809
(lower-prose-run state (item-node (car items))810
(caddr (car items)) (cadddr (car items)))811
(lower-mixed-run state items)))812
out))))814
(define (block-list state nodes)815
;; `out` and `run` are both reversed; `out` is reversed once at the end.816
(let loop ((xs nodes) (run '()) (out '()))817
(if (null? xs) (reverse (close-run state run out))818
(let ((node (car xs)))819
(cond820
((pseudo-kind? node 'text)821
;; The first segment continues the current run; every later822
;; segment follows a blank line, so it starts a new one.823
;;824
;; Two text nodes CAN be adjacent -- `scan-loop` flushes the825
;; prose before a document escape and emits the escape as its826
;; own text node -- so a run may legitimately hold several827
;; prose items with no command between them.828
(let seg ((parts (paragraph-segments state (cadddr node) (fifth node)))829
(run run) (out out))830
(let ((item (list 'text node (car (car parts)) (cadr (car parts)))))831
(if (null? (cdr parts))832
(loop (cdr xs) (cons item run) out)833
(seg (cdr parts) '() (close-run state (cons item run) out))))))834
((block-position? state node)835
;; SEQUENCED DELIBERATELY. Both `close-run` and `lower-block`836
;; can hoist an author expression into `parser-state-definitions`837
;; via `hoisted-author-expression`, so leaving them as two838
;; arguments to `append` would make the emitted definition order839
;; depend on the host's argument-evaluation order -- and, under840
;; left to right, REVERSE it relative to the manuscript, because841
;; the later block would be lowered before the earlier run was842
;; closed. This is the same hazard that made843
;; `parse-document-parts-with-profile` use an explicit fold844
;; instead of `map`.845
(let* ((closed (close-run state run out))846
(lowered (lower-block state node)))847
(loop (cdr xs) '() (append (reverse lowered) closed))))848
(else849
(loop (cdr xs) (cons (list 'node node) run) out)))))))851
;; Passive scan (finding I4). Returns the raw internal node tree and the852
;; parser state that owns its location index. It performs NO lowering: no853
;; `author-at`, no `author-normalize`, no generated syntax, no evaluation,854
;; and no root-container policy -- the root container is a lowering rule,855
;; enforced by `parse-document-parts-with-profile` below against a profile,856
;; and is not a fact about the grammar.857
;; `(sigil doc inspect)` is the documented surface over this.858
(define (parse-document-nodes source source-name header-end)859
(let* ((state (make-parser-state source source-name))860
(header-cursor (char-at-byte state header-end))861
(result (scan state header-cursor #f 0 0)))862
(list state (car result))))863
(define (parser-node-location state start end) (location state start end))864
(define (parser-node-cursor state index) (cursor-at state index))866
;; The ROOT POLICY, and the only place in this parser that has one.867
;;868
;; Returns `(author-forms roots)`, where `roots` is a list of lowered root869
;; syntaxes in manuscript order -- one element under `'one`, one or more870
;; under `'one-or-more`. The shape is a list for BOTH multiplicities so a871
;; delegating reader never has to discriminate on it.872
;;873
;; The policy is exactly three rules, and each is now read from the profile874
;; instead of written into this loop:875
;; 1. a top-level author expression is hoisted as a form;876
;; 2. a top-level command whose name is in the profile's root set becomes877
;; a root -- and NOTHING else may appear at top level;878
;; 3. every author form must precede every root, and the number of roots879
;; must satisfy the profile's multiplicity.880
;; Rules 1 and 2 discriminate on SLOT COUNT (see `command-node?` above), so881
;; a manuscript's `@text{}`/`@expr{}` command is never mistaken for one of882
;; the parser's own pseudo-nodes.883
;;884
;; TWO PASSES, and the split is load-bearing rather than tidiness. The885
;; pre-SD-P1 loop decided "is this the last node?" BEFORE lowering the886
;; chapter, so a policy error always preceded any lowering error. A887
;; single-pass rewrite that lowers each root as it meets it quietly inverts888
;; that for a doubly-malformed manuscript -- one with, say, a bad `@code`889
;; inside the root AND trailing top-level content would start reporting the890
;; `@code` error instead of the root-policy error. Deciding the entire891
;; policy first preserves the old precedence under every profile, and means892
;; a rejected manuscript is never partially lowered -- which matters to the893
;; folio indexer, whose normal case is scanning files it will reject.894
(define (parse-document-parts-with-profile profile source source-name header-end)895
(if (not (document-profile? profile))896
(error "sigil/doc: parse requires a document profile")897
(let* ((state (make-parser-state source source-name))898
(header-cursor (char-at-byte state header-end))899
(result (scan state header-cursor #f 0 0))900
(roots-allowed (document-profile-record-roots profile))901
(several? (eq? (document-profile-record-multiplicity profile) 'one-or-more))902
(nodes (filter (lambda (node)903
(not (and (pseudo-kind? node 'text)904
(= (string-length (trim (caddr node))) 0))))905
(car result))))906
;; PASS 1 -- decide the policy. Nothing is lowered here.907
(let classify ((xs nodes) (expressions '()) (root-nodes '()))908
(cond909
((null? xs)910
(if (null? root-nodes)911
(error (profile-missing-root-message profile))912
;; PASS 2 -- lower, strictly left to right. An explicit913
;; fold rather than `map`: hoisted author expressions are914
;; accumulated into the parse state AS a root's body is915
;; lowered, so an unspecified application order would make916
;; the emitted definition order depend on the host.917
(let lower ((pending (reverse root-nodes)) (roots '()))918
(if (pair? pending)919
(lower (cdr pending)920
(cons (command-expression state (car pending)921
(block-list state (caddr (car pending))))922
roots))923
;; The returned syntax owns every location/form it924
;; needs. The parser state -- manuscript, location925
;; index, accumulator -- is unreachable once this926
;; call returns, so no scratch state can retain the927
;; last manuscript or leak into a nested parse.928
(list (append (reverse expressions)929
(reverse (parser-state-definitions state)))930
(reverse roots))))))931
;; An author form after a root is the misplacement this parser932
;; has always rejected, and it stays rejected under every933
;; profile.934
((pseudo-kind? (car xs) 'expr)935
(if (pair? root-nodes)936
(error (profile-misplaced-message profile))937
(classify (cdr xs) (cons (caddr (car xs)) expressions) root-nodes)))938
((and (command-node? (car xs))939
(memq (car (car xs)) roots-allowed)940
(or several? (null? root-nodes)))941
(classify (cdr xs) expressions (cons (car xs) root-nodes)))942
(else (error (profile-misplaced-message profile))))))))944
;; `parse-document-source` and `parse-document-forms` keep their exact945
;; three-argument spellings and delegate with `sigil-doc-profile`. That is946
;; not politeness: `test/test-reader.sgl` calls them 26 times, and SD-P1's947
;; acceptance requires the D1/D2 suites green UNCHANGED.948
(define (profile-single-root profile roots)949
(if (null? (cdr roots)) (car roots)950
(error (string-append (document-profile-record-name profile)951
": this profile admits several roots; use"952
" parse-document-parts-with-profile"))))953
(define (parse-document-source-with-profile profile source source-name header-end)954
(let ((parts (parse-document-parts-with-profile profile source source-name header-end)))955
(if (null? (car parts)) (profile-single-root profile (cadr parts))956
(error (string-append (document-profile-record-name profile)957
": top-level author forms require parse-document-forms")))))958
(define (parse-document-forms-with-profile profile source source-name header-end)959
(let ((parts (parse-document-parts-with-profile profile source source-name header-end)))960
(append (car parts) (list (profile-single-root profile (cadr parts))))))961
(define (parse-document-source source source-name header-end)962
(parse-document-source-with-profile sigil-doc-profile source source-name header-end))963
(define (parse-document-forms source source-name header-end)964
(parse-document-forms-with-profile sigil-doc-profile source source-name header-end))))