AtlatestRepositorysigil-doc

sigil-doc / tree / src / sigil / doc / privateparser.sgl

1(define-library (sigil doc private parser)
2 (import (sigil core) (sigil struct) (sigil syntax))
3 (export parse-document-source parse-document-forms
4 parse-document-nodes
5 parse-document-source-with-profile parse-document-forms-with-profile
6 parse-document-parts-with-profile
7 make-document-profile document-profile? document-profile-name
8 document-profile-roots document-profile-multiplicity
9 sigil-doc-profile
10 parser-state? parser-state-source parser-state-source-name parser-state-steps
11 parser-node-location parser-node-cursor)
12 (begin
13 ;; Immutable configuration constants. These are not parser state: nothing
14 ;; assigns them, and two concurrent parses cannot observe each other through
15 ;; them. Every value that a parse *writes* lives in `parser-state` below.
16 (define max-nesting 64)
17 (define max-commands 8192)
18 ;; Bounds the candidate-']' retry in `read-arguments` (finding I5).
19 ;;
20 ;; This is a backstop, deliberately set far above any real manuscript: the
21 ;; candidates are counted over the rest of the FILE, not over the group, so
22 ;; a low bound rejects legal documents. At 1024 a chapter title containing
23 ;; 1024 ']' characters stopped parsing, which is a regression rather than a
24 ;; protection. It also does NOT bound the cost of a single delegated read;
25 ;; see docs/passive-parse-api.md on the pre-existing reader pathology that
26 ;; no bound at this layer can reach.
27 (define max-argument-candidates 65536)
29 ;; ---------------------------------------------------------------- profile
30 ;;
31 ;; SD-P1 (review finding B2). "Exactly one `@chapter`" was a constant in
32 ;; `parse-document-parts` below. It is a LOWERING POLICY, not a fact about
33 ;; the grammar -- the passive path enforces none of it -- and a delegating
34 ;; language (`#lang folio/doc`, whose roots are `@note`/`@project`) cannot
35 ;; be built on a parser that hardcodes it. The policy is now a value.
36 ;;
37 ;; A profile is immutable: nothing assigns it, its accessors hand back
38 ;; copies of the aggregates they own, and two concurrent parses cannot
39 ;; observe each other through one. It is configuration, like the bounds
40 ;; above -- NOT parse state, all of which lives in `parser-state`.
41 (define-struct document-profile-record (name) (roots) (multiplicity))
43 (define (profile-proper-list? xs)
44 (let loop ((slow xs) (fast xs))
45 (cond ((null? fast) #t) ((not (pair? fast)) #f)
46 ((null? (cdr fast)) #t) ((not (pair? (cdr fast))) #f)
47 ((eq? slow (cddr fast)) #f)
48 (else (loop (cdr slow) (cddr fast))))))
49 (define (profile-copy-list xs)
50 (let loop ((rest xs) (out '()))
51 (if (null? rest) (reverse out) (loop (cdr rest) (cons (car rest) out)))))
52 (define (profile-all-symbols? xs)
53 (or (null? xs) (and (symbol? (car xs)) (profile-all-symbols? (cdr xs)))))
54 (define (profile-has-duplicate? xs)
55 (and (pair? xs) (or (memq (car xs) (cdr xs)) (profile-has-duplicate? (cdr xs)))))
57 ;; `name` prefixes the two ROOT-POLICY diagnostics only, so a delegating
58 ;; profile reports `folio/doc:` where the policy genuinely differs.
59 ;; Grammar-level diagnostics (`fail-at`) keep the `sigil/doc:` prefix
60 ;; deliberately: the grammar is not what a profile varies, and D1/D2 pin
61 ;; those exact strings.
62 (define (make-document-profile name roots multiplicity)
63 (cond
64 ((or (not (string? name)) (= (string-length name) 0))
65 (error "make-document-profile: name must be a nonempty string"))
66 ((not (profile-proper-list? roots))
67 (error "make-document-profile: roots must be a proper list"))
68 ;; A profile admitting no root rejects every document with a message
69 ;; naming no command. Refuse it where it is constructed, not where it
70 ;; would silently look like a manuscript error.
71 ((null? roots)
72 (error "make-document-profile: roots must name at least one command"))
73 ((not (profile-all-symbols? roots))
74 (error "make-document-profile: roots must be command-name symbols"))
75 ((profile-has-duplicate? roots)
76 (error "make-document-profile: roots must be distinct"))
77 ;; Closed vocabulary. `zero-or-more` is deliberately absent: it would
78 ;; legalise a rootless document and so retire the diagnostic that
79 ;; catches an empty or truncated manuscript.
80 ((not (memq multiplicity '(one one-or-more)))
81 (error "make-document-profile: multiplicity must be one or one-or-more"))
82 (else
83 (document-profile-record name: (string-copy name)
84 roots: (profile-copy-list roots)
85 multiplicity: multiplicity))))
86 (define (document-profile? value) (document-profile-record? value))
87 (define (document-profile-name profile)
88 (string-copy (document-profile-record-name profile)))
89 (define (document-profile-roots profile)
90 (profile-copy-list (document-profile-record-roots profile)))
91 (define (document-profile-multiplicity profile)
92 (document-profile-record-multiplicity profile))
94 ;; `sigil/doc`'s own profile.
95 ;;
96 ;; SD-P8 (D8 ruling, 2026-08-22): the BASE language is GENERIC and GENRES
97 ;; ARE LANGUAGES. `#lang sigil/doc` admits exactly one `@article` -- the
98 ;; standalone document: README, design doc, memo, spec, blog post -- and
99 ;; `@chapter`, which is a PART of a book rather than a document anyone
100 ;; writes standalone, moved to the sibling genre language
101 ;; `#lang sigil/doc/book` (`src/sigil/doc/book/reader.sgl`).
102 ;;
103 ;; Roots come only from the `#lang` profile: the header line is the import
104 ;; statement for the root vocabulary, readable without evaluating anything,
105 ;; which is what keeps a document's TYPE statically inspectable.
106 (define sigil-doc-profile (make-document-profile "sigil/doc" '(article) 'one))
108 ;; The two root-policy diagnostics are DERIVED from the profile rather than
109 ;; written twice, so a profile change reaches its own error messages with no
110 ;; second copy to forget. SD-P8 is the first change to exercise that: moving
111 ;; the base root set from `chapter` to `article` reworded both messages with
112 ;; no edit here. A test pins both literals for the base AND for
113 ;; `sigil/doc/book`, so the formatter cannot quietly reword a shipped
114 ;; diagnostic -- and each pinned literal is shown able to fail.
115 (define (profile-root-names roots)
116 (let ((n (length roots)))
117 (cond
118 ((= n 1) (string-append "@" (symbol->string (car roots))))
119 ((= n 2) (string-append "@" (symbol->string (car roots))
120 " or @" (symbol->string (cadr roots))))
121 (else
122 (let loop ((xs roots) (first? #t) (out ""))
123 (cond
124 ((null? (cdr xs))
125 (string-append out ", or @" (symbol->string (car xs))))
126 (first?
127 (loop (cdr xs) #f (string-append "@" (symbol->string (car xs)))))
128 (else
129 (loop (cdr xs) #f (string-append out ", @" (symbol->string (car xs)))))))))))
130 (define (profile-root-requirement profile)
131 (string-append
132 (if (eq? (document-profile-record-multiplicity profile) 'one)
133 "exactly one " "at least one ")
134 (profile-root-names (document-profile-record-roots profile))))
135 (define (profile-missing-root-message profile)
136 (string-append (document-profile-record-name profile)
137 ": source must contain " (profile-root-requirement profile)))
138 (define (profile-misplaced-message profile)
139 (string-append (document-profile-record-name profile)
140 ": top-level author forms must precede "
141 (profile-root-requirement profile)))
143 ;; One value per parse, threaded as an argument. Replaces the module-level
144 ;; `set!` cells this parser used to carry (review finding I3): a nested parse
145 ;; -- author code loading another manuscript -- or a future concurrent folio
146 ;; indexer corrupted locations silently through them. `definitions` is the
147 ;; only mutable field; the location index is immutable after construction.
148 (define-struct parser-state
149 (source) (source-name)
150 (bytes) (lines) (columns) (scalars) (byte-length)
151 (steps)
152 (definitions default: '() mutable: #t))
154 (define (fail-at source-name line column message)
155 (error (string-append source-name ":" (number->string line) ":"
156 (number->string column) ": sigil/doc: " message)))
157 (define (fail-here state index message)
158 (let ((cursor (cursor-at state index)))
159 (fail-at (parser-state-source-name state) (car cursor) (cadr cursor) message)))
160 (define (utf8-width c)
161 (let ((n (char->integer c)))
162 (cond ((<= n #x7f) 1) ((<= n #x7ff) 2) ((<= n #xffff) 3) (else 4))))
163 ;; Single deterministic scalar pass. `steps` records its length so callers can
164 ;; assert the pass is linear without reaching into module state.
165 (define (make-parser-state source source-name)
166 (let* ((n (string-length source))
167 (bytes (make-vector (+ n 1) 0))
168 (lines (make-vector (+ n 1) 1))
169 (columns (make-vector (+ n 1) 1))
170 (scalars (make-vector (+ (* 4 n) 1) #f)))
171 ;; `visited` COUNTS the scalars this pass actually touches rather than
172 ;; being set to the manuscript's length afterwards. An earlier version
173 ;; stored `n` directly, which made the test asserting "one linear pass"
174 ;; assert `(string-length source)` against itself -- true under a
175 ;; quadratic index too.
176 (let loop ((i 0) (byte 0) (line 1) (column 1) (visited 0))
177 (vector-set! bytes i byte)
178 (vector-set! lines i line)
179 (vector-set! columns i column)
180 (vector-set! scalars byte i)
181 (if (= i n)
182 (parser-state source: source source-name: source-name
183 bytes: bytes lines: lines columns: columns
184 scalars: scalars byte-length: byte steps: visited)
185 (let ((c (string-ref source i)))
186 (if (char=? c #\newline)
187 (loop (+ i 1) (+ byte (utf8-width c)) (+ line 1) 1 (+ visited 1))
188 (loop (+ i 1) (+ byte (utf8-width c)) line (+ column 1) (+ visited 1))))))))
189 (define (position-at state end) (vector-ref (parser-state-bytes state) end))
190 (define (cursor-at state end)
191 (list (vector-ref (parser-state-lines state) end)
192 (vector-ref (parser-state-columns state) end)))
193 (define (char-at-byte state byte)
194 (let ((scalars (parser-state-scalars state))
195 (byte-length (parser-state-byte-length state)))
196 (if (and (>= byte 0) (<= byte byte-length) (vector-ref scalars byte))
197 (vector-ref scalars byte)
198 (error "sigil/doc: standard reader returned a non-scalar boundary"))))
199 (define (location state start end)
200 (let* ((cursor (cursor-at state start))
201 (start-byte (position-at state start)))
202 (make-srcloc (parser-state-source-name state) (car cursor) (cadr cursor)
203 start-byte (- (position-at state end) start-byte))))
204 (define (loc-datum state start end)
205 (let ((loc (location state start end)))
206 (list 'make-srcloc (srcloc-file loc) (srcloc-line loc) (srcloc-column loc)
207 (srcloc-position loc) (srcloc-span loc))))
208 (define (generated-location state start end)
209 (make-generated-srcloc "<sigil/doc-generated>" 1 1 0 1
210 (location state start end)))
211 ;; Generated authoring forms remain ordinary Sigil syntax, but every
212 ;; constructor, callee, and literal retains the manuscript command/text as
213 ;; its authoritative origin. Already-located child commands keep their own
214 ;; narrower origin instead of being collapsed into their parent.
215 (define (generated-tree state start end datum)
216 (let ((loc (generated-location state start end)))
217 (let recur ((value datum))
218 (if (syntax? value) value
219 (make-syntax
220 (if (pair? value) (map recur value) value)
221 loc)))))
222 (define (generated-call state start end datum)
223 (generated-tree state start end datum))
224 (define (author-syntax value)
225 (cond
226 ((syntax? value)
227 (make-syntax (author-syntax (syntax-datum value)) (syntax-srcloc value)))
228 ((pair? value) (cons (author-syntax (car value))
229 (author-syntax (cdr value))))
230 (else value)))
231 (define (argument-syntax state stx)
232 (let ((values (syntax-datum stx)))
233 (if (list? values) (map author-syntax values)
234 (error (string-append (parser-state-source-name state)
235 ": sigil/doc: argument group must be a proper list")))))
237 ;; Replacing only the opening '[' with '(' preserves every following byte.
238 ;; The standard reader therefore owns strings, comments, datum comments and
239 ;; the matching ']' boundary; document scanning never guesses it.
240 ;;
241 ;; Finding I5: the manuscript is copied ONCE and the two candidate bytes are
242 ;; mutated and restored per attempt, instead of copying the whole source per
243 ;; candidate; the candidate scan is bounded; and the delegated reader's own
244 ;; diagnostic is retained rather than swallowed, so a genuine Sigil syntax
245 ;; error inside `[...]` no longer surfaces as "missing ']'" at the opening
246 ;; bracket. The widest candidate's failure is the reported one: a real syntax
247 ;; error persists across every candidate, while closing too early only
248 ;; defeats the narrow ones.
249 (define (read-arguments state opening)
250 (let* ((source (parser-state-source state))
251 (n (string-length source))
252 (delegated (string-copy source))
253 (source-name (parser-state-source-name state)))
254 (string-set! delegated opening #\()
255 (let candidate ((closing (+ opening 1)) (tried 0) (delegated-failure #f))
256 ;; The delegated reader's message is ADDITIONAL information, not a
257 ;; replacement: it is reported alongside this parser's own message and
258 ;; location, never instead of them. Replacing them outright made some
259 ;; diagnostics worse -- a `]` inside a datum comment moved the reported
260 ;; location from the argument group into the comment -- and it also
261 ;; leaks the delegation trick, since the reader sees a '(' this parser
262 ;; substituted for the author's '['. Both hazards are named in the
263 ;; text rather than hidden.
264 (define (give-up reason)
265 (fail-here state opening
266 (if delegated-failure
267 (string-append
268 reason
269 "; while delimiting this argument group the standard reader reported: "
270 (exception-message delegated-failure)
271 " (a '(' named there is this parser's substitution for the opening '[')")
272 reason)))
273 (cond
274 ((= closing n) (give-up "unexpected EOF; missing ']'"))
275 ((not (char=? (string-ref source closing) #\]))
276 (candidate (+ closing 1) tried delegated-failure))
277 ((>= tried max-argument-candidates)
278 (give-up (string-append "argument group exceeded "
279 (number->string max-argument-candidates)
280 " candidate ']' boundaries")))
281 (else
282 (string-set! delegated closing #\))
283 (let ((outcome
284 (guard (exn (else (list 'failed exn)))
285 (let* ((reader (make-syntax-reader delegated source-name
286 (position-at state opening)))
287 (stx (syntax-reader-read! reader))
288 (end (char-at-byte state (syntax-reader-position reader))))
289 (if (= end (+ closing 1)) (list 'read stx end) (list 'short))))))
290 (string-set! delegated closing #\])
291 (cond
292 ((eq? (car outcome) 'read)
293 (list (argument-syntax state (cadr outcome)) (caddr outcome) (cadr outcome)))
294 ((eq? (car outcome) 'failed)
295 (candidate (+ closing 1) (+ tried 1) (cadr outcome)))
296 (else (candidate (+ closing 1) (+ tried 1) delegated-failure)))))))))
297 (define (read-author-expression state opening)
298 (let* ((reader (make-syntax-reader (parser-state-source state)
299 (parser-state-source-name state)
300 (position-at state opening)))
301 (stx (syntax-reader-read! reader))
302 (end (char-at-byte state (syntax-reader-position reader))))
303 (list (author-syntax stx) end)))
305 (define (command-char? c)
306 (or (and (char>=? c #\a) (char<=? c #\z)) (char=? c #\-)))
307 (define (horizontal-space? c) (or (char=? c #\space) (char=? c #\tab)))
308 (define (skip-horizontal source i)
309 (let ((n (string-length source)))
310 (let loop ((j i))
311 (if (and (< j n) (horizontal-space? (string-ref source j))) (loop (+ j 1)) j))))
312 (define (block-boundary? source i)
313 (let loop ((j (- i 1)))
314 (cond ((< j 0) #t)
315 ((char=? (string-ref source j) #\newline) #t)
316 ((horizontal-space? (string-ref source j)) (loop (- j 1)))
317 (else #f))))
318 (define (line-tail source i)
319 (let ((n (string-length source)))
320 (let loop ((j i))
321 (cond ((= j n) (list #t j))
322 ((char=? (string-ref source j) #\newline) (list #t (+ j 1)))
323 ((horizontal-space? (string-ref source j)) (loop (+ j 1)))
324 (else (list #f i))))))
325 ;; Return (name end) for a syntactically complete line-level @end marker.
326 (define (named-end source i)
327 (let ((n (string-length source)))
328 (and (block-boundary? source i)
329 (<= (+ i 5) n)
330 (string=? (substring source i (+ i 5)) "@end ")
331 (let name-loop ((j (+ i 5)))
332 (if (and (< j n) (command-char? (string-ref source j)))
333 (name-loop (+ j 1))
334 (and (> j (+ i 5))
335 (let ((tail (line-tail source j)))
336 (and (car tail) (list (substring source (+ i 5) j) (cadr tail))))))))))
337 (define (fold-prose text)
338 ;; A single line break is prose whitespace; blank lines are retained as
339 ;; paragraph separators. Spaces adjacent to a folded break collapse.
340 (let loop ((cs (string->list text)) (out '()) (pending-space? #f) (newlines 0))
341 (cond
342 ((null? cs)
343 (list->string (reverse (if pending-space? (cons #\space out) out))))
344 ((char=? (car cs) #\newline)
345 (loop (cdr cs) out #t (+ newlines 1)))
346 ((and pending-space? (char-whitespace? (car cs)))
347 (loop (cdr cs) out #t newlines))
348 (pending-space?
349 (if (>= newlines 2)
350 (loop cs (cons #\newline (cons #\newline out)) #f 0)
351 (loop cs (if (or (null? out) (char-whitespace? (car out))) out (cons #\space out)) #f 0)))
352 (else (loop (cdr cs) (cons (car cs) out) #f 0)))))
353 (define (trim text)
354 (let* ((n (string-length text))
355 (start (let loop ((i 0)) (if (and (< i n) (char-whitespace? (string-ref text i))) (loop (+ i 1)) i)))
356 (end (let loop ((i n)) (if (and (> i start) (char-whitespace? (string-ref text (- i 1)))) (loop (- i 1)) i))))
357 (substring text start end)))
359 ;; Whitespace immediately inside a command body is delimiter padding, not
360 ;; prose. Internal whitespace (including spacing around nested inline
361 ;; commands) remains untouched.
362 (define (trim-text-left state node)
363 (let ((source (parser-state-source state))
364 (end (fifth node)))
365 (let loop ((start (cadddr node)))
366 (if (and (< start end) (char-whitespace? (string-ref source start)))
367 (loop (+ start 1))
368 (and (< start end)
369 (list 'text '() (fold-prose (substring source start end)) start end))))))
370 (define (trim-text-right state node)
371 (let ((source (parser-state-source state))
372 (start (cadddr node)))
373 (let loop ((end (fifth node)))
374 (if (and (> end start) (char-whitespace? (string-ref source (- end 1))))
375 (loop (- end 1))
376 (and (< start end)
377 (list 'text '() (fold-prose (substring source start end)) start end))))))
378 ;; SLOT COUNT, not head symbol -- and this site is the subtle one. It does
379 ;; not merely MISREAD a six-slot `@text{}` command: `trim-text-left` rebuilds
380 ;; whatever it is given as a five-slot text pseudo-node carrying the folded
381 ;; source slice, so a head-symbol test here silently REWRITES an author's
382 ;; command INTO prose. Everything downstream then behaves correctly on the
383 ;; wrong node, which is why fixing `lower-block` alone left `@text{}` at
384 ;; block position unchanged.
385 (define (trim-body-edges state children)
386 (define (left xs)
387 (cond
388 ((null? xs) '())
389 ((not (pseudo-kind? (car xs) 'text)) xs)
390 (else
391 (let ((first (trim-text-left state (car xs))))
392 (if first (cons first (cdr xs)) (left (cdr xs)))))))
393 (define (right xs)
394 (cond
395 ((null? xs) '())
396 ((not (pseudo-kind? (car xs) 'text)) xs)
397 (else
398 (let ((last (trim-text-right state (car xs))))
399 (if last (cons last (cdr xs)) (right (cdr xs)))))))
400 (reverse (right (reverse (left children)))))
402 ;; Internal invocation: (name args children start end body-kind); text and
403 ;; author expressions are five-slot pseudo-nodes carrying no body kind. The
404 ;; SLOT COUNT, not the head symbol, is what distinguishes them -- `@text{}`
405 ;; and `@expr{}` are legal author commands whose head symbol collides with
406 ;; these pseudo-kinds. `(sigil doc inspect)` classifies on slot count for
407 ;; that reason; SD-P4 left the lowering path on its historical head-symbol
408 ;; tests, recording the defect for this phase.
409 ;;
410 ;; SD-P1 discharges it. Every site below that asked "is the head symbol
411 ;; `text`/`expr`?" now asks "is this a six-slot COMMAND node?" first, which
412 ;; is exact. Three consequences, all intended:
413 ;;
414 ;; * `@text{x}` and `@expr{x}` are dispatched as the ordinary author
415 ;; commands they are. A manuscript that defines `expr` gets its own
416 ;; command; one that does not gets an honest unbound-variable
417 ;; diagnostic -- instead of a paragraph containing the literal string
418 ;; `@text{x}`, or a raw internal node list spliced into generated
419 ;; syntax.
420 ;; * `@text{}` at top level no longer reaches the empty-prose filter in
421 ;; `parse-document-parts-with-profile`, where `caddr` of a COMMAND node
422 ;; is its child list and `trim` was being handed a list, not a string.
423 ;; * The root allowlist this phase adds is itself a head-symbol test, so
424 ;; leaving the collision in place would have let an author's `@text{}`
425 ;; be classified as parser-generated prose at exactly the point the
426 ;; phase exists to make configurable.
427 ;;
428 ;; Measured before/after: `test/sd-p1-classification-probe.sgl` and the two
429 ;; transcripts in `test/transcripts/sd-p1/`.
430 (define (command-node? node) (= (length node) 6))
431 (define (pseudo-kind? node kind)
432 (and (not (command-node? node)) (eq? (car node) kind)))
433 (define (emit-text state text-start end items)
434 (if (= text-start end) items
435 (let ((value (fold-prose (substring (parser-state-source state) text-start end))))
436 (if (= (string-length value) 0) items
437 (cons (list 'text '() value text-start end) items)))))
438 (define (raw-result state start end resume count)
439 (list (if (= start end) '()
440 (list (list 'text '() (substring (parser-state-source state) start end) start end)))
441 resume count))
442 ;; Block-position @code is the one reader-level raw form. Its contents are
443 ;; acquired without interpreting document escapes, commands, or @(...).
444 ;; Brace bodies balance literal braces; named bodies stop only at the exact
445 ;; line-level @end code marker; line bodies stop before the physical newline.
446 (define (scan-raw-code state start body-kind count)
447 (let* ((source (parser-state-source state))
448 (n (string-length source)))
449 (cond
450 ((eq? body-kind 'line)
451 (let loop ((i start))
452 (if (or (= i n) (char=? (string-ref source i) #\newline))
453 (raw-result state start i i count)
454 (loop (+ i 1)))))
455 ((eq? body-kind 'brace)
456 (let loop ((i start) (braces 0))
457 (cond
458 ((= i n) (fail-here state i "unexpected EOF; missing '}'"))
459 ((char=? (string-ref source i) #\{) (loop (+ i 1) (+ braces 1)))
460 ((char=? (string-ref source i) #\})
461 (if (= braces 0)
462 (raw-result state start i (+ i 1) count)
463 (loop (+ i 1) (- braces 1))))
464 (else (loop (+ i 1) braces)))))
465 (else
466 (let loop ((i start))
467 (cond
468 ((= i n) (fail-here state i "unexpected EOF; expected '@end code'"))
469 ((and (char=? (string-ref source i) #\@)
470 (named-end source i)
471 (string=? (car (named-end source i)) "code"))
472 (raw-result state start i (cadr (named-end source i)) count))
473 (else (loop (+ i 1)))))))))
474 (define (scan state start terminator depth count)
475 (if (> depth max-nesting) (error "sigil/doc: nesting exceeds 64")
476 (scan-loop state (string-length (parser-state-source state))
477 terminator depth start start '() count)))
478 (define (scan-loop state n terminator depth i text-start items count)
479 (let ((source (parser-state-source state)))
480 (cond
481 ((= i n)
482 (if (eq? terminator 'line)
483 (list (reverse (emit-text state text-start i items)) i count)
484 (if terminator
485 (fail-here state i
486 (if (eq? terminator 'brace) "unexpected EOF; missing '}'"
487 (string-append "unexpected EOF; expected '@end " terminator "'")))
488 (list (reverse (emit-text state text-start i items)) i count))))
489 ((and (eq? terminator 'line) (char=? (string-ref source i) #\newline))
490 (list (reverse (emit-text state text-start i items)) i count))
491 ((and (eq? terminator 'brace) (char=? (string-ref source i) #\}))
492 (list (reverse (emit-text state text-start i items)) (+ i 1) count))
493 ((and (string? terminator) (named-end source i))
494 (let* ((ending (named-end source i)) (actual (car ending)) (end (cadr ending)))
495 (if (string=? actual terminator)
496 (list (reverse (emit-text state text-start i items)) end count)
497 (fail-here state i
498 (string-append "mismatched named block terminator; expected '@end " terminator
499 "', got '@end " actual "'")))))
500 ((char=? (string-ref source i) #\@)
501 (let ((items (emit-text state text-start i items)))
502 (if (= (+ i 1) n)
503 (fail-here state i "dangling '@'")
504 (let ((next (string-ref source (+ i 1))))
505 (cond
506 ((memv next '(#\@ #\{ #\} #\[ #\]))
507 (let ((escaped (string next)))
508 (scan-loop state n terminator depth (+ i 2) (+ i 2)
509 (cons (list 'text '() escaped i (+ i 2)) items) count)))
510 ((char=? next #\()
511 (let* ((expression (read-author-expression state (+ i 1)))
512 (stx (car expression)) (end (cadr expression)))
513 (scan-loop state n terminator depth end end
514 (cons (list 'expr '() stx i end) items) (+ count 1))))
515 ((command-char? next)
516 (let name-loop ((j (+ i 1)))
517 (if (and (< j n) (command-char? (string-ref source j)))
518 (name-loop (+ j 1))
519 (let* ((name (substring source (+ i 1) j))
520 (after-name (skip-horizontal source j))
521 (arg-result (if (and (< after-name n) (char=? (string-ref source after-name) #\[))
522 (read-arguments state after-name)
523 (list '() j #f)))
524 (args (car arg-result))
525 (after-args (skip-horizontal source (cadr arg-result))))
526 (let* ((tail (line-tail source after-args))
527 (named? (and (block-boundary? source i) (car tail)))
528 (brace? (and (< after-args n) (char=? (string-ref source after-args) #\{)))
529 (line? (and (block-boundary? source i) (< after-args n)
530 (char=? (string-ref source after-args) #\:))))
531 (if (not (or brace? line? named?))
532 (fail-here state after-args
533 "command requires '{...}', a line body, or a line-level named body")
534 (let* ((body-start (cond (brace? (+ after-args 1))
535 (line? (skip-horizontal source (+ after-args 1)))
536 (else (cadr tail))))
537 (body-end (cond (brace? 'brace) (line? 'line) (else name)))
538 (child-result
539 (if (and (string=? name "code") (block-boundary? source i))
540 (scan-raw-code state body-start
541 (cond (brace? 'brace) (line? 'line) (else 'named))
542 (+ count 1))
543 (scan state body-start body-end (+ depth 1) (+ count 1))))
544 (children (trim-body-edges state (car child-result)))
545 (end (cadr child-result)) (new-count (caddr child-result)))
546 (if (> new-count max-commands) (error "sigil/doc: command count exceeds 8192")
547 (scan-loop state n terminator depth end end
548 (cons (list (string->symbol name) args children i end
549 (cond (brace? 'brace)
550 (line? 'line)
551 (else 'named)))
552 items)
553 new-count)))))))))
554 (else (fail-here state i "invalid '@' escape or command")))))))
555 ((and (not terminator) (char=? (string-ref source i) #\}))
556 (fail-here state i "unexpected '}'"))
557 (else (scan-loop state n terminator depth (+ i 1) text-start items count)))))
559 (define (inline-list state nodes)
560 (apply append (map (lambda (node) (lower-inline state node)) nodes)))
561 (define (fifth value) (car (cdr (cdr (cdr (cdr value))))))
562 (define (sixth value) (car (cdr (cdr (cdr (cdr (cdr value)))))))
563 (define (raw-children state children)
564 (let ((source (parser-state-source state)))
565 (apply string-append
566 (map (lambda (child) (substring source (cadddr child) (fifth child))) children))))
567 ;; SLOT COUNT again: `caddr` of a five-slot text node is its STRING, but of
568 ;; a six-slot command it is the child LIST, so a head-symbol test here hands
569 ;; `string-append` a list. `@code{@text{x}}` now takes the else branch and
570 ;; gets this procedure's own diagnostic instead of a type error from inside
571 ;; `string-append`.
572 (define (inline-code-content state start children)
573 (let loop ((xs children) (out '()))
574 (cond
575 ((null? xs) (apply string-append (reverse out)))
576 ((pseudo-kind? (car xs) 'text) (loop (cdr xs) (cons (caddr (car xs)) out)))
577 (else
578 (fail-here state start
579 "inline @code content must be text and document escapes")))))
580 (define (author-expression state start end expression name)
581 (generated-call state start end
582 (list 'author-at (loc-datum state start end)
583 (list 'lambda '()
584 (list 'author-normalize expression
585 (loc-datum state start end)
586 (list 'quote name))))))
587 (define (command-expression state node lowered-children)
588 (let ((kind (car node)) (args (cadr node)) (start (cadddr node)) (end (fifth node)))
589 (author-expression state start end
590 (cons kind (append args lowered-children)) kind)))
591 (define (hoisted-author-expression state start end stx)
592 ;; Deterministic, reader-private binding spelling: stable compiled output
593 ;; without occupying an ordinary author identifier such as `expression-1`.
594 (let ((name (string->symbol
595 (string-append "#%sigil/doc-expression:" (number->string start)))))
596 (set-parser-state-definitions! state
597 (cons (generated-call state start end
598 (list 'define name
599 (list 'lambda '()
600 (list 'author-normalize stx
601 (loc-datum state start end)
602 (list 'quote 'expression)))))
603 (parser-state-definitions state)))
604 (list name)))
605 (define (inline-author-expression state start end stx)
606 (generated-call state start end
607 (hoisted-author-expression state start end stx)))
608 (define (code-expression state node)
609 (let ((args (cadr node)) (children (caddr node))
610 (start (cadddr node)) (end (fifth node)) (body-kind (sixth node)))
611 (cond
612 ((and (null? args) (eq? body-kind 'brace))
613 (author-expression state start end
614 (list 'code (trim (inline-code-content state start children))) 'code))
615 ((or (null? args) (= (length args) 1))
616 (author-expression state start end
617 (list 'code (if (null? args) #f (car args))
618 (trim (raw-children state children)))
619 'code))
620 (else (fail-here state start "invalid arguments for @code")))))
621 ;; `paragraph-ranges` used to live here. SD-P3 splits its two jobs: the
622 ;; blank-line scan is `paragraph-segments`, and the value/location shaping is
623 ;; `lower-prose-run`, which reproduces this procedure's exact output for a
624 ;; prose-only run.
625 (define (lower-inline state node)
626 (let ((kind (car node)) (args (cadr node)) (children (caddr node)) (start (cadddr node)) (end (fifth node)))
627 (cond
628 ((pseudo-kind? node 'text)
629 (let ((value (caddr node)))
630 (if (= (string-length value) 0) '()
631 (list (generated-call state start end (list 'make-text value (loc-datum state start end)))))))
632 ((pseudo-kind? node 'expr)
633 (list (inline-author-expression state start end children)))
634 ((eq? kind 'code)
635 (list (code-expression state node)))
636 (else (list (command-expression state node
637 (inline-list state children)))))))
638 ;; `lower-block` now lowers only nodes that stand at BLOCK POSITION. Prose
639 ;; and everything folded into a prose run is `block-list`'s business below,
640 ;; so the historical `text` branch has moved rather than disappeared.
641 (define (lower-block state node)
642 (let ((kind (car node)) (children (caddr node)) (start (cadddr node)) (end (fifth node)))
643 (cond
644 ((pseudo-kind? node 'expr)
645 (list (inline-author-expression state start end children)))
646 ((eq? kind 'p)
647 (list (command-expression state node
648 (inline-list state children))))
649 ((eq? kind 'code)
650 (list (code-expression state node)))
651 (else (list (command-expression state node
652 (block-list state children)))))))
654 ;; ---------------------------------------------------- SD-P3: prose runs
655 ;;
656 ;; Review finding B4. Before this phase, a text run at block position was
657 ;; split into paragraphs and ANY command interleaved with it was lifted out
658 ;; as a sibling BLOCK child -- so `Prose with @code{x} in it.` produced a
659 ;; paragraph, a stray inline code node, and another paragraph, and the
660 ;; container rejected the code node with "child belongs to the wrong
661 ;; semantic category". Every marked-up paragraph therefore had to be
662 ;; `@p{...}`-wrapped.
663 ;;
664 ;; The rule, which is deliberately narrow:
665 ;;
666 ;; * a command (or `@(...)`) that is NOT at a block boundary folds into
667 ;; the enclosing paragraph run;
668 ;; * anything AT a block boundary keeps exactly its previous semantics;
669 ;; * blank lines remain the paragraph separator and close the run.
670 ;;
671 ;; The boundary case is load-bearing, not timidity: `Intro.` / `@note{...}`
672 ;; / `More.` on three consecutive lines with no blank line between them is a
673 ;; real, working pattern, and folding a line-initial command would turn it
674 ;; into a paragraph containing an admonition. The cost is that a command
675 ;; that happens to land at the start of a WRAPPED line is still treated as a
676 ;; block; that residue is recorded in `docs/prose-runs.md` and is a design
677 ;; question (when is a line break formatting, and when is it a separator?)
678 ;; rather than something to settle inside a parser.
679 ;;
680 ;; What this code does NOT decide is whether a folded run is a paragraph.
681 ;; It cannot: the parser has no command vocabulary, so block-versus-inline
682 ;; is only knowable after the command runs. A run that is entirely prose is
683 ;; emitted as today's paragraph, byte for byte; every other run is handed to
684 ;; `author-paragraph-run`, which decides once the values exist. See the
685 ;; commentary there.
686 (define (block-position? state node)
687 (block-boundary? (parser-state-source state) (cadddr node)))
689 ;; The same blank-line scan `paragraph-ranges` performs, but returning the
690 ;; RANGES only and keeping the empty ones: an empty segment still closes a
691 ;; run even though it contributes no text.
692 (define (paragraph-segments state start end)
693 (let ((source (parser-state-source state)))
694 (let loop ((i start) (part start) (out '()))
695 (cond
696 ((= i end) (reverse (cons (list part end) out)))
697 ((and (char=? (string-ref source i) #\newline)
698 (< (+ i 1) end) (char=? (string-ref source (+ i 1)) #\newline))
699 (loop (+ i 2) (+ i 2) (cons (list part i) out)))
700 (else (loop (+ i 1) part out))))))
702 (define (trimmed-start state start end)
703 (let ((source (parser-state-source state)))
704 (let loop ((i start))
705 (if (and (< i end) (char-whitespace? (string-ref source i))) (loop (+ i 1)) i))))
706 (define (trimmed-end state start end)
707 (let ((source (parser-state-source state)))
708 (let loop ((i end))
709 (if (and (> i start) (char-whitespace? (string-ref source (- i 1)))) (loop (- i 1)) i))))
711 ;; Run items: (text <node> start end) for a prose segment, (node <node>)
712 ;; for a folded command or author expression. A prose item carries its
713 ;; originating pseudo-node, not just a range -- see `segment-value`.
714 (define (text-item? item) (eq? (car item) 'text))
715 (define (item-node item) (cadr item))
717 ;; A TEXT PSEUDO-NODE'S VALUE IS NOT ALWAYS ITS SOURCE SLICE, and rebuilding
718 ;; one from the slice is a real defect rather than a shortcut. `scan-loop`
719 ;; emits a document escape as a text node whose VALUE is the single
720 ;; unescaped character while its SPAN covers the two source characters:
721 ;; `@@` -> `@`, `@{` -> `{`, and so on. Re-deriving from the slice turns
722 ;; `@@` back into two at-signs.
723 ;;
724 ;; `lower-inline` -- the path `@p{...}` takes -- reads `caddr`. So bare
725 ;; prose must read `caddr` too, or the two spellings disagree on exactly the
726 ;; documents that use an escape outside `@p{}`. Found by adversarial review;
727 ;; no manuscript in the 74-document corpus writes one, so nothing else here
728 ;; could have caught it.
729 ;;
730 ;; A segment that covers its node EXACTLY uses the node's own value. A
731 ;; segment that is a proper part of its node came from a blank-line split,
732 ;; so that node contains a newline and is ordinary prose rather than an
733 ;; escape; re-deriving from the slice is correct there and is what the
734 ;; pre-SD-P3 lowering did.
735 ;;
736 ;; For ordinary prose the two are the same string by construction:
737 ;; `emit-text` and `trim-body-edges` both build a text node's value as
738 ;; `(fold-prose (substring source start end))` over exactly that range.
739 (define (segment-value state node start end)
740 (if (and (= start (cadddr node)) (= end (fifth node)))
741 (caddr node)
742 (fold-prose (substring (parser-state-source state) start end))))
744 ;; A run made only of prose is emitted in EXACTLY the shape this parser has
745 ;; always emitted for a paragraph -- same constructors, same untrimmed
746 ;; segment range for both the paragraph and its text, same trimmed folded
747 ;; value. That is what makes the default lowering byte-identical for every
748 ;; manuscript containing no non-boundary command.
749 (define (lower-prose-run state node start end)
750 (let ((value (trim (segment-value state node start end))))
751 (if (= (string-length value) 0) '()
752 (list (generated-call state start end
753 (list 'make-paragraph
754 (list 'list (list 'make-text value (loc-datum state start end)))
755 (loc-datum state start end)))))))
757 ;; A mixed run. Each prose segment is trimmed on the side that faces the
758 ;; run's edge and left alone on the side that faces a command, so the folded
759 ;; text carries exactly the spacing the `@p{...}` spelling carries; its
760 ;; location is the trimmed range, which is what `trim-body-edges` gives the
761 ;; equivalent children inside `@p{...}`.
762 ;; `item-start`/`item-end` must agree with the range `lower-run-item`
763 ;; actually used, so all three derive it from one place.
764 (define (item-start state item first?)
765 (if (text-item? item)
766 (if first? (trimmed-start state (caddr item) (cadddr item)) (caddr item))
767 (cadddr (item-node item))))
768 (define (item-end state item first? last?)
769 (if (text-item? item)
770 (if last? (trimmed-end state (item-start state item first?) (cadddr item))
771 (cadddr item))
772 (fifth (item-node item))))
773 (define (lower-run-item state item first? last?)
774 (if (text-item? item)
775 (let ((start (item-start state item first?))
776 (end (item-end state item first? last?)))
777 (lower-inline state
778 (list 'text '() (segment-value state (item-node item) start end)
779 start end)))
780 (lower-inline state (item-node item))))
782 (define (lower-mixed-run state items)
783 ;; Drop the prose segments that trim away to nothing -- a run may open or
784 ;; close on whitespace -- BEFORE deciding the run's span, so the span
785 ;; slices to content rather than to layout.
786 (let* ((count (length items))
787 (kept (let loop ((xs items) (i 0) (out '()))
788 (if (null? xs) (reverse out)
789 (let* ((first? (= i 0)) (last? (= i (- count 1)))
790 (lowered (lower-run-item state (car xs) first? last?)))
791 (loop (cdr xs) (+ i 1)
792 (if (null? lowered) out
793 (cons (list (item-start state (car xs) first?)
794 (item-end state (car xs) first? last?)
795 lowered)
796 out))))))))
797 (if (null? kept) '()
798 (let ((start (car (car kept)))
799 (end (cadr (car (reverse kept)))))
800 (list (generated-call state start end
801 (list 'author-paragraph-run
802 (cons 'list (apply append (map caddr kept)))
803 (loc-datum state start end))))))))
805 (define (close-run state run out)
806 (if (null? run) out
807 (let ((items (reverse run)))
808 (append (reverse (if (and (null? (cdr items)) (text-item? (car items)))
809 (lower-prose-run state (item-node (car items))
810 (caddr (car items)) (cadddr (car items)))
811 (lower-mixed-run state items)))
812 out))))
814 (define (block-list state nodes)
815 ;; `out` and `run` are both reversed; `out` is reversed once at the end.
816 (let loop ((xs nodes) (run '()) (out '()))
817 (if (null? xs) (reverse (close-run state run out))
818 (let ((node (car xs)))
819 (cond
820 ((pseudo-kind? node 'text)
821 ;; The first segment continues the current run; every later
822 ;; segment follows a blank line, so it starts a new one.
823 ;;
824 ;; Two text nodes CAN be adjacent -- `scan-loop` flushes the
825 ;; prose before a document escape and emits the escape as its
826 ;; own text node -- so a run may legitimately hold several
827 ;; prose items with no command between them.
828 (let seg ((parts (paragraph-segments state (cadddr node) (fifth node)))
829 (run run) (out out))
830 (let ((item (list 'text node (car (car parts)) (cadr (car parts)))))
831 (if (null? (cdr parts))
832 (loop (cdr xs) (cons item run) out)
833 (seg (cdr parts) '() (close-run state (cons item run) out))))))
834 ((block-position? state node)
835 ;; SEQUENCED DELIBERATELY. Both `close-run` and `lower-block`
836 ;; can hoist an author expression into `parser-state-definitions`
837 ;; via `hoisted-author-expression`, so leaving them as two
838 ;; arguments to `append` would make the emitted definition order
839 ;; depend on the host's argument-evaluation order -- and, under
840 ;; left to right, REVERSE it relative to the manuscript, because
841 ;; the later block would be lowered before the earlier run was
842 ;; closed. This is the same hazard that made
843 ;; `parse-document-parts-with-profile` use an explicit fold
844 ;; instead of `map`.
845 (let* ((closed (close-run state run out))
846 (lowered (lower-block state node)))
847 (loop (cdr xs) '() (append (reverse lowered) closed))))
848 (else
849 (loop (cdr xs) (cons (list 'node node) run) out)))))))
851 ;; Passive scan (finding I4). Returns the raw internal node tree and the
852 ;; parser state that owns its location index. It performs NO lowering: no
853 ;; `author-at`, no `author-normalize`, no generated syntax, no evaluation,
854 ;; and no root-container policy -- the root container is a lowering rule,
855 ;; enforced by `parse-document-parts-with-profile` below against a profile,
856 ;; and is not a fact about the grammar.
857 ;; `(sigil doc inspect)` is the documented surface over this.
858 (define (parse-document-nodes source source-name header-end)
859 (let* ((state (make-parser-state source source-name))
860 (header-cursor (char-at-byte state header-end))
861 (result (scan state header-cursor #f 0 0)))
862 (list state (car result))))
863 (define (parser-node-location state start end) (location state start end))
864 (define (parser-node-cursor state index) (cursor-at state index))
866 ;; The ROOT POLICY, and the only place in this parser that has one.
867 ;;
868 ;; Returns `(author-forms roots)`, where `roots` is a list of lowered root
869 ;; syntaxes in manuscript order -- one element under `'one`, one or more
870 ;; under `'one-or-more`. The shape is a list for BOTH multiplicities so a
871 ;; delegating reader never has to discriminate on it.
872 ;;
873 ;; The policy is exactly three rules, and each is now read from the profile
874 ;; instead of written into this loop:
875 ;; 1. a top-level author expression is hoisted as a form;
876 ;; 2. a top-level command whose name is in the profile's root set becomes
877 ;; a root -- and NOTHING else may appear at top level;
878 ;; 3. every author form must precede every root, and the number of roots
879 ;; must satisfy the profile's multiplicity.
880 ;; Rules 1 and 2 discriminate on SLOT COUNT (see `command-node?` above), so
881 ;; a manuscript's `@text{}`/`@expr{}` command is never mistaken for one of
882 ;; the parser's own pseudo-nodes.
883 ;;
884 ;; TWO PASSES, and the split is load-bearing rather than tidiness. The
885 ;; pre-SD-P1 loop decided "is this the last node?" BEFORE lowering the
886 ;; chapter, so a policy error always preceded any lowering error. A
887 ;; single-pass rewrite that lowers each root as it meets it quietly inverts
888 ;; that for a doubly-malformed manuscript -- one with, say, a bad `@code`
889 ;; inside the root AND trailing top-level content would start reporting the
890 ;; `@code` error instead of the root-policy error. Deciding the entire
891 ;; policy first preserves the old precedence under every profile, and means
892 ;; a rejected manuscript is never partially lowered -- which matters to the
893 ;; folio indexer, whose normal case is scanning files it will reject.
894 (define (parse-document-parts-with-profile profile source source-name header-end)
895 (if (not (document-profile? profile))
896 (error "sigil/doc: parse requires a document profile")
897 (let* ((state (make-parser-state source source-name))
898 (header-cursor (char-at-byte state header-end))
899 (result (scan state header-cursor #f 0 0))
900 (roots-allowed (document-profile-record-roots profile))
901 (several? (eq? (document-profile-record-multiplicity profile) 'one-or-more))
902 (nodes (filter (lambda (node)
903 (not (and (pseudo-kind? node 'text)
904 (= (string-length (trim (caddr node))) 0))))
905 (car result))))
906 ;; PASS 1 -- decide the policy. Nothing is lowered here.
907 (let classify ((xs nodes) (expressions '()) (root-nodes '()))
908 (cond
909 ((null? xs)
910 (if (null? root-nodes)
911 (error (profile-missing-root-message profile))
912 ;; PASS 2 -- lower, strictly left to right. An explicit
913 ;; fold rather than `map`: hoisted author expressions are
914 ;; accumulated into the parse state AS a root's body is
915 ;; lowered, so an unspecified application order would make
916 ;; the emitted definition order depend on the host.
917 (let lower ((pending (reverse root-nodes)) (roots '()))
918 (if (pair? pending)
919 (lower (cdr pending)
920 (cons (command-expression state (car pending)
921 (block-list state (caddr (car pending))))
922 roots))
923 ;; The returned syntax owns every location/form it
924 ;; needs. The parser state -- manuscript, location
925 ;; index, accumulator -- is unreachable once this
926 ;; call returns, so no scratch state can retain the
927 ;; last manuscript or leak into a nested parse.
928 (list (append (reverse expressions)
929 (reverse (parser-state-definitions state)))
930 (reverse roots))))))
931 ;; An author form after a root is the misplacement this parser
932 ;; has always rejected, and it stays rejected under every
933 ;; profile.
934 ((pseudo-kind? (car xs) 'expr)
935 (if (pair? root-nodes)
936 (error (profile-misplaced-message profile))
937 (classify (cdr xs) (cons (caddr (car xs)) expressions) root-nodes)))
938 ((and (command-node? (car xs))
939 (memq (car (car xs)) roots-allowed)
940 (or several? (null? root-nodes)))
941 (classify (cdr xs) expressions (cons (car xs) root-nodes)))
942 (else (error (profile-misplaced-message profile))))))))
944 ;; `parse-document-source` and `parse-document-forms` keep their exact
945 ;; three-argument spellings and delegate with `sigil-doc-profile`. That is
946 ;; not politeness: `test/test-reader.sgl` calls them 26 times, and SD-P1's
947 ;; acceptance requires the D1/D2 suites green UNCHANGED.
948 (define (profile-single-root profile roots)
949 (if (null? (cdr roots)) (car roots)
950 (error (string-append (document-profile-record-name profile)
951 ": this profile admits several roots; use"
952 " parse-document-parts-with-profile"))))
953 (define (parse-document-source-with-profile profile source source-name header-end)
954 (let ((parts (parse-document-parts-with-profile profile source source-name header-end)))
955 (if (null? (car parts)) (profile-single-root profile (cadr parts))
956 (error (string-append (document-profile-record-name profile)
957 ": top-level author forms require parse-document-forms")))))
958 (define (parse-document-forms-with-profile profile source source-name header-end)
959 (let ((parts (parse-document-parts-with-profile profile source source-name header-end)))
960 (append (car parts) (list (profile-single-root profile (cadr parts))))))
961 (define (parse-document-source source source-name header-end)
962 (parse-document-source-with-profile sigil-doc-profile source source-name header-end))
963 (define (parse-document-forms source source-name header-end)
964 (parse-document-forms-with-profile sigil-doc-profile source source-name header-end))))