AtlatestRepositorysigil-doc

sigil-doc / tree / testtest-prose-runs.sgl

1;;; SD-P3 -- inline commands in bare prose (review finding B4).
2;;;
3;;; The central claim is an EQUALITY: bare prose containing inline commands
4;;; produces the same tree as the `@p{...}`-wrapped spelling of the same prose.
5;;; Equality, not absence of complaint -- so the comparison is over the shipped
6;;; canonical serializer with source locations stripped, the arm prints the
7;;; denominators it compared, and a positive control proves the comparator can
8;;; SEE a difference when one exists.
9(import (sigil test) (sigil core) (sigil fs) (sigil syntax) (sigil language)
10 (sigil doc) (sigil doc author)
11 (sigil doc private author-runtime) (sigil doc private parser))
13(define (eval-document source name)
14 (eval (parse-document-source source name 16)))
15;; The two fixtures are FILES, and they are the same artifact the gate's
16;; premise arm evaluates against a pristine pre-SD-P3 checkout. Reading them
17;; here rather than restating them keeps one source of truth: a fixture that
18;; drifts from the test asserting on it proves nothing about the other consumer.
19(define (load-fixture path)
20 (let* ((source (read-file-string path))
21 (header (inspect-language-source source path)))
22 (list source (eval (parse-document-source source path (language-header-end header))))))
23(define (eval-document-forms source name)
24 (let loop ((forms (parse-document-forms source name 16)) (last #f))
25 (if (null? forms) last (loop (cdr forms) (eval (car forms))))))
27;; ------------------------------------------------------------------ helpers
28;;
29;; `document->datum` embeds every location as a `(source file pos line col span)`
30;; list. Two spellings of the same prose CANNOT have equal locations -- the
31;; `@p{` and `}` shift every later byte -- so structural equality is asserted
32;; over the datum with those lists removed, and the locations are asserted
33;; separately, against independently recomputed byte/scalar values.
34;; The "this element was a location, drop it" signal is a FRESH PAIR used as a
35;; sentinel, compared with `eq?`. Signalling with `#f` would conflate a dropped
36;; location with a datum whose value genuinely IS `#f` -- `(language #f)` on an
37;; unlabelled code block is exactly that shape -- and silently erase the
38;; difference between `(language #f)` and `(language)` on BOTH sides of the
39;; comparison, which is the kind of erasure that makes an equality weaker than
40;; it looks.
41;; And the discriminator is the LOCATION'S SHAPE, not its head symbol. `source`
42;; is also the head of `(source <url>)` on a video and `(source <text>)` on an
43;; example (traversal.sgl), so matching on the symbol alone would erase two
44;; videos' differing URLs and make them compare equal.
45(define (location-datum? value)
46 (and (pair? value) (eq? (car value) 'source)
47 (= (length value) 6)
48 (string? (cadr value))
49 (integer? (caddr value)) (integer? (cadddr value))))
50(define dropped (list 'dropped))
51(define (strip-locations value)
52 (cond
53 ((location-datum? value) dropped)
54 ((pair? value)
55 (let loop ((xs value) (out '()))
56 (cond
57 ((null? xs) (reverse out))
58 ((not (pair? xs)) (append (reverse out) xs))
59 (else
60 (let ((kept (strip-locations (car xs))))
61 (loop (cdr xs) (if (eq? kept dropped) out (cons kept out))))))))
62 (else value)))
64;; Denominators. An equality over two trees that contain nothing passes for
65;; free, so every equality assertion below states what it actually compared.
66(define (count-tag datum tag)
67 (cond
68 ((and (pair? datum) (eq? (car datum) tag))
69 (+ 1 (apply + (map (lambda (x) (count-tag x tag)) (filter pair? (cdr datum))))))
70 ((pair? datum)
71 (apply + (map (lambda (x) (count-tag x tag)) (filter pair? datum))))
72 (else 0)))
74(define (replace-first haystack needle replacement)
75 (let ((hn (string-length haystack)) (nn (string-length needle)))
76 (let loop ((i 0))
77 (cond ((> (+ i nn) hn) haystack)
78 ((string=? (substring haystack i (+ i nn)) needle)
79 (string-append (substring haystack 0 i) replacement
80 (substring haystack (+ i nn) hn)))
81 (else (loop (+ i 1)))))))
83(define (contains-substring? haystack needle)
84 (let ((hn (string-length haystack)) (nn (string-length needle)))
85 (let loop ((i 0))
86 (cond ((> (+ i nn) hn) #f)
87 ((string=? (substring haystack i (+ i nn)) needle) #t)
88 (else (loop (+ i 1)))))))
90;; ------------------------------------------------------------ the fixtures
91;;
92;; The same prose twice. Everything that differs between them is a `@p{` and a
93;; `}`; `naïve` and `café` are here so the fixture has multi-byte scalars and a
94;; byte-versus-scalar difference the location arm can assert on.
95(define bare-loaded (load-fixture "test/fixtures/sd-p3-bare-prose.sgd"))
96(define wrapped-loaded (load-fixture "test/fixtures/sd-p3-wrapped.sgd"))
97(define bare (car bare-loaded))
98(define bare-doc (cadr bare-loaded))
99(define wrapped-doc (cadr wrapped-loaded))
100(define bare-datum (strip-locations (document->datum bare-doc)))
101(define wrapped-datum (strip-locations (document->datum wrapped-doc)))
103(test "bare prose with inline commands equals the @p-wrapped spelling"
104 ;; Denominators first: this equality must not be able to hold vacuously.
105 (assert-equal 2 (length (section-children (car (document-children bare-doc)))))
106 (assert-true (>= (count-tag bare-datum 'text) 6))
107 (assert-equal 2 (count-tag bare-datum 'code))
108 (assert-equal 1 (count-tag bare-datum 'reference))
109 (assert-equal 1 (count-tag bare-datum 'emphasis))
110 ;; SD-P2's inline url-link node, folded by an SD-P3 run. This combination
111 ;; exists only after the two phases are merged and NEITHER PHASE TESTED IT.
112 (assert-equal 1 (count-tag bare-datum 'url-link))
113 (assert-equal 2 (count-tag bare-datum 'paragraph))
114 ;; ... and the same denominators on the other side, so an equality between
115 ;; two identically-empty trees is impossible rather than merely unlikely.
116 (assert-equal (count-tag bare-datum 'text) (count-tag wrapped-datum 'text))
117 (assert-equal (count-tag bare-datum 'code) (count-tag wrapped-datum 'code))
118 (assert-equal bare-datum wrapped-datum)
119 (display "SD_P3_EQUALITY_COMPARED text ") (display (count-tag bare-datum 'text))
120 (display " code ") (display (count-tag bare-datum 'code))
121 (display " reference ") (display (count-tag bare-datum 'reference))
122 (display " emphasis ") (display (count-tag bare-datum 'emphasis))
123 (display " url-link ") (display (count-tag bare-datum 'url-link))
124 (display " paragraph ") (display (count-tag bare-datum 'paragraph))
125 (newline))
127(test "the comparator can see a difference when one exists"
128 ;; The positive control for the arm above. Without it, `assert-equal` over
129 ;; `strip-locations` is consistent with a stripper that removes everything.
130 ;; Derived from the fixture by one substitution, so the control cannot drift
131 ;; away from the thing it is controlling for.
132 (let* ((altered-source (replace-first bare "@code{define}" "@code{DEFINE}"))
133 (altered (eval-document altered-source "altered.sgd")))
134 (assert-false (string=? altered-source bare))
135 (assert-false (equal? bare-datum (strip-locations (document->datum altered))))
136 ;; ... and the stripper is not simply deleting the content it compares:
137 ;; the two data differ only in that one code node's content, so the
138 ;; difference the comparator saw is a real one inside the kept structure.
139 ;; Both counts are taken from ALTERED, the value under control -- counting
140 ;; `bare-datum` again would restate the previous test rather than say
141 ;; anything about this one.
142 (assert-equal (count-tag bare-datum 'code)
143 (count-tag (strip-locations (document->datum altered)) 'code))
144 (assert-true (> (count-tag (strip-locations (document->datum altered)) 'text) 0))))
146(test "every node of a bare-prose run carries its exact manuscript location"
147 ;; Independently recomputed from the fixture bytes: the byte position is the
148 ;; UTF-8 length of everything before the node, the span is the UTF-8 length of
149 ;; the node's own text, and line/column are counted in SCALARS.
150 (let* ((section (car (document-children bare-doc)))
151 ;; BOTH paragraphs: the second carries the `@@` document escape, whose
152 ;; node value ("@") deliberately differs from its two-character source
153 ;; slice ("@@"). Walking only the first would leave the one node in the
154 ;; fixture whose value is not its slice unexamined.
155 (children (apply append
156 (map paragraph-children (section-children section))))
157 (checked 0))
158 (define (utf8-length s)
159 (let loop ((i 0) (n 0))
160 (if (= i (string-length s)) n
161 (let ((c (char->integer (string-ref s i))))
162 (loop (+ i 1) (+ n (cond ((<= c #x7f) 1) ((<= c #x7ff) 2)
163 ((<= c #xffff) 3) (else 4))))))))
164 ;; The precondition that stops this arm passing vacuously: the fixture has
165 ;; multi-byte scalars, so byte offsets and scalar offsets DIVERGE. If they
166 ;; agreed, an implementation that returned scalars everywhere would pass.
167 (assert-true (> (utf8-length bare) (string-length bare)))
168 (for-each
169 (lambda (node)
170 (let* ((loc (node-location node))
171 (position (srcloc-position loc))
172 (span (srcloc-span loc))
173 ;; Recover the node's source slice from the byte offsets by
174 ;; walking the manuscript in scalars and counting bytes.
175 (slice (let loop ((i 0) (byte 0) (from #f))
176 (cond
177 ((and (not from) (= byte position)) (loop i byte i))
178 ((and from (= byte (+ position span))) (substring bare from i))
179 ((= i (string-length bare)) (if from (substring bare from i) ""))
180 (else
181 (let ((c (char->integer (string-ref bare i))))
182 (loop (+ i 1)
183 (+ byte (cond ((<= c #x7f) 1) ((<= c #x7ff) 2)
184 ((<= c #xffff) 3) (else 4)))
185 from)))))))
186 (set! checked (+ checked 1))
187 (cond
188 ;; A document escape's node value is ONE character and its span
189 ;; covers TWO -- `@@` -> `@`. That is the shape a run must not
190 ;; rebuild from the slice, so it is asserted explicitly rather than
191 ;; papered over with a looser comparison.
192 ((and (text? node) (= (string-length slice) 2)
193 (char=? (string-ref slice 0) #\@))
194 (assert-equal (string-append "@" (text-content node)) slice))
195 ((text? node) (assert-equal (text-content node) slice))
196 ((code? node) (assert-equal (string-append "@code{" (code-content node) "}") slice))
197 (else (assert-true (> (string-length slice) 0))))
198 ;; line/column are scalar-counted and must address the slice's start
199 (assert-equal (substring slice 0 1)
200 (let* ((line (srcloc-line loc)) (column (srcloc-column loc)))
201 (let loop ((i 0) (l 1) (c 1))
202 (if (and (= l line) (= c column)) (substring bare i (+ i 1))
203 (if (char=? (string-ref bare i) #\newline)
204 (loop (+ i 1) (+ l 1) 1)
205 (loop (+ i 1) l (+ c 1)))))))))
206 children)
207 ;; Each paragraph spans from ITS first child to ITS last -- BOTH ends.
208 ;; Asserting only the start passes for a paragraph that stops at its first
209 ;; child, which is a span an indexer would silently truncate on.
210 (for-each
211 (lambda (p)
212 (let* ((kids (paragraph-children p))
213 (last (car (reverse kids))))
214 (assert-equal (srcloc-position (node-location (car kids)))
215 (srcloc-position (paragraph-location p)))
216 (assert-equal (+ (srcloc-position (node-location last))
217 (srcloc-span (node-location last)))
218 (+ (srcloc-position (paragraph-location p))
219 (srcloc-span (paragraph-location p))))
220 ;; ... and each run really does have more than one child, so the two
221 ;; assertions above are not the same assertion twice.
222 (assert-true (> (length kids) 1))))
223 (section-children section))
224 (display "SD_P3_LOCATIONS_CHECKED ") (display checked) (display " nodes") (newline)
225 (assert-true (>= checked 6))))
227(test "a run whose only value is a block node is spliced, exactly as before"
228 ;; `test/fixtures/minimal.sgd` writes `@tip{@p{...}}` and
229 ;; `@exercise[...]{@p{...}}`: the `@p` opens the body on the same line as the
230 ;; brace, so it is NOT at a block boundary and this phase folds it into a run.
231 ;; Folding it INTO a paragraph would nest a paragraph in a paragraph. The
232 ;; run's values decide, and one block value means "hand it back unchanged".
233 (let* ((doc (eval-document
234 (string-append "#lang sigil/doc\n@article[\"T\" \"A\"]{\n"
235 "@aside{@p{Remember @code{Sigil}.}}\n}")
236 "splice.sgd"))
237 (admonition (car (document-children doc))))
238 (assert-true (admonition? admonition))
239 (assert-equal 1 (length (admonition-children admonition)))
240 (assert-true (paragraph? (car (admonition-children admonition))))
241 (assert-true (code? (cadr (paragraph-children (car (admonition-children admonition))))))))
243(test "a run that mixes inline and block values is refused with an exact location"
244 ;; The corpus contains zero instances of this shape, so it gets a deliberate
245 ;; test rather than incidental coverage.
246 ;;
247 ;; `@anchor` is the right probe: it is a BLOCK node that accepts no children,
248 ;; so it evaluates successfully inside a run and the run itself is what has to
249 ;; refuse it. A block command that also demands BLOCK CHILDREN (`@aside{…}`)
250 ;; never reaches this branch -- see the next test, which pins that separately.
251 (let ((message
252 (guard (exn (else (exception-message exn)))
253 (eval-document
254 (string-append "#lang sigil/doc\n@article[\"T\" \"A\"]{\n"
255 "@section[\"S\" id: \"s\"]{\n"
256 "Prose then @anchor[\"mid\"]{} then more prose.\n}\n}")
257 "mixed.sgd"))))
258 (assert-true (string? message))
259 (assert-true (> (string-length message) 0))
260 (display "SD_P3_MIXED_RUN_MESSAGE ") (display message) (newline)
261 ;; names the file, a line, a column, and the reason
262 (assert-true (contains-substring? message "mixed.sgd:"))
263 (assert-true (contains-substring? message "mixes inline and block content"))))
265(test "an inline url-link folds into a bare-prose run"
266 ;; CROSS-PHASE. SD-P2 added `@url` as an INLINE node; SD-P3 folds
267 ;; non-boundary commands into paragraph runs. `@url` mid-paragraph in BARE
268 ;; prose therefore exercises a path that exists only on the merged tree, and
269 ;; neither phase's own gate covered it: SD-P2's fixtures put every `@url`
270 ;; inside an explicit `@p{...}`, and SD-P3's predate the node entirely.
271 ;;
272 ;; The mechanism it depends on is that `url-link?` is in `inline-node?`
273 ;; (model.sgl), so a run containing one is all-inline and becomes a
274 ;; paragraph rather than a "mixes inline and block content" refusal.
275 (let* ((doc (eval-document
276 (string-append "#lang sigil/doc\n@article[\"T\" \"A\"]{\n"
277 "@section[\"S\" id: \"s\"]{\n"
278 "See @url[\"https://example.org/a\"]{the spec} and "
279 "@url[\"https://example.org/b\"]{the guide} for more.\n}\n}")
280 "urls.sgd"))
281 (children (section-children (car (document-children doc)))))
282 (assert-equal 1 (length children))
283 (assert-true (paragraph? (car children)))
284 ;; text, url, text, url, text
285 (assert-equal 5 (length (paragraph-children (car children))))
286 (assert-true (url-link? (cadr (paragraph-children (car children)))))
287 (assert-true (url-link? (cadddr (paragraph-children (car children)))))
288 ;; the space BETWEEN two inline url-links is content and must survive
289 (assert-equal " and " (text-content (caddr (paragraph-children (car children)))))))
291(test "a folded command that wants block children fails exactly as @p does"
292 ;; A folded command's children are lowered INLINE, because that is what the
293 ;; overwhelmingly common case (`@code`, `@ref`, `@em`) needs. So a command
294 ;; that demands BLOCK children -- `@aside{…}` -- fails inside its own
295 ;; constructor before the run ever gets to judge its value.
296 ;;
297 ;; It is EXACTLY what the `@p{…}`-wrapped spelling of the same prose has
298 ;; always done, and this phase's claim is that the two spellings agree.
299 ;;
300 ;; BUT NOTE WHAT THIS ASSERTION IS AND IS NOT. `fail` in the model builds
301 ;; `"<who>: <message>"` with no file, line or column, so any two manuscripts
302 ;; failing in `make-admonition` produce the same constant string. The
303 ;; assertion therefore catches one spelling succeeding while the other fails,
304 ;; or one failing somewhere else entirely -- it does NOT verify that the two
305 ;; failures carry the same manuscript position, because neither carries one.
306 ;;
307 ;; It is also a BREAKING CHANGE for the bare spelling, which used to be
308 ;; accepted (the `@aside` was lifted to block position and got block children).
309 ;; See `docs/prose-runs.md`.
310 (define (message-of source name)
311 (guard (exn (else (exception-message exn)))
312 (eval-document source name)))
313 (let ((bare-message
314 (message-of (string-append "#lang sigil/doc\n@article[\"T\" \"A\"]{\n"
315 "@section[\"S\" id: \"s\"]{\n"
316 "Prose then @aside{a block} then more.\n}\n}")
317 "folded.sgd"))
318 (wrapped-message
319 (message-of (string-append "#lang sigil/doc\n@article[\"T\" \"A\"]{\n"
320 "@section[\"S\" id: \"s\"]{\n"
321 "@p{Prose then @aside{a block} then more.}\n}\n}")
322 "folded-wrapped.sgd")))
323 (assert-true (string? bare-message))
324 (assert-true (contains-substring? bare-message "wrong semantic category"))
325 (assert-equal wrapped-message bare-message)
326 ;; ... and the control: this is a real failure, not two empty strings.
327 (assert-true (> (string-length bare-message) 0))))
329;; The rule's narrowness is what protects existing manuscripts:
330;; `Intro.` / `@aside{...}` / `More.` on consecutive lines is a real pattern
331;; and must stay three siblings, not one paragraph containing an admonition.
332(test "a block-boundary command between prose lines stays a sibling block"
333 (let* ((doc (eval-document
334 (string-append "#lang sigil/doc\n@article[\"T\" \"A\"]{\n"
335 "@section[\"S\" id: \"s\"]{\n"
336 "Intro prose.\n@aside{An aside.}\nMore prose.\n}\n}")
337 "boundary.sgd"))
338 (children (section-children (car (document-children doc)))))
339 (assert-equal 3 (length children))
340 (assert-true (paragraph? (car children)))
341 (assert-true (admonition? (cadr children)))
342 (assert-true (paragraph? (caddr children)))
343 (assert-equal "Intro prose." (text-content (car (paragraph-children (car children)))))
344 (assert-equal "More prose." (text-content (car (paragraph-children (caddr children)))))))
346(test "a blank line still closes a run"
347 (let* ((doc (eval-document
348 (string-append "#lang sigil/doc\n@article[\"T\" \"A\"]{\n"
349 "@section[\"S\" id: \"s\"]{\n"
350 "One with @code{a}.\n\nTwo with @code{b}.\n}\n}")
351 "blank.sgd"))
352 (children (section-children (car (document-children doc)))))
353 (assert-equal 2 (length children))
354 (assert-true (paragraph? (car children)))
355 (assert-true (paragraph? (cadr children)))
356 (assert-equal 3 (length (paragraph-children (car children))))
357 (assert-equal 3 (length (paragraph-children (cadr children))))))
359(test "prose with no commands is unchanged, value and location"
360 ;; The byte-identity property the corpus arm measures at scale, asserted here
361 ;; at the level of one paragraph: the run's paragraph and text share the
362 ;; UNTRIMMED segment range, exactly as the pre-SD-P3 lowering did.
363 ;; TWO paragraphs, deliberately: with a single one the body's only child is
364 ;; also the body's first AND last child, so `trim-body-edges` has already
365 ;; trimmed the node's range before paragraph formation ever sees it, and the
366 ;; "the range is untrimmed" property below would be untestable on that shape.
367 (let* ((source (string-append "#lang sigil/doc\n@article[\"T\" \"A\"]{\n"
368 "@section[\"S\" id: \"s\"]{\n Plain prose. \n\n Second. \n}\n}"))
369 (doc (eval-document source "plain.sgd"))
370 (paragraph (car (section-children (car (document-children doc)))))
371 (text-node (car (paragraph-children paragraph))))
372 (assert-equal 2 (length (section-children (car (document-children doc)))))
373 (assert-equal "Plain prose." (text-content text-node))
374 (assert-equal (srcloc-position (paragraph-location paragraph))
375 (srcloc-position (text-location text-node)))
376 (assert-equal (srcloc-span (paragraph-location paragraph))
377 (srcloc-span (text-location text-node)))
378 ;; untrimmed: the span covers the trailing spaces the value does not carry
379 (assert-true (> (srcloc-span (text-location text-node))
380 (string-length (text-content text-node))))))
382(test "an author expression that is not at a block boundary folds too"
383 ;; Deviation D-1, ruled 2026-08-11. The written rule named commands; the
384 ;; census found zero non-boundary `@(...)` in 74 real manuscripts, so folding
385 ;; them changes no existing document and removes an asymmetry no author could
386 ;; reconstruct.
387 (let* ((doc (eval-document-forms
388 (string-append "#lang sigil/doc\n"
389 "@(define (shout) (text \"LOUD\"))\n"
390 "@article[\"T\" \"A\"]{\n"
391 "@section[\"S\" id: \"s\"]{\n"
392 "Prose then @(shout) then more.\n}\n}")
393 "expr.sgd"))
394 (children (section-children (car (document-children doc)))))
395 (assert-equal 1 (length children))
396 (assert-true (paragraph? (car children)))
397 (assert-equal 3 (length (paragraph-children (car children))))
398 (assert-equal "LOUD" (text-content (cadr (paragraph-children (car children)))))))
400(test "values that are neither inline nor block nodes still splice"
401 ;; `list-item` is `document-node?` but not `block-node?`, so the negative test
402 ;; in `author-paragraph-run` is "no value is inline", not "every value is a
403 ;; block". Written on one line, both items are non-boundary.
404 (let* ((doc (eval-document
405 (string-append "#lang sigil/doc\n@article[\"T\" \"A\"]{\n"
406 "@items{@item{First.} @item{Second.}}\n}")
407 "items.sgd"))
408 (list-node (car (document-children doc))))
409 (assert-true (item-list? list-node))
410 (assert-equal 2 (length (item-list-items list-node)))))
412(test "an all-inline cell body is still one paragraph"
413 ;; 19 of the 21 non-boundary nodes in the measured corpus are `@code` inside
414 ;; `@cell{...}`. They worked before because `block-children` wrapped an
415 ;; all-inline body; they work now because the run folds. Same shape either
416 ;; way -- what changes is that the paragraph's location is now the RUN's span
417 ;; rather than the CELL's.
418 (let* ((doc (eval-document
419 (string-append "#lang sigil/doc\n@article[\"T\" \"A\"]{\n"
420 "@table[columns: '(left)]{\n@row{\n"
421 "@cell{Open a file with @code{C-x C-f}}\n}\n}\n}")
422 "cell.sgd"))
423 (cell (car (table-row-cells (car (table-rows (car (document-children doc)))))))
424 (paragraph (car (table-cell-children cell))))
425 (assert-equal 1 (length (table-cell-children cell)))
426 (assert-true (paragraph? paragraph))
427 (assert-equal 2 (length (paragraph-children paragraph)))
428 (assert-true (code? (cadr (paragraph-children paragraph))))
429 ;; the recorded meaning-change: the paragraph is narrower than the cell
430 (assert-true (< (srcloc-span (paragraph-location paragraph))
431 (srcloc-span (table-cell-location cell))))))