AtlatestRepositorysigil-doc
1
;;; SD-P3 -- inline commands in bare prose (review finding B4).2
;;;3
;;; The central claim is an EQUALITY: bare prose containing inline commands4
;;; produces the same tree as the `@p{...}`-wrapped spelling of the same prose.5
;;; Equality, not absence of complaint -- so the comparison is over the shipped6
;;; canonical serializer with source locations stripped, the arm prints the7
;;; denominators it compared, and a positive control proves the comparator can8
;;; SEE a difference when one exists.9
(import (sigil test) (sigil core) (sigil fs) (sigil syntax) (sigil language)10
(sigil doc) (sigil doc author)11
(sigil doc private author-runtime) (sigil doc private parser))13
(define (eval-document source name)14
(eval (parse-document-source source name 16)))15
;; The two fixtures are FILES, and they are the same artifact the gate's16
;; premise arm evaluates against a pristine pre-SD-P3 checkout. Reading them17
;; here rather than restating them keeps one source of truth: a fixture that18
;; drifts from the test asserting on it proves nothing about the other consumer.19
(define (load-fixture path)20
(let* ((source (read-file-string path))21
(header (inspect-language-source source path)))22
(list source (eval (parse-document-source source path (language-header-end header))))))23
(define (eval-document-forms source name)24
(let loop ((forms (parse-document-forms source name 16)) (last #f))25
(if (null? forms) last (loop (cdr forms) (eval (car forms))))))27
;; ------------------------------------------------------------------ helpers28
;;29
;; `document->datum` embeds every location as a `(source file pos line col span)`30
;; list. Two spellings of the same prose CANNOT have equal locations -- the31
;; `@p{` and `}` shift every later byte -- so structural equality is asserted32
;; over the datum with those lists removed, and the locations are asserted33
;; separately, against independently recomputed byte/scalar values.34
;; The "this element was a location, drop it" signal is a FRESH PAIR used as a35
;; sentinel, compared with `eq?`. Signalling with `#f` would conflate a dropped36
;; location with a datum whose value genuinely IS `#f` -- `(language #f)` on an37
;; unlabelled code block is exactly that shape -- and silently erase the38
;; difference between `(language #f)` and `(language)` on BOTH sides of the39
;; comparison, which is the kind of erasure that makes an equality weaker than40
;; it looks.41
;; And the discriminator is the LOCATION'S SHAPE, not its head symbol. `source`42
;; is also the head of `(source <url>)` on a video and `(source <text>)` on an43
;; example (traversal.sgl), so matching on the symbol alone would erase two44
;; videos' differing URLs and make them compare equal.45
(define (location-datum? value)46
(and (pair? value) (eq? (car value) 'source)47
(= (length value) 6)48
(string? (cadr value))49
(integer? (caddr value)) (integer? (cadddr value))))50
(define dropped (list 'dropped))51
(define (strip-locations value)52
(cond53
((location-datum? value) dropped)54
((pair? value)55
(let loop ((xs value) (out '()))56
(cond57
((null? xs) (reverse out))58
((not (pair? xs)) (append (reverse out) xs))59
(else60
(let ((kept (strip-locations (car xs))))61
(loop (cdr xs) (if (eq? kept dropped) out (cons kept out))))))))62
(else value)))64
;; Denominators. An equality over two trees that contain nothing passes for65
;; free, so every equality assertion below states what it actually compared.66
(define (count-tag datum tag)67
(cond68
((and (pair? datum) (eq? (car datum) tag))69
(+ 1 (apply + (map (lambda (x) (count-tag x tag)) (filter pair? (cdr datum))))))70
((pair? datum)71
(apply + (map (lambda (x) (count-tag x tag)) (filter pair? datum))))72
(else 0)))74
(define (replace-first haystack needle replacement)75
(let ((hn (string-length haystack)) (nn (string-length needle)))76
(let loop ((i 0))77
(cond ((> (+ i nn) hn) haystack)78
((string=? (substring haystack i (+ i nn)) needle)79
(string-append (substring haystack 0 i) replacement80
(substring haystack (+ i nn) hn)))81
(else (loop (+ i 1)))))))83
(define (contains-substring? haystack needle)84
(let ((hn (string-length haystack)) (nn (string-length needle)))85
(let loop ((i 0))86
(cond ((> (+ i nn) hn) #f)87
((string=? (substring haystack i (+ i nn)) needle) #t)88
(else (loop (+ i 1)))))))90
;; ------------------------------------------------------------ the fixtures91
;;92
;; The same prose twice. Everything that differs between them is a `@p{` and a93
;; `}`; `naïve` and `café` are here so the fixture has multi-byte scalars and a94
;; byte-versus-scalar difference the location arm can assert on.95
(define bare-loaded (load-fixture "test/fixtures/sd-p3-bare-prose.sgd"))96
(define wrapped-loaded (load-fixture "test/fixtures/sd-p3-wrapped.sgd"))97
(define bare (car bare-loaded))98
(define bare-doc (cadr bare-loaded))99
(define wrapped-doc (cadr wrapped-loaded))100
(define bare-datum (strip-locations (document->datum bare-doc)))101
(define wrapped-datum (strip-locations (document->datum wrapped-doc)))103
(test "bare prose with inline commands equals the @p-wrapped spelling"104
;; Denominators first: this equality must not be able to hold vacuously.105
(assert-equal 2 (length (section-children (car (document-children bare-doc)))))106
(assert-true (>= (count-tag bare-datum 'text) 6))107
(assert-equal 2 (count-tag bare-datum 'code))108
(assert-equal 1 (count-tag bare-datum 'reference))109
(assert-equal 1 (count-tag bare-datum 'emphasis))110
;; SD-P2's inline url-link node, folded by an SD-P3 run. This combination111
;; exists only after the two phases are merged and NEITHER PHASE TESTED IT.112
(assert-equal 1 (count-tag bare-datum 'url-link))113
(assert-equal 2 (count-tag bare-datum 'paragraph))114
;; ... and the same denominators on the other side, so an equality between115
;; two identically-empty trees is impossible rather than merely unlikely.116
(assert-equal (count-tag bare-datum 'text) (count-tag wrapped-datum 'text))117
(assert-equal (count-tag bare-datum 'code) (count-tag wrapped-datum 'code))118
(assert-equal bare-datum wrapped-datum)119
(display "SD_P3_EQUALITY_COMPARED text ") (display (count-tag bare-datum 'text))120
(display " code ") (display (count-tag bare-datum 'code))121
(display " reference ") (display (count-tag bare-datum 'reference))122
(display " emphasis ") (display (count-tag bare-datum 'emphasis))123
(display " url-link ") (display (count-tag bare-datum 'url-link))124
(display " paragraph ") (display (count-tag bare-datum 'paragraph))125
(newline))127
(test "the comparator can see a difference when one exists"128
;; The positive control for the arm above. Without it, `assert-equal` over129
;; `strip-locations` is consistent with a stripper that removes everything.130
;; Derived from the fixture by one substitution, so the control cannot drift131
;; away from the thing it is controlling for.132
(let* ((altered-source (replace-first bare "@code{define}" "@code{DEFINE}"))133
(altered (eval-document altered-source "altered.sgd")))134
(assert-false (string=? altered-source bare))135
(assert-false (equal? bare-datum (strip-locations (document->datum altered))))136
;; ... and the stripper is not simply deleting the content it compares:137
;; the two data differ only in that one code node's content, so the138
;; difference the comparator saw is a real one inside the kept structure.139
;; Both counts are taken from ALTERED, the value under control -- counting140
;; `bare-datum` again would restate the previous test rather than say141
;; anything about this one.142
(assert-equal (count-tag bare-datum 'code)143
(count-tag (strip-locations (document->datum altered)) 'code))144
(assert-true (> (count-tag (strip-locations (document->datum altered)) 'text) 0))))146
(test "every node of a bare-prose run carries its exact manuscript location"147
;; Independently recomputed from the fixture bytes: the byte position is the148
;; UTF-8 length of everything before the node, the span is the UTF-8 length of149
;; the node's own text, and line/column are counted in SCALARS.150
(let* ((section (car (document-children bare-doc)))151
;; BOTH paragraphs: the second carries the `@@` document escape, whose152
;; node value ("@") deliberately differs from its two-character source153
;; slice ("@@"). Walking only the first would leave the one node in the154
;; fixture whose value is not its slice unexamined.155
(children (apply append156
(map paragraph-children (section-children section))))157
(checked 0))158
(define (utf8-length s)159
(let loop ((i 0) (n 0))160
(if (= i (string-length s)) n161
(let ((c (char->integer (string-ref s i))))162
(loop (+ i 1) (+ n (cond ((<= c #x7f) 1) ((<= c #x7ff) 2)163
((<= c #xffff) 3) (else 4))))))))164
;; The precondition that stops this arm passing vacuously: the fixture has165
;; multi-byte scalars, so byte offsets and scalar offsets DIVERGE. If they166
;; agreed, an implementation that returned scalars everywhere would pass.167
(assert-true (> (utf8-length bare) (string-length bare)))168
(for-each169
(lambda (node)170
(let* ((loc (node-location node))171
(position (srcloc-position loc))172
(span (srcloc-span loc))173
;; Recover the node's source slice from the byte offsets by174
;; walking the manuscript in scalars and counting bytes.175
(slice (let loop ((i 0) (byte 0) (from #f))176
(cond177
((and (not from) (= byte position)) (loop i byte i))178
((and from (= byte (+ position span))) (substring bare from i))179
((= i (string-length bare)) (if from (substring bare from i) ""))180
(else181
(let ((c (char->integer (string-ref bare i))))182
(loop (+ i 1)183
(+ byte (cond ((<= c #x7f) 1) ((<= c #x7ff) 2)184
((<= c #xffff) 3) (else 4)))185
from)))))))186
(set! checked (+ checked 1))187
(cond188
;; A document escape's node value is ONE character and its span189
;; covers TWO -- `@@` -> `@`. That is the shape a run must not190
;; rebuild from the slice, so it is asserted explicitly rather than191
;; papered over with a looser comparison.192
((and (text? node) (= (string-length slice) 2)193
(char=? (string-ref slice 0) #\@))194
(assert-equal (string-append "@" (text-content node)) slice))195
((text? node) (assert-equal (text-content node) slice))196
((code? node) (assert-equal (string-append "@code{" (code-content node) "}") slice))197
(else (assert-true (> (string-length slice) 0))))198
;; line/column are scalar-counted and must address the slice's start199
(assert-equal (substring slice 0 1)200
(let* ((line (srcloc-line loc)) (column (srcloc-column loc)))201
(let loop ((i 0) (l 1) (c 1))202
(if (and (= l line) (= c column)) (substring bare i (+ i 1))203
(if (char=? (string-ref bare i) #\newline)204
(loop (+ i 1) (+ l 1) 1)205
(loop (+ i 1) l (+ c 1)))))))))206
children)207
;; Each paragraph spans from ITS first child to ITS last -- BOTH ends.208
;; Asserting only the start passes for a paragraph that stops at its first209
;; child, which is a span an indexer would silently truncate on.210
(for-each211
(lambda (p)212
(let* ((kids (paragraph-children p))213
(last (car (reverse kids))))214
(assert-equal (srcloc-position (node-location (car kids)))215
(srcloc-position (paragraph-location p)))216
(assert-equal (+ (srcloc-position (node-location last))217
(srcloc-span (node-location last)))218
(+ (srcloc-position (paragraph-location p))219
(srcloc-span (paragraph-location p))))220
;; ... and each run really does have more than one child, so the two221
;; assertions above are not the same assertion twice.222
(assert-true (> (length kids) 1))))223
(section-children section))224
(display "SD_P3_LOCATIONS_CHECKED ") (display checked) (display " nodes") (newline)225
(assert-true (>= checked 6))))227
(test "a run whose only value is a block node is spliced, exactly as before"228
;; `test/fixtures/minimal.sgd` writes `@tip{@p{...}}` and229
;; `@exercise[...]{@p{...}}`: the `@p` opens the body on the same line as the230
;; brace, so it is NOT at a block boundary and this phase folds it into a run.231
;; Folding it INTO a paragraph would nest a paragraph in a paragraph. The232
;; run's values decide, and one block value means "hand it back unchanged".233
(let* ((doc (eval-document234
(string-append "#lang sigil/doc\n@article[\"T\" \"A\"]{\n"235
"@aside{@p{Remember @code{Sigil}.}}\n}")236
"splice.sgd"))237
(admonition (car (document-children doc))))238
(assert-true (admonition? admonition))239
(assert-equal 1 (length (admonition-children admonition)))240
(assert-true (paragraph? (car (admonition-children admonition))))241
(assert-true (code? (cadr (paragraph-children (car (admonition-children admonition))))))))243
(test "a run that mixes inline and block values is refused with an exact location"244
;; The corpus contains zero instances of this shape, so it gets a deliberate245
;; test rather than incidental coverage.246
;;247
;; `@anchor` is the right probe: it is a BLOCK node that accepts no children,248
;; so it evaluates successfully inside a run and the run itself is what has to249
;; refuse it. A block command that also demands BLOCK CHILDREN (`@aside{…}`)250
;; never reaches this branch -- see the next test, which pins that separately.251
(let ((message252
(guard (exn (else (exception-message exn)))253
(eval-document254
(string-append "#lang sigil/doc\n@article[\"T\" \"A\"]{\n"255
"@section[\"S\" id: \"s\"]{\n"256
"Prose then @anchor[\"mid\"]{} then more prose.\n}\n}")257
"mixed.sgd"))))258
(assert-true (string? message))259
(assert-true (> (string-length message) 0))260
(display "SD_P3_MIXED_RUN_MESSAGE ") (display message) (newline)261
;; names the file, a line, a column, and the reason262
(assert-true (contains-substring? message "mixed.sgd:"))263
(assert-true (contains-substring? message "mixes inline and block content"))))265
(test "an inline url-link folds into a bare-prose run"266
;; CROSS-PHASE. SD-P2 added `@url` as an INLINE node; SD-P3 folds267
;; non-boundary commands into paragraph runs. `@url` mid-paragraph in BARE268
;; prose therefore exercises a path that exists only on the merged tree, and269
;; neither phase's own gate covered it: SD-P2's fixtures put every `@url`270
;; inside an explicit `@p{...}`, and SD-P3's predate the node entirely.271
;;272
;; The mechanism it depends on is that `url-link?` is in `inline-node?`273
;; (model.sgl), so a run containing one is all-inline and becomes a274
;; paragraph rather than a "mixes inline and block content" refusal.275
(let* ((doc (eval-document276
(string-append "#lang sigil/doc\n@article[\"T\" \"A\"]{\n"277
"@section[\"S\" id: \"s\"]{\n"278
"See @url[\"https://example.org/a\"]{the spec} and "279
"@url[\"https://example.org/b\"]{the guide} for more.\n}\n}")280
"urls.sgd"))281
(children (section-children (car (document-children doc)))))282
(assert-equal 1 (length children))283
(assert-true (paragraph? (car children)))284
;; text, url, text, url, text285
(assert-equal 5 (length (paragraph-children (car children))))286
(assert-true (url-link? (cadr (paragraph-children (car children)))))287
(assert-true (url-link? (cadddr (paragraph-children (car children)))))288
;; the space BETWEEN two inline url-links is content and must survive289
(assert-equal " and " (text-content (caddr (paragraph-children (car children)))))))291
(test "a folded command that wants block children fails exactly as @p does"292
;; A folded command's children are lowered INLINE, because that is what the293
;; overwhelmingly common case (`@code`, `@ref`, `@em`) needs. So a command294
;; that demands BLOCK children -- `@aside{…}` -- fails inside its own295
;; constructor before the run ever gets to judge its value.296
;;297
;; It is EXACTLY what the `@p{…}`-wrapped spelling of the same prose has298
;; always done, and this phase's claim is that the two spellings agree.299
;;300
;; BUT NOTE WHAT THIS ASSERTION IS AND IS NOT. `fail` in the model builds301
;; `"<who>: <message>"` with no file, line or column, so any two manuscripts302
;; failing in `make-admonition` produce the same constant string. The303
;; assertion therefore catches one spelling succeeding while the other fails,304
;; or one failing somewhere else entirely -- it does NOT verify that the two305
;; failures carry the same manuscript position, because neither carries one.306
;;307
;; It is also a BREAKING CHANGE for the bare spelling, which used to be308
;; accepted (the `@aside` was lifted to block position and got block children).309
;; See `docs/prose-runs.md`.310
(define (message-of source name)311
(guard (exn (else (exception-message exn)))312
(eval-document source name)))313
(let ((bare-message314
(message-of (string-append "#lang sigil/doc\n@article[\"T\" \"A\"]{\n"315
"@section[\"S\" id: \"s\"]{\n"316
"Prose then @aside{a block} then more.\n}\n}")317
"folded.sgd"))318
(wrapped-message319
(message-of (string-append "#lang sigil/doc\n@article[\"T\" \"A\"]{\n"320
"@section[\"S\" id: \"s\"]{\n"321
"@p{Prose then @aside{a block} then more.}\n}\n}")322
"folded-wrapped.sgd")))323
(assert-true (string? bare-message))324
(assert-true (contains-substring? bare-message "wrong semantic category"))325
(assert-equal wrapped-message bare-message)326
;; ... and the control: this is a real failure, not two empty strings.327
(assert-true (> (string-length bare-message) 0))))329
;; The rule's narrowness is what protects existing manuscripts:330
;; `Intro.` / `@aside{...}` / `More.` on consecutive lines is a real pattern331
;; and must stay three siblings, not one paragraph containing an admonition.332
(test "a block-boundary command between prose lines stays a sibling block"333
(let* ((doc (eval-document334
(string-append "#lang sigil/doc\n@article[\"T\" \"A\"]{\n"335
"@section[\"S\" id: \"s\"]{\n"336
"Intro prose.\n@aside{An aside.}\nMore prose.\n}\n}")337
"boundary.sgd"))338
(children (section-children (car (document-children doc)))))339
(assert-equal 3 (length children))340
(assert-true (paragraph? (car children)))341
(assert-true (admonition? (cadr children)))342
(assert-true (paragraph? (caddr children)))343
(assert-equal "Intro prose." (text-content (car (paragraph-children (car children)))))344
(assert-equal "More prose." (text-content (car (paragraph-children (caddr children)))))))346
(test "a blank line still closes a run"347
(let* ((doc (eval-document348
(string-append "#lang sigil/doc\n@article[\"T\" \"A\"]{\n"349
"@section[\"S\" id: \"s\"]{\n"350
"One with @code{a}.\n\nTwo with @code{b}.\n}\n}")351
"blank.sgd"))352
(children (section-children (car (document-children doc)))))353
(assert-equal 2 (length children))354
(assert-true (paragraph? (car children)))355
(assert-true (paragraph? (cadr children)))356
(assert-equal 3 (length (paragraph-children (car children))))357
(assert-equal 3 (length (paragraph-children (cadr children))))))359
(test "prose with no commands is unchanged, value and location"360
;; The byte-identity property the corpus arm measures at scale, asserted here361
;; at the level of one paragraph: the run's paragraph and text share the362
;; UNTRIMMED segment range, exactly as the pre-SD-P3 lowering did.363
;; TWO paragraphs, deliberately: with a single one the body's only child is364
;; also the body's first AND last child, so `trim-body-edges` has already365
;; trimmed the node's range before paragraph formation ever sees it, and the366
;; "the range is untrimmed" property below would be untestable on that shape.367
(let* ((source (string-append "#lang sigil/doc\n@article[\"T\" \"A\"]{\n"368
"@section[\"S\" id: \"s\"]{\n Plain prose. \n\n Second. \n}\n}"))369
(doc (eval-document source "plain.sgd"))370
(paragraph (car (section-children (car (document-children doc)))))371
(text-node (car (paragraph-children paragraph))))372
(assert-equal 2 (length (section-children (car (document-children doc)))))373
(assert-equal "Plain prose." (text-content text-node))374
(assert-equal (srcloc-position (paragraph-location paragraph))375
(srcloc-position (text-location text-node)))376
(assert-equal (srcloc-span (paragraph-location paragraph))377
(srcloc-span (text-location text-node)))378
;; untrimmed: the span covers the trailing spaces the value does not carry379
(assert-true (> (srcloc-span (text-location text-node))380
(string-length (text-content text-node))))))382
(test "an author expression that is not at a block boundary folds too"383
;; Deviation D-1, ruled 2026-08-11. The written rule named commands; the384
;; census found zero non-boundary `@(...)` in 74 real manuscripts, so folding385
;; them changes no existing document and removes an asymmetry no author could386
;; reconstruct.387
(let* ((doc (eval-document-forms388
(string-append "#lang sigil/doc\n"389
"@(define (shout) (text \"LOUD\"))\n"390
"@article[\"T\" \"A\"]{\n"391
"@section[\"S\" id: \"s\"]{\n"392
"Prose then @(shout) then more.\n}\n}")393
"expr.sgd"))394
(children (section-children (car (document-children doc)))))395
(assert-equal 1 (length children))396
(assert-true (paragraph? (car children)))397
(assert-equal 3 (length (paragraph-children (car children))))398
(assert-equal "LOUD" (text-content (cadr (paragraph-children (car children)))))))400
(test "values that are neither inline nor block nodes still splice"401
;; `list-item` is `document-node?` but not `block-node?`, so the negative test402
;; in `author-paragraph-run` is "no value is inline", not "every value is a403
;; block". Written on one line, both items are non-boundary.404
(let* ((doc (eval-document405
(string-append "#lang sigil/doc\n@article[\"T\" \"A\"]{\n"406
"@items{@item{First.} @item{Second.}}\n}")407
"items.sgd"))408
(list-node (car (document-children doc))))409
(assert-true (item-list? list-node))410
(assert-equal 2 (length (item-list-items list-node)))))412
(test "an all-inline cell body is still one paragraph"413
;; 19 of the 21 non-boundary nodes in the measured corpus are `@code` inside414
;; `@cell{...}`. They worked before because `block-children` wrapped an415
;; all-inline body; they work now because the run folds. Same shape either416
;; way -- what changes is that the paragraph's location is now the RUN's span417
;; rather than the CELL's.418
(let* ((doc (eval-document419
(string-append "#lang sigil/doc\n@article[\"T\" \"A\"]{\n"420
"@table[columns: '(left)]{\n@row{\n"421
"@cell{Open a file with @code{C-x C-f}}\n}\n}\n}")422
"cell.sgd"))423
(cell (car (table-row-cells (car (table-rows (car (document-children doc)))))))424
(paragraph (car (table-cell-children cell))))425
(assert-equal 1 (length (table-cell-children cell)))426
(assert-true (paragraph? paragraph))427
(assert-equal 2 (length (paragraph-children paragraph)))428
(assert-true (code? (cadr (paragraph-children paragraph))))429
;; the recorded meaning-change: the paragraph is narrower than the cell430
(assert-true (< (srcloc-span (paragraph-location paragraph))431
(srcloc-span (table-cell-location cell))))))