AtlatestRepositoryfolio
1
;;; F2a -- the passive inspector.2
;;;3
;;; WHAT THIS SUITE DELIBERATELY DOES NOT DO: it does not decide whether the4
;;; spans are exact. It CANNOT: every check in this file is written in the same5
;;; language, against the same descriptor, using the same coordinate arithmetic6
;;; the descriptor was built with, so it agrees with itself no matter what the7
;;; code does. Span exactness is decided by `test/f2a-span-verify.py`, which8
;;; opens the manuscript in binary mode and recomputes every offset from the9
;;; file. Taxonomy shape 13: instruments that share a blind spot are one10
;;; measurement wearing several hats.11
;;;12
;;; What this suite owns instead is BEHAVIOUR the byte verifier cannot see:13
;;; vocabulary classification, the duplicate-id contract, diagnostics and their14
;;; locations, the revision's shape validation, and one genuinely independent15
;;; cross-check -- the line/column this module computes against the line/column16
;;; the PARSER computed, two indexes built separately over the same text.17
(import (sigil core) (sigil fs) (sigil language) (sigil syntax) (sigil test)18
(sigil doc inspect) (folio doc inspect))20
(define (load-source path) (read-file-string path))22
(define zero-revision (string-append "sha256:" (make-string 64 #\0)))24
;; THE SUITE PLAYS THE CALLER. `header-end` is a parameter of the inspector,25
;; not something it scans for, because scanning would put `(sigil language)` --26
;; and through it `(sigil io)`, `(sigil socket)` and the async scheduler -- into27
;; the passive core's transitive closure. Every consumer already holds the28
;; filesystem; this is the two lines it costs them.29
(define (header-end-of source path)30
(let ((header (inspect-language-source source path)))31
(and (language-header? header) (language-header-end header))))33
(define (inspect path)34
(let ((source (load-source path)))35
(inspect-folio-source source path zero-revision (header-end-of source path))))37
(define corpus (inspect "test/fixtures/f2a-corpus.sgd"))38
(define duplicates (inspect "test/fixtures/f2a-duplicate.sgd"))39
(define malformed (inspect "test/fixtures/f2a-malformed.sgd"))40
(define headerless (inspect "test/fixtures/f2a-no-header.sgd"))41
(define bom (inspect "test/fixtures/f2a-bom.sgd"))42
(define shebang (inspect "test/fixtures/f2a-shebang.sgd"))43
(define edges (inspect "test/fixtures/f2a-argument-edges.sgd"))45
(define (all-nodes descriptor)46
(let walk ((pending (descriptor-nodes descriptor)) (out '()))47
(if (null? pending)48
(reverse out)49
(walk (append (inspected-node-children (car pending)) (cdr pending))50
(cons (car pending) out)))))52
(define (nodes-named descriptor name)53
(let loop ((rest (all-nodes descriptor)) (out '()))54
(cond ((null? rest) (reverse out))55
((eq? (inspected-node-name (car rest)) name) (loop (cdr rest) (cons (car rest) out)))56
(else (loop (cdr rest) out)))))58
(define (node-with-id descriptor id)59
(let loop ((rest (all-nodes descriptor)))60
(cond ((null? rest) #f)61
((equal? (inspected-node-id (car rest)) id) (car rest))62
(else (loop (cdr rest))))))64
(define (field-named node name)65
(let loop ((rest (inspected-node-fields node)))66
(cond ((null? rest) #f)67
((eq? (field-name (car rest)) name) (car rest))68
(else (loop (cdr rest))))))70
(define (diagnostic-codes descriptor)71
(map diagnostic-code (descriptor-diagnostics descriptor)))73
(define (count-where pred xs)74
(let loop ((rest xs) (n 0))75
(cond ((null? rest) n)76
((pred (car rest)) (loop (cdr rest) (+ n 1)))77
(else (loop (cdr rest) n)))))79
;; ---------------------------------------------------------------------------80
;; The clean corpus81
;; ---------------------------------------------------------------------------83
(test "a clean manuscript parses and reports no diagnostics"84
(assert-true (descriptor-parsed? corpus))85
(assert-equal '() (diagnostic-codes corpus)))87
(test "the descriptor records both lengths and the header end"88
;; The manuscript's first paragraph is multi-byte on purpose, so these two89
;; numbers MUST differ. If they were ever equal the whole byte-versus-scalar90
;; distinction would be untested by this corpus.91
(assert-true (> (descriptor-byte-length corpus) (descriptor-scalar-length corpus)))92
(assert-equal 16 (descriptor-header-end corpus)))94
(test "the revision is carried through unchanged"95
(assert-equal zero-revision (descriptor-revision corpus)))97
(test "the descriptor holds no callable value"98
;; A detector, not a promise: it walks every recorded field looking for a99
;; procedure. The sabotage driver plants one to prove it can report.100
(assert-equal '() (descriptor-callable-values corpus)))102
;; ---------------------------------------------------------------------------103
;; Vocabulary classification104
;; ---------------------------------------------------------------------------106
(test "every frozen folio command is classified folio, and nothing else is"107
(let loop ((names (folio-vocabulary-names)))108
(if (pair? names)109
(begin (assert-true (folio-command? (car names)))110
(loop (cdr names)))))111
(assert-equal #f (folio-command? 'sidebar))112
(assert-equal #f (folio-command? 'p))113
;; Both of sigil-doc's root commands, neither of which folio owns. `chapter`114
;; was the base language's root until D8 (`2ec9df9`) moved the base to115
;; `article` and put `chapter` behind `#lang sigil/doc/book`; this pair is116
;; checked rather than just the current one so the assertion keeps its meaning117
;; whichever side of that change a reader arrives from.118
(assert-equal #f (folio-command? 'chapter))119
(assert-equal #f (folio-command? 'article)))121
(test "an unknown author command is OPAQUE and never index-authoritative"122
;; `@sidebar["t-byte-spans" ...]` in the corpus deliberately leads with a123
;; string that COLLIDES with a real task id. If an opaque command's first124
;; positional were read as a node id, this document would report a duplicate.125
;; It does not, and that is the assertion -- the corpus's clean diagnostics126
;; above are only meaningful because this collision is in it.127
(let ((sidebars (nodes-named corpus 'sidebar)))128
(assert-equal 1 (length sidebars))129
(assert-equal 'opaque (inspected-node-vocabulary (car sidebars)))130
(assert-equal #f (inspected-node-id (car sidebars)))131
(assert-equal 'unique (descriptor-id-status corpus "t-byte-spans"))))133
(test "a folio relational node declares no id even though it leads with a string"134
(let ((related (car (nodes-named corpus 'related))))135
(assert-equal 'folio (inspected-node-vocabulary related))136
(assert-equal #f (inspected-node-id related))137
(assert-equal #f (folio-command-declares-id? 'related))138
(assert-equal #f (folio-command-declares-id? 'transclude))139
(assert-true (folio-command-declares-id? 'task))))141
(test "an author expression is recorded as source text, never as syntax"142
(let ((exprs (count-where (lambda (n) (eq? (inspected-node-kind n) 'expr))143
(all-nodes corpus))))144
(assert-true (> exprs 0)))145
(let loop ((rest (all-nodes corpus)))146
(if (pair? rest)147
(begin148
(if (eq? (inspected-node-kind (car rest)) 'expr)149
(assert-true (string? (inspected-node-source (car rest)))))150
(loop (cdr rest))))))152
;; ---------------------------------------------------------------------------153
;; Fields154
;; ---------------------------------------------------------------------------156
(test "keyword arguments become named fields with their own spans"157
;; The first version of this module tested `symbol?` for a keyword, which is158
;; #f -- a keyword is its own type. Every node still had an id and a span and159
;; the document still reported clean; there were simply no named fields160
;; anywhere. This test is what that failure would have hit.161
(let* ((task (node-with-id corpus "t-byte-spans"))162
(status (field-named task 'status))163
(priority (field-named task 'priority))164
(due (field-named task 'due)))165
(assert-true (field? status))166
(assert-equal 'ready (field-value status))167
(assert-equal 'high (field-value priority))168
(assert-equal "2026-08-20" (field-value due))))170
(test "a field's span is distinct from its node's span"171
;; The plan's requirement in one line: a task's `status` span is not the172
;; task's span. The byte verifier asserts the same thing from the file's173
;; side; this is the cheap version that fails first.174
(let* ((task (node-with-id corpus "t-byte-spans"))175
(node-span (inspected-node-span task))176
(status (field-span (field-named task 'status))))177
(assert-true (> (span-byte-start status) (span-byte-start node-span)))178
(assert-true (< (span-byte-length status) (span-byte-length node-span)))179
(assert-true (>= (span-byte-start status) (span-byte-start node-span)))180
(assert-true (<= (span-byte-end status) (span-byte-end node-span)))))182
(test "the keyword token has its own span, separate from the value"183
(let* ((task (node-with-id corpus "t-byte-spans"))184
(status (field-named task 'status)))185
(assert-true (span? (field-name-span status)))186
(assert-true (< (span-byte-end (field-name-span status))187
(span-byte-start (field-span status))))))189
(test "a single leading quote is unwrapped and nothing else is interpreted"190
(let* ((note (car (descriptor-nodes corpus)))191
(tags (field-named note 'tags))192
(related (car (nodes-named corpus 'related)))193
(kind (field-named related 'kind)))194
(assert-equal '("folio" "inspector") (field-value tags))195
(assert-equal 'depends-on (field-value kind))))197
(test "titles are surfaced only for the frozen vocabulary"198
(assert-equal "F2a inspection corpus" (inspected-node-title (car (descriptor-nodes corpus))))199
(assert-equal #f (inspected-node-title (car (nodes-named corpus 'sidebar)))))201
;; ---------------------------------------------------------------------------202
;; THE PARSER'S LINE/COLUMN vs THIS MODULE'S, COMPARED FOR REAL.203
;;204
;; This test used to assert `(integer? line)`, `(>= line 1)` and `(>= column 1)`205
;; and call itself a cross-check. Those three are STRUCTURALLY GUARANTEED --206
;; `build-index` seeds both vectors at 1 and only ever increments them -- so the207
;; only falsifiable clause in it was a corpus-size floor, and two comments in the208
;; product cited it as evidence. An adversarial review found it. It now walks209
;; the parser's OWN tree alongside the descriptor and compares `srcloc-line` and210
;; `srcloc-column` node by node.211
;;212
;; AND THE HONEST LIMIT, because the previous version's crime was overclaiming:213
;; the two indexes are NOT independent implementations. `build-index` here and214
;; `make-parser-state` in sigil-doc's parser are the same algorithm -- same215
;; `utf8-width` branches, same "newline advances the line, anything else advances216
;; the column" rule. Agreement between them catches a TRANSCRIPTION error (a217
;; wrong vector, an off-by-one, a swapped line/column) and would NOT catch a218
;; shared misconception about what a column is. The byte-level check that does219
;; not share an algorithm is `test/f2a-span-verify.py`.220
;; ---------------------------------------------------------------------------222
(define corpus-source (load-source "test/fixtures/f2a-corpus.sgd"))223
(define corpus-tree224
(inspect-document-source corpus-source225
"test/fixtures/f2a-corpus.sgd"226
(header-end-of corpus-source "test/fixtures/f2a-corpus.sgd")))228
(test "line and column agree with the parser's own srcloc, node by node"229
(let ((compared 0))230
(let walk ((mine (descriptor-nodes corpus))231
(theirs (document-tree-roots corpus-tree)))232
;; SAME SHAPE, not just same prefix: a walk that stopped at the shorter233
;; list would compare a matching prefix and pass while the descriptor had234
;; dropped every node after it.235
(assert-equal (length mine) (length theirs))236
(if (and (pair? mine) (pair? theirs))237
(let* ((m (car mine))238
(p (car theirs))239
(span (inspected-node-span m))240
(loc (tree-node-location p)))241
;; the two walks must stay in step, or the comparison is between242
;; unrelated nodes and would still mostly pass243
(assert-equal (tree-node-start p) (span-scalar-start span))244
(assert-equal (tree-node-end p) (span-scalar-end span))245
(assert-equal (srcloc-position loc) (span-byte-start span))246
(assert-equal (srcloc-span loc) (span-byte-length span))247
(assert-equal (srcloc-line loc) (span-line span))248
(assert-equal (srcloc-column loc) (span-column span))249
(assert-equal (tree-node-end-line p) (span-end-line span))250
(assert-equal (tree-node-end-column p) (span-end-column span))251
(set! compared (+ compared 1))252
(walk (inspected-node-children m) (tree-node-children p))253
(walk (cdr mine) (cdr theirs)))))254
;; both walks must have consumed the same shape, and a real amount of it255
(assert-true (> compared 15))))257
;; ---------------------------------------------------------------------------258
;; Duplicate ids -- the contract259
;; ---------------------------------------------------------------------------261
(test "a duplicate id keeps BOTH nodes"262
(assert-true (descriptor-parsed? duplicates))263
(assert-equal 2 (count-where (lambda (n) (equal? (inspected-node-id n) "t-7681"))264
(all-nodes duplicates)))265
(assert-equal 'ambiguous (descriptor-id-status duplicates "t-7681")))267
(test "a duplicate id does not poison the rest of the document"268
(assert-equal 'unique (descriptor-id-status duplicates "t-unique"))269
(assert-true (inspected-node? (descriptor-node-by-id duplicates "t-unique"))))271
(test "an absent id and an ambiguous id are DIFFERENT answers"272
;; The whole reason `descriptor-id-status` exists. A lookup returning #f for273
;; both is taxonomy shape 17: a caller checking only for #f treats a duplicated274
;; id as missing and mints a second one.275
(assert-equal 'absent (descriptor-id-status duplicates "t-does-not-exist"))276
(assert-equal #f (descriptor-node-by-id duplicates "t-does-not-exist"))277
(assert-equal 'ambiguous (descriptor-id-status duplicates "t-7681")))279
(test "resolving an ambiguous id is REFUSED, not resolved by scan order"280
(assert-error (descriptor-node-by-id duplicates "t-7681")))282
(test "every occurrence of a duplicate id gets its own located diagnostic"283
(let* ((ds (descriptor-diagnostics duplicates))284
(dups (let loop ((rest ds) (out '()))285
(cond ((null? rest) (reverse out))286
((eq? (diagnostic-code (car rest)) 'duplicate-node-id)287
(loop (cdr rest) (cons (car rest) out)))288
(else (loop (cdr rest) out))))))289
;; TWO, not one: neither occurrence is privileged, not even in the report.290
(assert-equal 2 (length dups))291
(let loop ((rest dups))292
(if (pair? rest)293
(begin294
(assert-equal 'error (diagnostic-severity (car rest)))295
(assert-true (span? (diagnostic-span (car rest))))296
(assert-true (>= (span-line (diagnostic-span (car rest))) 1))297
(loop (cdr rest)))))298
;; ...and they are at DIFFERENT positions.299
(assert-true (not (= (span-byte-start (diagnostic-span (car dups)))300
(span-byte-start (diagnostic-span (cadr dups))))))))302
;; ---------------------------------------------------------------------------303
;; Argument edge cases -- all three found by adversarial review, none of which304
;; any fixture reached before it.305
;; ---------------------------------------------------------------------------307
(test "an empty-string id is not an id"308
;; `""` is truthy, so it sailed past the missing-id check and would have309
;; reached F4a's graph tables as a node id.310
(assert-true (descriptor-parsed? edges))311
(let ((task (car (nodes-named edges 'task))))312
(assert-equal #f (inspected-node-id task)))313
(assert-true (memq 'missing-node-id (diagnostic-codes edges))))315
(test "a keyword with no value is reported, not silently reclassified"316
(assert-true (memq 'dangling-keyword (diagnostic-codes edges)))317
(let* ((decision (car (nodes-named edges 'decision)))318
(dangling (field-named decision '|dangling-keyword|)))319
(assert-true (field? dangling))320
;; `(string->symbol "status:")`, NOT `'status:` -- the latter READS AS A321
;; KEYWORD, which is precisely the type distinction this whole code path322
;; exists for, and comparing against it silently compares two different types.323
(assert-equal (string->symbol "status:") (field-value dangling))324
;; it must NOT have been counted as the node's first positional, which is325
;; what silently cost such a node its id326
(assert-equal "d-dangling" (inspected-node-id decision))))328
(test "every diagnostic on this fixture names a position"329
(let loop ((ds (descriptor-diagnostics edges)))330
(if (pair? ds)331
(begin332
(assert-true (span? (diagnostic-span (car ds))))333
(assert-true (>= (span-line (diagnostic-span (car ds))) 1))334
(loop (cdr ds))))))336
(test "no manuscript yields a callable value"337
;; RETITLED. This asserts an ABSENCE over real manuscripts and nothing more: it338
;; cannot go red as written, because the record constructors are unexported and339
;; `syntax->datum` cannot produce a procedure from any source text. Its previous340
;; title claimed it detected a NESTED callable, which it never exercised.341
;;342
;; The nested walk is proved where it can be: sabotage leg S12 plants343
;; `(list 1 (lambda (x) x))` in the product and requires the verifier to refuse344
;; the descriptor as not inert. An absence assertion here plus a sabotage there345
;; is the pair; either alone is half an argument.346
(assert-equal '() (descriptor-callable-values edges))347
(assert-equal '() (descriptor-callable-values corpus)))349
;; ---------------------------------------------------------------------------350
;; Diagnostics351
;; ---------------------------------------------------------------------------353
(test "a malformed manuscript yields a located diagnostic, not an exception"354
(assert-equal #f (descriptor-parsed? malformed))355
(assert-equal '(parse-error) (diagnostic-codes malformed))356
(let ((d (car (descriptor-diagnostics malformed))))357
(assert-equal 'error (diagnostic-severity d))358
(assert-equal "test/fixtures/f2a-malformed.sgd" (diagnostic-source-name d))359
;; LOCATED is the assertion. `parse-error-unlocated` is a different code on360
;; purpose, so a change in sigil-doc's message format degrades loudly instead361
;; of turning every diagnostic in the estate into a positionless one.362
(assert-true (span? (diagnostic-span d)))363
(assert-true (>= (span-line (diagnostic-span d)) 1))364
(assert-true (>= (span-byte-start (diagnostic-span d)) 0))))366
(test "a malformed manuscript still carries its identity"367
;; It is read-only, not invisible: F3 needs a name and a revision to record368
;; that it refused.369
(assert-equal zero-revision (descriptor-revision malformed))370
(assert-true (> (descriptor-byte-length malformed) 0)))372
(test "a source with no #lang header is refused with its own code"373
(assert-equal #f (descriptor-parsed? headerless))374
(assert-equal '(no-language-header) (diagnostic-codes headerless))375
(assert-true (span? (diagnostic-span (car (descriptor-diagnostics headerless))))))377
(test "a diagnostic renders with its file and position"378
(let ((rendered (diagnostic->string (car (descriptor-diagnostics malformed)))))379
(assert-true (string? rendered))380
(assert-true (> (string-length rendered) 20))))382
;; ---------------------------------------------------------------------------383
;; BOM and shebang384
;; ---------------------------------------------------------------------------386
(test "a BOM shifts bytes without shifting scalars"387
;; A BOM is ONE scalar and THREE bytes, so the very first node in the file388
;; already has a byte offset that differs from its scalar offset. This is the389
;; smallest possible manuscript that can tell the two coordinate systems apart.390
(assert-true (descriptor-parsed? bom))391
(assert-equal '() (diagnostic-codes bom))392
(let ((root (car (descriptor-nodes bom))))393
(assert-equal 2 (- (span-byte-start (inspected-node-span root))394
(span-scalar-start (inspected-node-span root))))))396
(test "a shebang line before the header is inspected normally"397
(assert-true (descriptor-parsed? shebang))398
(assert-equal '() (diagnostic-codes shebang))399
(assert-equal "topics/f2a-shebang" (inspected-node-id (car (descriptor-nodes shebang)))))401
;; ---------------------------------------------------------------------------402
;; The revision is an INPUT, and its shape is a boundary403
;; ---------------------------------------------------------------------------405
(test "the revision shape is validated"406
(assert-true (valid-revision? (string-append "sha256:" (make-string 64 #\a))))407
(assert-equal #f (valid-revision? "sha256:tooshort"))408
(assert-equal #f (valid-revision? (make-string 71 #\a)))409
(assert-equal #f (valid-revision? 42))410
(assert-equal #f (valid-revision? (string-append "sha256:" (make-string 63 #\a))))411
(assert-equal #f (valid-revision? (string-append "sha256:" (make-string 65 #\a)))))413
(test "uppercase hex is refused rather than folded"414
;; F3 compares revisions for equality. A comparison that is case-sensitive415
;; over values that are not is a stale-write check that fails open on some416
;; clients and not others, so there is exactly one accepted spelling.417
(assert-equal #f (valid-revision? (string-append "sha256:" (make-string 64 #\A)))))419
(test "a malformed revision is a CALLER error and raises"420
;; Asymmetric on purpose: a broken manuscript is data this function exists to421
;; describe, so it becomes a diagnostic; a revision that is not a revision is422
;; a bug upstream, and filing it under the document's name would hide it.423
(assert-error (inspect-folio-source "#lang sigil/doc\n" "x.sgd" "not-a-revision" 16)))425
;; ---------------------------------------------------------------------------426
;; The rest of the exported surface.427
;;428
;; Adversarial review found these exported and asserted by NOTHING -- not the429
;; suite, not the manifest probe, not the witness probe. An export nobody430
;; exercises is a promise nobody has checked, and F4a and F2b consume this431
;; surface.432
;; ---------------------------------------------------------------------------434
(test "descriptor-node-count counts every node, nested included"435
;; Derived on both sides rather than pinned: the count must equal the length of436
;; the flattened walk, whatever the corpus grows into.437
(assert-equal (length (all-nodes corpus)) (descriptor-node-count corpus))438
(assert-true (> (descriptor-node-count corpus) 20)))440
(test "descriptor-ids enumerates exactly the ids the nodes declare"441
(let ((declared (let loop ((rest (all-nodes corpus)) (out (quote ())))442
(cond ((null? rest) (reverse out))443
((inspected-node-id (car rest))444
(loop (cdr rest) (cons (inspected-node-id (car rest)) out)))445
(else (loop (cdr rest) out))))))446
(assert-equal (length declared) (length (descriptor-ids corpus)))447
(let loop ((rest declared))448
(if (pair? rest)449
(begin (assert-true (memq (descriptor-id-status corpus (car rest))450
(quote (unique ambiguous))))451
(loop (cdr rest)))))))453
(test "body-kind is recorded for commands and #f for prose"454
(let ((note (car (descriptor-nodes corpus))))455
(assert-equal (quote named) (inspected-node-body-kind note)))456
(let ((task (node-with-id corpus "t-byte-spans")))457
(assert-equal (quote brace) (inspected-node-body-kind task)))458
(let loop ((rest (all-nodes corpus)))459
(if (pair? rest)460
(begin461
(if (eq? (inspected-node-kind (car rest)) (quote text))462
(assert-equal #f (inspected-node-body-kind (car rest))))463
(loop (cdr rest))))))465
(test "a raw @code body's folded text is NOT its content, but its source slice is"466
;; SD-P4 is explicit that `tree-node-text` folds newlines even inside a raw467
;; `@code` body, so the authoritative text is always the source slice named by468
;; the span. The module says so; nothing tested it until now.469
(let* ((code (car (nodes-named corpus (quote code))))470
(child (car (inspected-node-children code))))471
(assert-equal (quote text) (inspected-node-kind child))472
;; the raw slice keeps its newlines...473
(assert-true (> (string-length (inspected-node-source child)) 0))474
;; ...and the folded value differs from it on a multi-line body475
(assert-true (not (equal? (inspected-node-text child)476
(inspected-node-source child))))))478
;; ---------------------------------------------------------------------------479
;; Ownership480
;; ---------------------------------------------------------------------------482
(test "aggregate ownership is mutation-proof"483
(let* ((nodes (descriptor-nodes corpus))484
(root (car nodes))485
(fields (inspected-node-fields root)))486
(set-car! nodes 'clobbered)487
(set-car! fields 'clobbered)488
(assert-true (inspected-node? (car (descriptor-nodes corpus))))489
(assert-true (field? (car (inspected-node-fields root))))))