AtlatestRepositoryfolio

folio / tree / toolsm0_convert.py

1#!/usr/bin/env python3
2"""M0: a deterministic Markdown -> `folio/doc` converter for the constrained Folio dialect.
3
4WHAT THIS IS, AND WHAT IT IS DELIBERATELY NOT
5---------------------------------------------
6This is MIGRATION TOOLING, not product infrastructure (ruled 2026-08-12). It reads
7Markdown sources and writes `.sgd` manuscripts plus one audit record per file. It
8NEVER writes into the source tree, never deletes, and never renames. M0 converts
9nothing anyone relies on; M3 is the only phase that changes what is canonical.
11WHY PYTHON, when the reader that consumes the output is Sigil (ruled 2026-08-12)
12-------------------------------------------------------------------------------
13Two reasons, and the second is the load-bearing one:
15 1. F0's frozen census tooling is Python, and this module REUSES its `classify`,
16 `comment_mask`, `indent_width` and task-block semantics rather than restating
17 them. The golden corpus was measured with that grammar; a second grammar here
18 could drift from the evidence it is supposed to reconcile against.
20 2. A Python converter validated by a Sigil reader is TWO INDEPENDENT
21 IMPLEMENTATIONS. A Sigil converter checked by a Sigil reader could share a
22 misconception invisibly, which the estate's own rule forbids: never validate a
23 parser exclusively with values produced by the same parser.
25THE ONE-BIT CONTRACT (ruled 2026-08-12)
26---------------------------------------
27The source dialect has NO task status field. `grep '"status"' src/folio/*.sgl` finds
28it only in `web.sgl`, as a UI filter over open/done/all derived from the checkbox.
29The source carries EXACTLY ONE BIT. `make-task` demands a status from six values, so
30mapping one bit onto six would be invention in four directions. This converter
31therefore restricts the codomain so the map stays a bijection on the bit that exists:
33 - [ ] -> status: 'ready - [x] / - [X] -> status: 'done
35That is a DECLARED CONTRACT, not an inference, and it is stated in every audit record
36so a reader of the output knows the codomain was restricted deliberately rather than
37by omission.
39HEADINGS ARE CONVERTED FAITHFULLY AND NEVER RENAMED (ruled 2026-08-12, reversal)
40--------------------------------------------------------------------------------
41An earlier direction had this converter disambiguate repeated heading slugs. Fable
42reversed it: a migration must not rename what it can instead report. Repeated slugs
43are WARNINGS WITH LOCATIONS, counted in the denominator. The disambiguator survives
44behind `--disambiguate-anchors` for anyone who wants it, and is never the default.
46This assumes core anchor nodes are not semantically addressable (F4's open decision).
47Nothing emitted here targets a section with `related[node:]`, so an F4 reversal that
48re-merges the namespaces for addressing does not invalidate this output.
49"""
51from __future__ import annotations
53import argparse
54import json
55import re
56import secrets
57import sys
59# Importing a sibling module writes .pyc INTO tools/. An adversarial review found
60# `tools/__pycache__/` and correctly called the read-only claim false; `.gitignore`
61# was a workaround, not a fix. Set before any sibling import.
62sys.dont_write_bytecode = True
63from pathlib import Path
65sys.path.insert(0, str(Path(__file__).resolve().parent))
67# Reused from F0 rather than restated -- see the module docstring.
68from f0_inventory import ( # noqa: E402
69 ID_RE,
70 task_blocks,
71 KEY_RE,
72 LINK_RE,
73 META_RE,
74 TASK_RE,
75 classify,
76 comment_mask,
77 indent_width,
78 sha256,
81CONVERTER_VERSION = "m0.1"
84class IdAllocator:
85 """Random id ASSIGNMENT through a persistent table. NOT derivation.
87 D4 forbids ids DERIVED from title, heading, path or rowid, because a derived id
88 is unstable under edit and collides. Random ASSIGNMENT is a different act -- the
89 same one `folio_task-add` performs at creation today -- and an id-less task has
90 no identity to lose. Ruled 2026-08-12.
92 The table is what keeps repeated runs deterministic without the id being
93 content-derived: the first run allocates, every later run REUSES. Path and title
94 are the LOOKUP KEY only; no bit of the id comes from them.
96 Shape is folio's own: `t-` + 6 hex. `src/folio/id.sgl` records why six and not
97 four -- four is a 65,536 space which predicted ~12.5 collisions at this corpus
98 size, and the estate had 14.
99 """
101 def __init__(self, path: Path | None) -> None:
102 self.path = path
103 self.table: dict[str, dict] = {}
104 self.dirty = False
105 self.minted_this_run = 0
106 if path and path.exists():
107 self.table = json.loads(path.read_text())["allocations"]
109 def known(self) -> set[str]:
110 return {v["id"] for v in self.table.values()}
112 def allocate(self, key: str, rel: str, title: str, line: int, taken: set[str]) -> str:
113 if key in self.table:
114 return self.table[key]["id"]
115 for _ in range(1000):
116 candidate = "t-" + secrets.token_hex(3)
117 if candidate not in taken and candidate not in self.known():
118 break
119 else: # pragma: no cover -- 16.7M space; a thousand collisions is a bug
120 raise SourceError("id allocation failed to find a free id in 1000 attempts")
121 self.table[key] = {"id": candidate, "path": rel, "title": title, "line": line}
122 self.dirty = True
123 self.minted_this_run += 1
124 return candidate
126 def save(self) -> None:
127 if not (self.path and self.dirty):
128 return
129 self.path.parent.mkdir(parents=True, exist_ok=True)
130 self.path.write_text(
131 json.dumps({"version": 1, "note": (
132 "M0 id allocations for source tasks that carry no id. Random "
133 "ASSIGNMENT, not derivation: path and title are the lookup key only. "
134 "Commit this file -- it is what makes repeated runs reuse rather than "
135 "re-mint, and it is the record of a naming event."
136 ), "allocations": self.table}, indent=2, sort_keys=True) + "\n",
137 encoding="utf-8",
138 )
141class SourceError(Exception):
142 """A source file this converter cannot read. Isolated per file, never fatal."""
144# The file classes M0 converts. Every other class F0 knows -- `auxiliary`,
145# `generated-index`, `ignored-hidden`, `unclassified` -- is SKIPPED AND NAMED in the
146# summary, never dropped silently.
147CONVERTIBLE_KINDS = frozenset(
148 {"inbox", "note", "project", "archived-note", "archived-project"}
151# --------------------------------------------------------------------------
152# The frozen vocabulary's field sets, as data.
154# These are the fields each record CAN carry, transcribed from
155# `src/folio/doc/model.sgl`'s `define-struct` forms. A source metadata key that is
156# not in the relevant set has NOWHERE TO GO -- it is not droppable silently, it is a
157# warning plus an audit-record entry. That list is the input to the vocabulary
158# decision the leader filed for before M1.
159# --------------------------------------------------------------------------
160TASK_FIELDS = {"id", "title", "status", "priority", "tags", "due"}
161NOTE_FIELDS = {"id", "title", "tags"}
162PROJECT_FIELDS = {"id", "title", "status", "tags"}
164TASK_PRIORITIES = {"high", "medium", "low"} # `normal` renamed to `medium` 2026-08-12
165CONTAINER_STATUSES = {"planned", "active", "paused", "done", "cancelled"}
167# `[a-z0-9]+(-[a-z0-9]+)*` -- model.sgl:181. NOT a hex assumption: the estate holds
168# five hand-authored slug ids (t-wasm-lib-bundle-prune, three t-016-*, t-folio-dup-h1)
169# that a `t-[0-9a-f]+` capture would truncate silently.
170NODE_ID_RE = re.compile(r"^[a-z0-9]+(-[a-z0-9]+)*$")
171SEGMENT_RE = re.compile(r"^[a-z0-9]+([-.][a-z0-9]+)*$")
172DOC_ID_RE = re.compile(r"^[a-z0-9]+([-.][a-z0-9]+)*(/[a-z0-9]+([-.][a-z0-9]+)*)*$")
173DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
175FENCE_RE = re.compile(r"^(\s*)(`{3,}|~{3,})\s*([A-Za-z0-9_+-]*)\s*$")
176HEADING_RE = re.compile(r"^(#{1,6})\s+(.*?)\s*$")
177BULLET_RE = re.compile(r"^(\s*)[-*+]\s+(.*)$")
178LOG_RE = re.compile(r"^\s*[-*+]\s+\*\*(\d{4}-\d{2}-\d{2})\*\*\s*:?\s*(.*)$")
179TABLE_SEP_RE = re.compile(r"^\s*\|?[\s:-]*\|[\s|:-]*$")
182# ==========================================================================
183# Warnings. ONE SINK, so the denominator has ONE SOURCE.
184# ==========================================================================
185class Warn:
186 """The warning vocabulary, and the TOTAL/PARTIAL declaration per rule.
188 Every source construct dispatches through exactly one lowering rule. A rule is
189 TOTAL (a bijection or an injection into the frozen vocabulary) or PARTIAL (it
190 needs a choice the source does not determine). THE DENOMINATOR IS THE COUNT OF
191 PARTIAL FIRINGS, and it is exact because dispatch is exhaustive: the default arm
192 is `unrecognized-construct`, itself a warning class, so nothing falls through
193 silently.
194 """
196 W1 = "status-supplied-by-contract"
197 W2 = "task-metadata-key-has-no-frozen-field"
198 W3 = "frontmatter-key-has-no-frozen-field"
199 W4 = "ambiguous-link-target"
200 W5 = "inline-relation-hoisted"
201 W6 = "unrecognized-construct"
202 W7 = "id-present-but-not-visible"
203 W8 = "duplicate-id-in-source"
204 W9 = "value-outside-frozen-grammar"
205 W10 = "mixed-list-split"
206 W11 = "duplicate-anchor-slug"
207 W12 = "archived-source-address-derived"
208 W13 = "title-derived-from-address"
209 W14 = "code-language-supplied"
210 W15 = "table-row-width-normalized"
211 W16 = "empty-section-dropped"
212 W17 = "buried-id-in-source"
213 W18 = "task-id-minted"
214 W19 = "provenance-key-not-a-relation"
215 W20 = "container-status-mapped"
217 ALL = [W1, W2, W3, W4, W5, W6, W7, W8, W9, W10, W11, W12, W13, W14, W15, W16,
218 W17, W18, W19, W20]
221class Report:
222 def __init__(self) -> None:
223 self.warnings: list[dict] = []
224 self.partial_firings = 0
226 def warn(self, code: str, line: int, detail: str, **extra) -> None:
227 if code not in Warn.ALL: # a class the vocabulary does not know is a bug, loudly
228 raise AssertionError(f"unknown warning class {code!r}")
229 entry = {"code": code, "line": line, "detail": detail}
230 entry.update(extra)
231 self.warnings.append(entry)
232 self.partial_firings += 1
234 def counts(self) -> dict[str, int]:
235 # EVERY class is present with an explicit count, including zero. A class
236 # that is absent from the report is indistinguishable from one that fired
237 # zero times, and that difference is exactly what sabotage S4 attacks.
238 out = {code: 0 for code in Warn.ALL}
239 for w in self.warnings:
240 out[w["code"]] += 1
241 return out
244# ==========================================================================
245# Emission primitives
246# ==========================================================================
247def esc(text: str) -> str:
248 """Escape the document escapes: `@@ @{ @}` (parser.sgl:492).
250 `[` and `]` are escapable too but are not special inside a brace body, so they
251 are left alone -- escaping them would add bytes that carry no meaning.
252 """
253 return text.replace("@", "@@").replace("{", "@{").replace("}", "@}")
256def sstr(value: str) -> str:
257 """A Sigil string literal."""
258 return '"' + value.replace("\\", "\\\\").replace('"', '\\"') + '"'
261def slist(values: list[str]) -> str:
262 """A quoted list of strings: `'("a" "b")`, and `'()` when empty."""
263 if not values:
264 return "'()"
265 return "'(" + " ".join(sstr(v) for v in values) + ")"
268def sym(value: str) -> str:
269 """A QUOTED symbol. Measured, not stylistic: the parser splices argument syntax
270 into the lowered call unchanged, so a bare `active` is an unbound variable."""
271 return "'" + value
274def slug(text: str) -> str:
275 """A heading's anchor slug. Deterministic and lossy-by-design; collisions are
276 REPORTED (W11), not renamed."""
277 out = []
278 for ch in text.lower():
279 if ch.isalnum() and ch.isascii():
280 out.append(ch)
281 elif out and out[-1] != "-":
282 out.append("-")
283 return "".join(out).strip("-") or "section"
286# ==========================================================================
287# Source model
288# ==========================================================================
289class Source:
290 """A Markdown file, lexed into lines with MASKS.
292 The masks decide what is NOT a construct, and they own that decision so the
293 parse stage never has to. This is where `{id: t-commented}` (HTML comment),
294 `{id: t-fencequote}` (fenced code) and `{id: t-bodyquote}` (inline code on a
295 body line) are held out of the visible population.
296 """
298 def __init__(self, rel: str, data: bytes) -> None:
299 self.rel = rel
300 self.data = data
301 self.sha256 = sha256(data)
302 try:
303 self.text = data.decode("utf-8")
304 except UnicodeDecodeError as exc:
305 # A real source condition, not a crash. `m0_vocabulary_gap` already
306 # handled this; the converter did not, so one bad byte anywhere aborted
307 # the entire corpus.
308 raise SourceError(f"{rel}: not valid UTF-8: {exc}") from exc
309 self.lines = self.text.split("\n")
310 self.masked = comment_mask(self.lines)
311 self.kind, self.owner = classify(rel)
313 self.fence_lines: set[int] = set()
314 self.frontmatter: dict[str, object] = {}
315 self.frontmatter_lines: set[int] = set()
316 self.body_start = 0
317 self._scan_fences()
318 self._scan_frontmatter()
320 def _scan_fences(self) -> None:
321 open_fence = None
322 for i, line in enumerate(self.masked):
323 m = FENCE_RE.match(line)
324 if open_fence is None:
325 if m:
326 open_fence = (i, m.group(2)[0], len(m.group(2)))
327 self.fence_lines.add(i)
328 else:
329 self.fence_lines.add(i)
330 if m and m.group(2)[0] == open_fence[1] and len(m.group(2)) >= open_fence[2]:
331 open_fence = None
332 # An unterminated fence is a real source condition, not a crash.
333 self.unterminated_fence = open_fence[0] + 1 if open_fence else None
335 def _scan_frontmatter(self) -> None:
336 if not self.lines or self.lines[0] != "---":
337 return
338 try:
339 end = self.lines[1:].index("---") + 1
340 except ValueError:
341 self.malformed_frontmatter = True
342 return
343 self.malformed_frontmatter = False
344 self.frontmatter_lines = set(range(0, end + 1))
345 self.body_start = end + 1
346 key = None
347 for line in self.lines[1:end]:
348 if not line.strip():
349 continue
350 if line.lstrip().startswith("- ") and key:
351 self.frontmatter.setdefault(key, [])
352 if isinstance(self.frontmatter[key], list):
353 self.frontmatter[key].append(line.lstrip()[2:].strip())
354 continue
355 if ":" in line and not line.startswith((" ", "\t")):
356 key, _, value = line.partition(":")
357 key = key.strip()
358 value = value.strip()
359 self.frontmatter[key] = value if value else []
361 def in_fence(self, i: int) -> bool:
362 return i in self.fence_lines
364 def document_id(self) -> str:
365 """`notes/topics/foo.md` -> `topics/foo`; `projects/bar.md` -> `projects/bar`;
366 `inbox.md` -> `inbox`. Matches the addresses `f1b-note.sgd` writes."""
367 rel = self.rel[:-3] if self.rel.endswith(".md") else self.rel
368 if rel.startswith("notes/"):
369 rel = rel[len("notes/"):]
370 return rel
373def parse_metadata(meta: str) -> list[tuple[str, str]]:
374 """`{key: val, key: val}` -> ordered pairs.
376 Splitting on `,` is the source dialect's own rule (`task.sgl:298-315`). A `tags`
377 value is SPACE-separated WITHIN its comma-delimited field; conflating the two
378 separators is sabotage S2 and is exactly what this function must not do.
379 """
380 pairs = []
381 for part in meta.split(","):
382 if ":" not in part:
383 continue
384 key, _, value = part.partition(":")
385 pairs.append((key.strip(), value.strip()))
386 return pairs
389# ==========================================================================
390# The converter
391# ==========================================================================
392class Converter:
393 def __init__(self, disambiguate_anchors: bool = False,
394 allocator: "IdAllocator | None" = None,
395 conversion_date: str | None = None) -> None:
396 self.disambiguate_anchors = disambiguate_anchors
397 # An INPUT, never a clock read. `make-update` needs a timestamp for the
398 # archival stamp, and the source does not carry one -- but reading the clock
399 # inside the converter would make two runs differ, which is precisely the
400 # property the gate's determinism arm exists to hold.
401 self.conversion_date = conversion_date
402 self.allocator = allocator or IdAllocator(None)
403 # Corpus-wide, not per-file: a minted id must not collide with an id used
404 # anywhere in the corpus, including files converted later in the same run.
405 self.all_known_ids: set[str] = set()
407 def convert(self, src: Source) -> tuple[str, dict]:
408 rep = Report()
409 self.rep = rep
410 self.src = src
411 self.seen_ids: dict[str, list[int]] = {}
412 self.seen_slugs: dict[str, list[int]] = {}
413 self.edges: list[dict] = []
414 self.visible_tasks: list[dict] = []
415 self.comment_lines: list[dict] = []
416 self.minted_ids: list[dict] = []
417 self.residue: dict[str, str] = {}
418 # EVERY BODY LINE MUST BE ACCOUNTED FOR.
419 # Sabotage S1 (drop an indented task-body line) walked straight past a gate
420 # whose accounting covered IDS ONLY: a converter that always drops the line
421 # is perfectly deterministic, emits valid .sgd, and loses content silently.
422 # Ids are not content. This set is what makes "nothing was dropped" checkable.
423 self.consumed: set[int] = set()
424 self.tags_emitted = 0
425 self.tags_seen = 0
427 self._account_ids()
428 self._detect_mixed_lists()
430 body = self._parse_body()
431 self._collect_lines(body)
432 root = self._emit_root(body)
434 audit = self._audit(rep)
435 return root, audit
437 # ---------------------------------------------------------------- ids
438 def _account_ids(self) -> None:
439 """Classify EVERY `t-*` occurrence in the file into visible / not-visible /
440 unreachable. Population stated, gap accounted for."""
441 self.id_population: list[dict] = []
442 # A SOURCE BURIAL is an id inside a task block but NOT on its header line.
443 # F0 measured 8 genuine ones in the estate (1500 ids on 1497 lines; 9 off
444 # a header line, of which 1 was inside a terminated HTML comment). This is
445 # the population the plan's "no buried item" gate arm is actually about.
446 self.buried: list[dict] = []
447 block_span = {}
448 for blk in task_blocks(self.src.masked):
449 for ln in range(blk["start"], blk["end"] + 1):
450 block_span[ln] = blk["start"]
451 visible_lines = set()
452 for i, line in enumerate(self.src.masked):
453 if self.src.in_fence(i) or i in self.src.frontmatter_lines:
454 continue
455 if TASK_RE.match(line):
456 visible_lines.add(i)
458 # THE POPULATION IS SCANNED FROM THE RAW BYTES, NOT FROM THE MASK.
459 #
460 # This was a real defect, caught by the pressure corpus on the first run:
461 # scanning `masked` meant `comment_mask` had already blanked the comment
462 # text, so `t-commented` and `t-comment2` were invisible to the accounting
463 # and the population came out 7 where the source holds 9. An accounting
464 # arm can only be green against a population it can SEE, so the mask must
465 # CLASSIFY the occurrences rather than decide which ones exist.
466 #
467 # Task-body lines are consumed by their header, so a checkbox-looking line
468 # inside a body is NOT a header. Positional context decides.
469 for i, line in enumerate(self.src.lines):
470 masked_line = self.src.masked[i]
471 for m in ID_RE.finditer(line):
472 tid = m.group(1)
473 # A position blanked by the comment mask but not blank in the source
474 # is inside an HTML comment. Checked per-occurrence, not per-line:
475 # a line can carry a comment and live prose at once.
476 in_comment = (
477 m.start() < len(masked_line)
478 and masked_line[m.start()] != line[m.start()]
479 )
480 cls = "visible" if (i in visible_lines and not in_comment) else "not-visible"
481 reason = None
482 if cls != "visible":
483 if in_comment:
484 reason = "html-comment"
485 elif self.src.in_fence(i):
486 reason = "fenced-code"
487 elif "`" in line:
488 reason = "inline-code"
489 else:
490 reason = "loose-in-prose"
491 cls = "unreachable"
492 buried_in = None
493 # A BURIAL is an id that reads as real metadata but sits on the
494 # wrong line. An id inside a fence, inline code, or a comment is
495 # QUOTED TEXT, not metadata that landed badly -- F0 made the same
496 # distinction when it counted 9 off-header ids and called only 8
497 # genuine, excluding the one inside a terminated HTML comment.
498 # Without this, the pressure corpus reports 2 burials for
499 # `t-bodyquote` and `t-fencequote`, which are deliberately quoted.
500 if reason == "loose-in-prose":
501 owner = block_span.get(i + 1)
502 if owner is not None and owner != i + 1:
503 buried_in = owner
504 self.buried.append(
505 {"id": tid, "line": i + 1, "task_header_line": owner}
506 )
507 self.rep.warn(
508 Warn.W17, i + 1,
509 f"id {tid} sits inside the task block opening at line "
510 f"{owner} but NOT on its header line; the source has "
511 "buried it and this converter mints no node for it",
512 id=tid, task_header_line=owner,
513 )
514 self.id_population.append(
515 {"id": tid, "line": i + 1, "class": cls, "reason": reason,
516 "buried_in_task_at": buried_in}
517 )
518 if cls != "visible":
519 self.rep.warn(
520 Warn.W7, i + 1,
521 f"id {tid} is present but not a visible task ({reason}); "
522 "it is preserved as literal text and mints no node",
523 id=tid, reason=reason,
524 )
525 self.seen_ids.setdefault(tid, []).append(i + 1)
526 self.all_known_ids.add(tid)
528 for tid, lines in sorted(self.seen_ids.items()):
529 if len(lines) > 1:
530 self.rep.warn(
531 Warn.W8, lines[0],
532 f"id {tid} occurs {len(lines)} times in this source, at lines "
533 + ", ".join(str(n) for n in lines)
534 + "; the source already contains this duplicate",
535 id=tid, lines=lines,
536 )
538 def _collect_lines(self, blocks: list[dict]) -> None:
539 """Derive the consumed set FROM THE BLOCK TREE THAT BECOMES THE OUTPUT.
541 An earlier version incremented a counter as the parser walked, which
542 measured "the parser SAW this line" -- a strictly weaker property than "this
543 line reached the manuscript", and the two diverge exactly when a lowering
544 step drops something. Sabotage S1 proved it: deleting the append that
545 records a task-body line left the side-effect marker in place, so the
546 reconciliation still balanced while the content was gone. A ledger derived
547 from the artifact cannot drift from the artifact.
548 """
549 for b in blocks:
550 for n in b.get("lines", [b["line"]]):
551 self.consumed.add(n)
552 if b.get("body"):
553 self._collect_lines(b["body"])
554 if b.get("children"):
555 self._collect_lines(b["children"])
557 def _detect_mixed_lists(self) -> None:
558 """Find authored lists that mix checkbox items with plain bullets.
560 This has to run over the WHOLE file before parsing, not inside the list
561 parser. The list parser never saw a mixed run: `TASK_RE` is dispatched
562 before `BULLET_RE`, so the checkbox items were consumed as `@task` siblings
563 and only the plain-bullet remainder ever reached `_parse_list`. The list
564 WAS split and nothing warned -- a warning class reading zero because the
565 code path that raises it is unreachable, which is worse than a wrong count
566 because it looks like evidence of absence.
567 """
568 lines = self.src.lines
569 masked = self.src.masked
570 i = self.src.body_start
571 n = len(lines)
572 while i < n:
573 if self.src.in_fence(i) or not BULLET_RE.match(masked[i]):
574 i += 1
575 continue
576 start = i
577 indent = indent_width(BULLET_RE.match(masked[i]).group(1))
578 checkbox = plain = 0
579 while i < n and lines[i].strip() and not self.src.in_fence(i):
580 m = BULLET_RE.match(masked[i])
581 if m and indent_width(m.group(1)) == indent:
582 if TASK_RE.match(masked[i]):
583 checkbox += 1
584 else:
585 plain += 1
586 elif not m and indent_width(lines[i]) <= indent:
587 break
588 i += 1
589 if checkbox and plain:
590 self.rep.warn(
591 Warn.W10, start + 1,
592 f"one authored list mixes {checkbox} checkbox item(s) with "
593 f"{plain} plain bullet(s); the checkbox items become sibling "
594 "@task nodes and the plain bullets an @items block, splitting "
595 "one authored list into separate blocks",
596 checkbox=checkbox, plain=plain,
597 )
599 # -------------------------------------------------------------- parse
600 def _parse_body(self) -> list[dict]:
601 """Lex the body into blocks. Every block carries its 1-based source line."""
602 blocks: list[dict] = []
603 i = self.src.body_start
604 lines = self.src.lines
605 masked = self.src.masked
606 n = len(lines)
608 while i < n:
609 raw = lines[i]
610 if not raw.strip():
611 i += 1
612 continue
614 # AN HTML COMMENT IS NOT CONTENT.
615 #
616 # Caught by the pressure corpus: emitting a comment line as `@p{...}`
617 # promoted invisible annotation into visible prose -- and re-introduced
618 # `{id: t-commented}` as rendered text after the accounting had
619 # correctly excluded it. Comments are dropped from the manuscript and
620 # RECORDED with their locations in the audit record, so nothing is lost
621 # silently. This is a TOTAL rule (a comment is never content), so it is
622 # an audit fact rather than a warning.
623 if not self.src.masked[i].strip():
624 self.comment_lines.append({"line": i + 1, "text": raw.strip()})
625 self.consumed.add(i + 1)
626 i += 1
627 continue
629 if self.src.in_fence(i):
630 m = FENCE_RE.match(masked[i])
631 if m:
632 lang = m.group(3)
633 j = i + 1
634 content = []
635 while j < n and self.src.in_fence(j) and not FENCE_RE.match(masked[j]):
636 content.append(lines[j])
637 j += 1
638 blocks.append({"t": "code", "line": i + 1, "lang": lang,
639 "lines": list(range(i + 1, min(j + 1, n) + 1)),
640 "text": "\n".join(content)})
641 i = j + 1 if j < n else j
642 continue
643 i += 1
644 continue
646 m = HEADING_RE.match(masked[i])
647 if m:
648 blocks.append({"t": "heading", "line": i + 1, "lines": [i + 1],
649 "level": len(m.group(1)), "text": lines[i].split("#", 1)[-1].lstrip("# ").strip()})
650 i += 1
651 continue
653 if TASK_RE.match(masked[i]):
654 block, i = self._parse_task_block(i)
655 blocks.append(block)
656 continue
658 if LOG_RE.match(masked[i]):
659 lm = LOG_RE.match(lines[i]) or LOG_RE.match(masked[i])
660 blocks.append({"t": "update", "line": i + 1, "lines": [i + 1],
661 "date": lm.group(1), "text": lm.group(2)})
662 i += 1
663 continue
665 if BULLET_RE.match(masked[i]):
666 block, i = self._parse_list(i)
667 blocks.append(block)
668 continue
670 if "|" in masked[i] and i + 1 < n and TABLE_SEP_RE.match(masked[i + 1]):
671 block, i = self._parse_table(i)
672 blocks.append(block)
673 continue
675 # paragraph: consume to blank line / next construct
676 start = i
677 para = []
678 while i < n and lines[i].strip() and not self.src.in_fence(i):
679 if i > start and (HEADING_RE.match(masked[i]) or TASK_RE.match(masked[i])
680 or BULLET_RE.match(masked[i])):
681 break
682 para.append(lines[i])
683 i += 1
684 blocks.append({"t": "para", "line": start + 1,
685 "lines": list(range(start + 1, i + 1)),
686 "text": "\n".join(para)})
687 return blocks
689 def _parse_task_block(self, i: int) -> tuple[dict, int]:
690 lines = self.src.lines
691 masked = self.src.masked
692 n = len(lines)
693 m = TASK_RE.match(masked[i])
694 base = indent_width(m.group("indent"))
695 header_raw = TASK_RE.match(lines[i]) or TASK_RE.match(masked[i])
696 content = header_raw.group("body")
698 meta_m = META_RE.search(content)
699 meta_str = meta_m.group(1) if meta_m else ""
700 text = content[: meta_m.start()].rstrip() if meta_m else content.rstrip()
702 body_lines: list[tuple[int, str]] = []
703 children: list[dict] = []
704 j = i + 1
705 while j < n:
706 if not lines[j].strip():
707 k = j + 1
708 while k < n and not lines[k].strip():
709 k += 1
710 if k < n and indent_width(lines[k]) >= base + 2:
711 j = k
712 continue
713 break
714 if indent_width(lines[j]) < base + 2:
715 break
716 if TASK_RE.match(masked[j]) and indent_width(TASK_RE.match(masked[j]).group("indent")) >= base + 2:
717 child, j = self._parse_task_block(j)
718 children.append(child)
719 continue
720 # A FENCE INSIDE A TASK BODY IS STILL A FENCE.
721 #
722 # Caught by the pressure corpus: treating body lines as uniformly
723 # prose turned ```scheme / (display ...) / ``` into three paragraphs
724 # whose backticks became literal text. That is a semantic corruption
725 # of the exact construct the plan names ("code fences"), and it is
726 # invisible unless a fixture puts a fence inside a task body -- which
727 # is precisely why F0 put one there.
728 if self.src.in_fence(j):
729 fm = FENCE_RE.match(masked[j])
730 if fm:
731 lang = fm.group(3)
732 fence_indent = len(fm.group(1))
733 k = j + 1
734 content = []
735 while k < n and self.src.in_fence(k) and not FENCE_RE.match(masked[k]):
736 line = lines[k]
737 content.append(line[fence_indent:] if line[:fence_indent].strip() == "" else line)
738 k += 1
739 body_lines.append({"t": "code", "line": j + 1, "lang": lang,
740 "lines": list(range(j + 1, min(k + 1, n) + 1)),
741 "text": "\n".join(content)})
742 j = k + 1 if k < n else k
743 continue
744 body_lines.append({"t": "para", "line": j + 1, "lines": [j + 1],
745 "text": lines[j].strip()})
746 j += 1
748 return ({"t": "task", "line": i + 1, "lines": [i + 1], "done": header_raw.group("done").lower() == "x",
749 "text": text, "meta": meta_str, "body": body_lines,
750 "children": children}, j)
752 def _parse_list(self, i: int) -> tuple[dict, int]:
753 lines = self.src.lines
754 masked = self.src.masked
755 n = len(lines)
756 items = []
757 had_checkbox = False
758 start = i
759 while i < n and BULLET_RE.match(masked[i]):
760 if TASK_RE.match(masked[i]):
761 had_checkbox = True
762 break
763 raw_bullet = BULLET_RE.match(lines[i])
764 # The MASK can match where the RAW line does not: `comment_mask` blanks
765 # comment characters, so `<!-- x --> - a bullet` matches masked and
766 # returns None on raw. An unguarded .group() there aborted every file
767 # in the corpus, not just this one.
768 items.append((i + 1, raw_bullet.group(2) if raw_bullet else lines[i].strip()))
769 i += 1
770 # Mixed runs are detected once, up front, by `_detect_mixed_lists`; warning
771 # here as well would double-count the denominator.
772 del had_checkbox
773 return ({"t": "items", "line": start + 1,
774 "lines": [ln for ln, _ in items], "items": items}, i)
776 def _parse_table(self, i: int) -> tuple[dict, int]:
777 lines = self.src.lines
778 n = len(lines)
779 rows = []
781 def cells(line: str) -> list[str]:
782 body = line.strip()
783 if body.startswith("|"):
784 body = body[1:]
785 if body.endswith("|"):
786 body = body[:-1]
787 return [c.strip() for c in body.split("|")]
789 header = cells(lines[i])
790 rows.append((i + 1, header, True))
791 # `columns:` is a NONEMPTY LIST OF ALIGNMENTS (left/center/right), not a
792 # count -- `make-table` refuses an integer. Caught by the parse arm; the
793 # alignment is carried in the Markdown separator row, so emitting a count
794 # would have thrown away information the source actually states.
795 aligns = []
796 for spec in cells(lines[i + 1]):
797 spec = spec.strip()
798 left, right = spec.startswith(":"), spec.endswith(":")
799 aligns.append("center" if left and right else "right" if right else "left")
800 i += 2 # header + separator
801 while i < n and "|" in lines[i] and lines[i].strip():
802 rows.append((i + 1, cells(lines[i]), False))
803 i += 1
804 return ({"t": "table", "line": rows[0][0], "rows": rows,
805 "lines": [rows[0][0] + 1] + [ln for ln, _, _ in rows],
806 "aligns": aligns or ["left"] * len(header)}, i)
808 # -------------------------------------------------------------- lower
809 def _inline(self, text: str, line: int) -> str:
810 """Inline content destined for a BRACE BODY: record edges, then escape."""
811 return esc(self._record_links(text, line))
813 def _record_links(self, text: str, line: int) -> str:
814 """Record wikilink edges and return the text UNESCAPED.
816 Split from `_inline` because the two destinations need DIFFERENT escaping and
817 conflating them corrupted every title silently. `esc()` is for a brace body,
818 where `@ { }` are syntax. A STRING LITERAL (`title: "..."`) needs only
819 `sstr()`; applying `esc()` first put the escape characters INTO the value, so
820 a task titled `Fix {braces}` became the string `Fix @{braces@}` -- which
821 parses perfectly and is simply wrong. Nothing in the gate could see it: the
822 manuscript is valid, the line is consumed, the id is accounted for.
824 Wikilinks CANNOT become `@ref`: `make-reference` requires a flat node id
825 (`valid-id?`), and a document address contains `/`. So the visible text is
826 preserved verbatim and the EDGE is hoisted to a block-level `@related` --
827 the ruling being that an edge living only in an audit record is not a
828 backlink, and post-migration `folio_note-links` would silently return less
829 than the Markdown estate does.
830 """
831 for m in LINK_RE.finditer(text):
832 target = m.group(1).strip()
833 label = None
834 whole = m.group(0)
835 if "|" in whole:
836 label = whole[2:-2].split("|", 1)[1].strip()
837 edge = {"document": target, "label": label, "line": line, "minted": True}
838 if not DOC_ID_RE.match(target):
839 self.rep.warn(
840 Warn.W4, line,
841 f"link target {target!r} is not a valid document address; "
842 "the edge is recorded but not emitted",
843 target=target,
844 )
845 edge["emitted"] = False
846 else:
847 edge["emitted"] = True
848 self.rep.warn(
849 Warn.W5, line,
850 f"inline link to {target!r} hoisted to a block-level @related "
851 "with CONVERTER-MINTED kind 'mentions; the source states no kind",
852 target=target,
853 )
854 self.edges.append(edge)
855 return text
857 def _emit_related(self, indent: str) -> list[str]:
858 out = []
859 for e in self.edges:
860 if not e.get("emitted") or e.get("written"):
861 continue
862 e["written"] = True
863 label = esc(e["label"]) if e["label"] else ""
864 out.append(
865 f'{indent}@related[document: {sstr(e["document"])} kind: {sym("mentions")}]'
866 f"{{{label}}}"
867 )
868 return out
870 def _emit_task(self, block: dict, indent: str) -> list[str]:
871 rep = self.rep
872 line = block["line"]
873 pairs = parse_metadata(block["meta"])
874 meta = {k: v for k, v in pairs}
876 tid = meta.get("id")
877 if not tid:
878 # A visible task with no id. F0 counts 26 of these in the estate.
879 tid = None
880 elif not NODE_ID_RE.match(tid):
881 # DROP it, do not emit it. Warning and then emitting anyway produced a
882 # manuscript `make-task` refuses -- the one W9 path that made output
883 # unparseable, while the priority and due paths correctly drop.
884 rep.warn(
885 Warn.W9, line,
886 f"task id {tid!r} is outside the flat node-id grammar and is DROPPED; "
887 "the task is emitted without an id rather than with one the reader refuses",
888 value=tid,
889 )
890 tid = None
892 # THE ONE-BIT CONTRACT.
893 status = "done" if block["done"] else "ready"
894 rep.warn(
895 Warn.W1, line,
896 f"status {status!r} supplied by the declared conversion contract; the "
897 "source dialect has no task status field and carries only the checkbox bit",
898 status=status,
899 )
901 priority = meta.get("priority")
902 if priority and priority not in TASK_PRIORITIES:
903 rep.warn(Warn.W9, line, f"priority {priority!r} is outside the frozen set", value=priority)
904 priority = None
906 tags = []
907 if "tags" in meta:
908 # Space-separated WITHIN the comma-delimited field. Conflating the two
909 # separators is sabotage S2.
910 for tag in meta["tags"].split():
911 if SEGMENT_RE.match(tag):
912 tags.append(tag)
913 else:
914 rep.warn(Warn.W9, line, f"tag {tag!r} is outside the segment grammar", value=tag)
915 self.tags_emitted += len(tags)
916 self.tags_seen += len(meta["tags"].split())
918 due = meta.get("due")
919 if due and not DATE_RE.match(due):
920 rep.warn(Warn.W9, line, f"due {due!r} is not YYYY-MM-DD", value=due)
921 due = None
923 # RULING 2(b): `completed` is an EVENT, and @update is the event record --
924 # the same shape F3's atomic status+@update requirement builds going
925 # forward, so history and future use one pattern. No new field.
926 completed = meta.get("completed")
927 completion_update = []
928 if completed:
929 if DATE_RE.match(completed):
930 completion_update = [
931 f'{indent}@update[#f timestamp: {sstr(completed)}]{{',
932 f"{indent}@p{{Completed on {esc(completed)}. Recorded from the "
933 f"source task's @@code{{completed:}} metadata during M0 conversion.}}",
934 f"{indent}}}",
935 ]
936 else:
937 rep.warn(Warn.W9, line,
938 f"completed {completed!r} is not YYYY-MM-DD; no @update emitted",
939 value=completed)
941 # RULING 2(d), and the part of it the data does NOT support. `source` (223)
942 # and `ref` (46) are PROVENANCE, not relations: `source` is a system label
943 # (`chat` 101, `bureau` 68) and `ref` is that system's own record id
944 # (`StnJsf2730Po`, `em-StpPAgdELh1c`). Neither names a folio document or
945 # node, so a @related edge would be dangling by construction -- and
946 # `StnJsf2730Po` is not even a valid node id. Held and reported.
947 for key in ("source", "ref"):
948 if key in meta:
949 rep.warn(
950 Warn.W19, line,
951 f"{key}={meta[key]!r} is provenance, not a relation: it names an "
952 "external system or that system's record id, not a folio "
953 "document or node. No @related edge is emitted; the value is "
954 "preserved in this audit record",
955 key=key, value=meta[key],
956 )
958 for key, value in pairs:
959 if key in ("completed",) and completion_update:
960 continue # it has a home now
961 if key in TASK_FIELDS or key in ("source", "ref"):
962 continue
963 if True:
964 rep.warn(
965 Warn.W2, line,
966 f"task metadata key {key!r}={value!r} has no field in the frozen "
967 "vocabulary (task-record carries exactly id/title/status/priority/"
968 "tags/due/children); preserved in this audit record only",
969 key=key, value=value,
970 )
972 title = block["text"].strip()
973 if not title:
974 rep.warn(Warn.W6, line, "task has no title text; make-task requires a nonempty title")
975 title = "(untitled)"
977 self.visible_tasks.append({"id": tid, "line": line, "status": status})
979 if not tid:
980 # `make-task` REQUIRES a valid node id -- unlike `make-update` and
981 # `make-query` it does not accept #f -- so an id-less task cannot be
982 # emitted without one. D4 forbids DERIVING one; ruling 2026-08-12
983 # permits random ASSIGNMENT through a persistent allocation table, which
984 # is the act `folio_task-add` already performs at creation.
985 #
986 # The naming event is deliberately loud: a converter that invents
987 # identity must say so every single time, even when it is authorised.
988 occurrence = sum(
989 1 for e in self.minted_ids if e["title"] == title
990 )
991 key = f"{self.src.rel}::{title}::{occurrence}"
992 tid = self.allocator.allocate(
993 key, self.src.rel, title, line, self.all_known_ids
994 )
995 self.all_known_ids.add(tid)
996 self.minted_ids.append({"id": tid, "line": line, "title": title, "key": key})
997 rep.warn(
998 Warn.W18, line,
999 f"visible task {title!r} carries no id; id {tid} was MINTED by "
1000 "random assignment through the allocation table (D4 forbids "
1001 "deriving one; assignment is not derivation)",
1002 title=title, minted_id=tid,
1005 args = []
1006 args.append(sstr(tid))
1007 args.append(f"title: {sstr(self._record_links(title, line))}")
1008 args.append(f"status: {sym(status)}")
1009 if priority:
1010 args.append(f"priority: {sym(priority)}")
1011 if tags:
1012 args.append(f"tags: {slist(tags)}")
1013 if due:
1014 args.append(f"due: {sstr(due)}")
1016 out = [f'{indent}@task[{" ".join(args)}]{{']
1017 for item in block["body"]:
1018 out.extend(self._emit_block(item, indent))
1019 for child in block["children"]:
1020 out.extend(self._emit_task(child, indent))
1021 out.extend(completion_update)
1022 out.extend(self._emit_related(indent))
1023 out.append(f"{indent}}}")
1024 return out
1026 def _emit_block(self, block: dict, indent: str) -> list[str]:
1027 t = block["t"]
1028 line = block["line"]
1029 if t == "para":
1030 body = self._inline(block["text"], line)
1031 return [f"{indent}@p{{{body}}}"] + self._emit_related(indent)
1032 if t == "code":
1033 # BLOCK-POSITION @code IS A RAW BODY. `parser.sgl:428` -- "its contents
1034 # are acquired without interpreting document escapes, commands, or
1035 # @(...). Brace bodies balance literal braces."
1037 # So escaping code content is actively WRONG: `@{` is not an escape here,
1038 # it is two literal characters, and every fence containing a brace came
1039 # out corrupted while parsing perfectly. Literal braces are fine as long
1040 # as they BALANCE; when they do not, the brace body ends early, so the
1041 # named form -- which stops only at a line-level `@end code` -- is the
1042 # only spelling that survives.
1043 # `@code{...}` with no first positional builds an INLINE code node
1044 # (`author.sgl`: `(code value . rest)` -> `make-code` when rest is
1045 # empty), which is not a block node and is refused inside a section.
1046 # This was 46 of 77 parse failures on a 171-file real-corpus slice --
1047 # every unlabelled ``` fence in the estate. `@code[""]` is refused too
1048 # ("invalid language or content"), so a language must be supplied; the
1049 # source states none, which makes this a PARTIAL rule, not a silent fix.
1050 lang = block["lang"]
1051 if not lang:
1052 lang = "text"
1053 self.rep.warn(
1054 Warn.W14, line,
1055 "fenced code block states no language; 'text' supplied because a "
1056 "language-less @code lowers to an INLINE node that a section refuses",
1058 head = f"@code[{sstr(lang)}]"
1059 text = block["text"]
1060 depth = 0
1061 balanced = True
1062 for ch in text:
1063 if ch == "{":
1064 depth += 1
1065 elif ch == "}":
1066 depth -= 1
1067 if depth < 0:
1068 balanced = False
1069 break
1070 if depth != 0:
1071 balanced = False
1072 if balanced:
1073 return [f"{indent}{head}{{{text}}}"]
1074 if any(ln.strip().startswith("@end code") for ln in text.split("\n")):
1075 self.rep.warn(
1076 Warn.W6, line,
1077 "code block has unbalanced braces AND contains a line-level "
1078 "'@end code'; neither the brace body nor the named body can "
1079 "carry it verbatim",
1081 return [f"{indent}{head}{{{esc(text)}}}"]
1082 return [f"{indent}{head}", text, "@end code"]
1083 if t == "task":
1084 return self._emit_task(block, indent)
1085 if t == "items":
1086 # `make-item-list` refuses an empty child list, so an item run that
1087 # collected nothing must emit no block at all rather than `@items{}`.
1088 if not block["items"]:
1089 return []
1090 out = [f"{indent}@items{{"]
1091 for iline, itext in block["items"]:
1092 out.append(f"{indent}@item{{{self._inline(itext, iline)}}}")
1093 out.append(f"{indent}}}")
1094 return out + self._emit_related(indent)
1095 if t == "table":
1096 aligns = "'(" + " ".join(block["aligns"]) + ")"
1097 out = [f"{indent}@table[columns: {aligns}]{{"]
1098 width = len(block["aligns"])
1099 for rline, cells, is_header in block["rows"]:
1100 if len(cells) != width:
1101 # `make-table` refuses a ragged row. Markdown tolerates them and
1102 # the estate writes them; padding/truncating is the only
1103 # representable choice, and it is a choice, so it warns.
1104 self.rep.warn(
1105 Warn.W15, rline,
1106 f"table row has {len(cells)} cell(s) but the table declares "
1107 f"{width} column(s); the row is "
1108 + ("padded" if len(cells) < width else "truncated"),
1109 cells=len(cells), columns=width,
1111 cells = (cells + [""] * width)[:width]
1112 head = "@row[header: #t]" if is_header else "@row"
1113 out.append(f"{indent}{head}{{")
1114 for c in cells:
1115 out.append(f"{indent}@cell{{{self._inline(c, rline)}}}")
1116 out.append(f"{indent}}}")
1117 out.append(f"{indent}}}")
1118 return out + self._emit_related(indent)
1119 if t == "update":
1120 # `@update` takes its id POSITIONALLY and the source states none, so #f
1121 # is written explicitly -- model.sgl permits `id` to be #f.
1122 body = self._inline(block["text"], line)
1123 return [
1124 f'{indent}@update[#f timestamp: {sstr(block["date"])}]{{',
1125 f"{indent}@p{{{body}}}",
1126 f"{indent}}}",
1127 ] + self._emit_related(indent)
1128 self.rep.warn(Warn.W6, line, f"unrecognized construct {t!r}")
1129 return []
1131 def _emit_root(self, blocks: list[dict]) -> str:
1132 src = self.src
1133 fm = src.frontmatter
1134 doc_id = src.document_id()
1135 is_project = src.kind in ("project", "archived-project")
1137 # Title: the first level-1 heading, else the document address's last segment.
1138 title = None
1139 consumed = None
1140 for idx, b in enumerate(blocks):
1141 if b["t"] == "heading" and b["level"] == 1:
1142 title = b["text"]
1143 consumed = idx
1144 break
1145 if not title:
1146 title = doc_id.rsplit("/", 1)[-1]
1147 self.rep.warn(
1148 Warn.W13, 1,
1149 "no level-1 heading; title derived from the document address because "
1150 "make-note/make-project require a nonempty title",
1153 tags = []
1154 raw_tags = fm.get("tags", [])
1155 if isinstance(raw_tags, str):
1156 # Frontmatter tags come in four real spellings in this estate, and
1157 # whitespace-splitting handled only one. `tags: []` is YAML for NO TAGS
1158 # and was being read as a tag literally named "[]" -- 360 files carry
1159 # that spelling. `tags: a, b` left trailing commas on every token.
1161 # This is NOT the comma/tag conflation hazard: that one is about the
1162 # task-metadata `{k: v, k: v}` block, where `,` separates FIELDS and a
1163 # space separates tags WITHIN one. Here the whole value is the tag list,
1164 # so both characters are item separators.
1165 s = raw_tags.strip()
1166 if s in ("[]", ""):
1167 raw_tags = []
1168 elif s.startswith("[") and s.endswith("]"):
1169 raw_tags = [x.strip().strip("\"'") for x in s[1:-1].split(",") if x.strip()]
1170 else:
1171 raw_tags = [x for x in re.split(r"[,\s]+", s) if x]
1172 for tag in raw_tags:
1173 tag = str(tag).strip()
1174 if SEGMENT_RE.match(tag):
1175 tags.append(tag)
1176 elif tag:
1177 self.rep.warn(Warn.W9, 1, f"frontmatter tag {tag!r} is outside the segment grammar", value=tag)
1179 # `notes/archive/**` (635 files) classifies as `note` and yields the real
1180 # folio address -- `archive/tasks/f2a-passive-inspector` is a live note name,
1181 # so stripping the `notes/` prefix is exactly right there.
1183 # The top-level `archive/projects/**` (11 files) is different: F0 records it
1184 # as "not discovered by current project parser", so the product has never had
1185 # an address for it and this converter is CHOOSING one. Plausible is not the
1186 # same as stated, so it warns rather than deciding quietly.
1187 if src.kind in ("archived-note", "archived-project"):
1188 self.rep.warn(
1189 Warn.W12, 1,
1190 f"document address {doc_id!r} was DERIVED for an archived source; "
1191 "F0 records this class as not discovered by the current parser, so "
1192 "the source states no address and the converter chose one",
1193 document_id=doc_id, kind=src.kind,
1196 # RULING 2(d): frontmatter `tasks:` is a list of NODE IDS, so it maps
1197 # cleanly to @related. A custom kind is used deliberately: F4 admits any
1198 # symbol, keeps it inspectable, and excludes it from built-in invariants --
1199 # which is exactly right for an edge a migration introduced. `contains`
1200 # would overclaim, since the task does not live in this note.
1201 task_edges = []
1202 raw_assoc = fm.get("tasks", [])
1203 if isinstance(raw_assoc, str):
1204 raw_assoc = [x for x in re.split(r"[,\s]+", raw_assoc.strip()) if x]
1205 for node_id in raw_assoc:
1206 node_id = str(node_id).strip()
1207 if NODE_ID_RE.match(node_id):
1208 task_edges.append(
1209 f'@related[node: {sstr(node_id)} kind: {sym("associated-task")}]{{}}'
1211 elif node_id:
1212 self.rep.warn(Warn.W9, 1,
1213 f"frontmatter task association {node_id!r} is not a flat node id",
1214 value=node_id)
1216 # RULING 2(c): `last-updated` is DERIVED metadata whose authority is the
1217 # workspace, and conversion breaks its Git derivation exactly once. So it
1218 # gets no field -- created is authored, last-updated is observed, and fields
1219 # are for the authored -- but the break is stamped as ONE conversion @update.
1220 conversion_update = []
1221 last_updated = fm.get("last-updated")
1222 if isinstance(last_updated, str) and DATE_RE.match(last_updated.strip()):
1223 lu = last_updated.strip()
1224 conversion_update = [
1225 f'@update[#f timestamp: {sstr(lu)}]{{',
1226 "@p{Converted from Markdown to folio/doc by M0. The source carried "
1227 f"@@code{{last-updated: {esc(lu)}}}, which is derived metadata with no "
1228 "field in the frozen vocabulary; this record stamps the conversion so "
1229 "the break in its Git derivation is visible exactly once.}",
1230 "}",
1233 allowed = PROJECT_FIELDS if is_project else NOTE_FIELDS
1234 mapped = {"tasks": "@related edges (kind: associated-task)" if task_edges else None,
1235 "last-updated": "one conversion @update" if conversion_update else None}
1236 for key, value in sorted(fm.items()):
1237 if mapped.get(key):
1238 self.residue[key] = mapped[key]
1239 continue
1240 if key not in allowed:
1241 self.residue.setdefault(key, "recorded in the audit record only")
1242 self.rep.warn(
1243 Warn.W3, 1,
1244 f"frontmatter key {key!r}={value!r} has no field in the frozen "
1245 f"vocabulary ({'project' if is_project else 'note'}-record carries "
1246 f"{sorted(allowed)}); preserved in this audit record only",
1247 key=key, value=str(value),
1250 archived_stamp: list[str] = []
1251 lines = ["#lang folio/doc"]
1252 if is_project:
1253 status = str(fm.get("status", "")).strip()
1254 # RULING 4(c)/(d): two source statuses map, each with an audit row.
1256 # `archived` is NOT added to the frozen set (4(a)): it is location
1257 # authority, F0's `archived-project` classification already carries it,
1258 # and a status field mirroring a location is the state-versus-record
1259 # drift trap -- the record says archived, the file moves back, and two
1260 # authorities disagree with no owner.
1262 # It converts to `paused`, the least-claiming in-set value. `done` or
1263 # `cancelled` would make positive claims the adjacent @update
1264 # contradicts. The resumability implication is neutralised by that
1265 # @update sitting next to it saying the manner was never recorded.
1266 if status == "archived":
1267 self.residue["status"] = "archived -> paused, plus an archival @update"
1268 self.rep.warn(
1269 Warn.W20, 1,
1270 "project status 'archived' maps to 'paused' -- the least-claiming "
1271 "in-set value. Archived-ness is location authority (F0 classifies "
1272 "the file), never a workflow state, and the adjacent @update "
1273 "records that the manner of archiving was not written down",
1274 source_value="archived", mapped_to="paused",
1276 status = "paused"
1277 if self.conversion_date:
1278 archived_stamp = [
1279 f'@update[#f timestamp: {sstr(self.conversion_date)}]{{',
1280 "@p{Archived; workflow state at archiving not recorded in "
1281 "source. TIMESTAMP SUBSTITUTED: this is the M0 conversion "
1282 "date, not the date of archiving, which the source does not "
1283 "carry.}",
1284 "}",
1286 else:
1287 self.rep.warn(
1288 Warn.W6, 1,
1289 "archival @update not emitted: no --conversion-date supplied "
1290 "and the source carries no archiving date. Reading a clock "
1291 "here would break byte-repeatability, so the date is an "
1292 "INPUT, never a clock read",
1294 elif status == "completed":
1295 self.residue["status"] = "completed -> done"
1296 self.rep.warn(
1297 Warn.W20, 1,
1298 "project status 'completed' maps to 'done' -- an obvious mapping, "
1299 "but still a mapping, so it gets a row rather than silence",
1300 source_value="completed", mapped_to="done",
1302 status = "done"
1303 if status not in CONTAINER_STATUSES:
1304 self.rep.warn(
1305 Warn.W9, 1,
1306 f"project status {status!r} is outside the frozen container set "
1307 f"{sorted(CONTAINER_STATUSES)}; defaulted to 'active",
1308 value=status,
1310 status = "active"
1311 lines.append(
1312 f'@project[{sstr(doc_id)} title: {sstr(self._record_links(title, 1))} '
1313 f"status: {sym(status)} tags: {slist(tags)}]"
1315 closer = "@end project"
1316 else:
1317 lines.append(
1318 f'@note[{sstr(doc_id)} title: {sstr(self._record_links(title, 1))} tags: {slist(tags)}]'
1320 closer = "@end note"
1322 lines.append("")
1323 lines.extend(self._emit_sections(blocks, consumed))
1324 lines.extend(task_edges)
1325 lines.extend(archived_stamp if is_project else [])
1326 lines.extend(conversion_update)
1327 lines.append(closer)
1328 return "\n".join(lines) + "\n"
1330 def _emit_sections(self, blocks: list[dict], skip: int | None) -> list[str]:
1331 """Headings become `@section`, nested by level. Slugs are FAITHFUL: a repeat
1332 is a warning with locations, never a rename (ruled 2026-08-12)."""
1333 out: list[str] = []
1334 open_levels: list[int] = []
1336 for idx, b in enumerate(blocks):
1337 if idx == skip:
1338 continue
1339 if b["t"] == "heading":
1340 while open_levels and open_levels[-1] >= b["level"]:
1341 open_levels.pop()
1342 out.append("}")
1343 base = slug(b["text"])
1344 self.seen_slugs.setdefault(base, []).append(b["line"])
1345 sid = base
1346 occurrences = self.seen_slugs[base]
1347 if len(occurrences) > 1:
1348 self.rep.warn(
1349 Warn.W11, b["line"],
1350 f"heading slug {base!r} repeats (also at line(s) "
1351 + ", ".join(str(n) for n in occurrences[:-1])
1352 + "); converted faithfully and NOT renamed -- anchor "
1353 "uniqueness is the publication layer's contract",
1354 slug=base, lines=occurrences,
1356 if self.disambiguate_anchors:
1357 sid = f"{base}-{len(occurrences)}"
1358 out.append(
1359 f"@section[{sstr(self._record_links(b['text'], b['line']))} "
1360 f"id: {sstr(sid)}]{{"
1362 open_levels.append(b["level"])
1363 continue
1364 out.extend(self._emit_block(b, ""))
1366 while open_levels:
1367 open_levels.pop()
1368 out.append("}")
1370 # `make-section` refuses an empty child list, and a heading with nothing
1371 # under it is an ordinary thing to write. Collapse to a fixpoint, because
1372 # a section whose only child was an empty section becomes empty in turn.
1373 while True:
1374 for i in range(len(out) - 1):
1375 if out[i].startswith("@section[") and out[i].endswith("{") and out[i + 1] == "}":
1376 text = out[i][len("@section["):].split('" id:')[0].lstrip('"')
1377 self.rep.warn(
1378 Warn.W16, 1,
1379 f"heading {text!r} has no content; the section is dropped "
1380 "because make-section refuses an empty child list. The "
1381 "heading text is preserved here and nowhere else",
1382 heading=text,
1384 del out[i:i + 2]
1385 break
1386 else:
1387 break
1388 return out
1390 # -------------------------------------------------------------- audit
1391 def _content_accounting(self) -> dict:
1392 """Reconcile SOURCE BODY LINES against what reached the manuscript.
1394 The population is every non-blank line below the frontmatter. Fence
1395 delimiters are named separately because they are syntax rather than content:
1396 a fence's ``` line carries no text of its own and is not lost when the fence
1397 becomes `@code`. Everything else must be consumed, and `unaccounted` must be
1398 zero -- with the actual line numbers, so a failure names what was dropped
1399 rather than only that something was.
1400 """
1401 body = [
1402 i + 1
1403 for i, line in enumerate(self.src.lines)
1404 if i >= self.src.body_start and line.strip()
1406 fence_delims = {
1407 i + 1
1408 for i in range(len(self.src.lines))
1409 if i >= self.src.body_start
1410 and self.src.in_fence(i)
1411 and FENCE_RE.match(self.src.masked[i])
1413 unaccounted = sorted(set(body) - self.consumed - fence_delims)
1414 return {
1415 "source_body_lines": len(body),
1416 "consumed": len(set(body) & self.consumed),
1417 "fence_delimiters": len(fence_delims),
1418 "unaccounted": unaccounted,
1419 "unaccounted_count": len(unaccounted),
1420 "tags_emitted": self.tags_emitted,
1421 "tags_seen": self.tags_seen,
1422 "note": (
1423 "Ids are not content. This reconciliation exists because a converter "
1424 "that drops a task-body line is still deterministic, still emits "
1425 "valid .sgd, and still accounts for every id -- so an id-only ledger "
1426 "cannot see the loss."
1427 ),
1430 def _audit(self, rep: Report) -> dict:
1431 src = self.src
1432 counts = rep.counts()
1433 return {
1434 "converter_version": CONVERTER_VERSION,
1435 "source": {
1436 "path": src.rel,
1437 "sha256": src.sha256,
1438 "bytes": len(src.data),
1439 "kind": src.kind,
1440 "owner": src.owner,
1441 },
1442 "document_id": src.document_id(),
1443 "contract": {
1444 "checkbox_to_status": {"unchecked": "ready", "checked": "done"},
1445 "note": (
1446 "DECLARED CONTRACT, not an inference. The source dialect has no "
1447 "task status field; it carries exactly one bit. The codomain is "
1448 "restricted to two values so the mapping stays a bijection on "
1449 "that bit."
1450 ),
1451 "minted_edge_kind": "mentions",
1452 "minted_edge_note": (
1453 "Every @related emitted by this converter is CONVERTER-MINTED "
1454 "with kind 'mentions. The source states no kind. No authored "
1455 "@related exists in the Markdown dialect, so every edge in the "
1456 "output is minted and none is indistinguishable from an authored one."
1457 ),
1458 "anchors": (
1459 "Heading slugs are converted faithfully. Repeats are reported "
1460 "(duplicate-anchor-slug), never renamed. --disambiguate-anchors "
1461 "is opt-in and is not the default."
1462 ),
1463 },
1464 "identity": {
1465 "population": self.id_population,
1466 "population_size": len(self.id_population),
1467 "visible": sum(1 for x in self.id_population if x["class"] == "visible"),
1468 "not_visible": sum(1 for x in self.id_population if x["class"] == "not-visible"),
1469 "unreachable": sum(1 for x in self.id_population if x["class"] == "unreachable"),
1470 "duplicates": {k: v for k, v in sorted(self.seen_ids.items()) if len(v) > 1},
1471 "buried": self.buried,
1472 "buried_count": len(self.buried),
1473 },
1474 "tasks": {
1475 "visible": len(self.visible_tasks),
1476 "without_id": sum(1 for t in self.visible_tasks if not t["id"]),
1477 "minted": {
1478 "count": len(self.minted_ids),
1479 "note": (
1480 "Source tasks that carried no id. Ids are RANDOMLY ASSIGNED "
1481 "through the committed allocation table, never derived from "
1482 "path or title -- D4 forbids derivation, and assignment is a "
1483 "different act. The table makes repeated runs reuse rather "
1484 "than re-mint."
1485 ),
1486 "entries": self.minted_ids,
1487 },
1488 "by_status": {
1489 "ready": sum(1 for t in self.visible_tasks if t["status"] == "ready"),
1490 "done": sum(1 for t in self.visible_tasks if t["status"] == "done"),
1491 },
1492 },
1493 "edges": {
1494 "total": len(self.edges),
1495 "emitted": sum(1 for e in self.edges if e.get("emitted")),
1496 "recorded_only": sum(1 for e in self.edges if not e.get("emitted")),
1497 "targets": sorted({e["document"] for e in self.edges}),
1498 },
1499 "anchors": {k: v for k, v in sorted(self.seen_slugs.items())},
1500 "content": self._content_accounting(),
1501 "key_dispositions": {
1502 "note": (
1503 "RULING 2(e): every source key that is not a frozen field gets a "
1504 "DOCUMENTED disposition here -- none silent, none guessed. A key "
1505 "that recurs at estate scale comes back as its own measured ruling."
1506 ),
1507 "entries": dict(sorted(self.residue.items())),
1508 },
1509 "comments_dropped": {
1510 "count": len(self.comment_lines),
1511 "note": (
1512 "HTML comments are not content and are not emitted into the "
1513 "manuscript. They are recorded here in full so nothing is lost "
1514 "silently. Ids inside them are classified html-comment in the "
1515 "identity population and mint no node."
1516 ),
1517 "entries": self.comment_lines,
1518 },
1519 "warnings": {
1520 "counts": counts,
1521 "denominator": rep.partial_firings,
1522 "denominator_derivation": (
1523 "Every source construct dispatches through exactly one lowering "
1524 "rule, declared TOTAL (bijection/injection into the frozen "
1525 "vocabulary) or PARTIAL (needs a choice the source does not "
1526 "determine). The denominator is the count of PARTIAL firings. It "
1527 "is exact because dispatch is exhaustive: the default arm is "
1528 "unrecognized-construct, itself a warning class, so nothing falls "
1529 "through silently."
1530 ),
1531 "entries": rep.warnings,
1532 },
1536# ==========================================================================
1537# Driver
1538# ==========================================================================
1539def discover(root: Path) -> list[str]:
1540 out = []
1541 for path in sorted(root.rglob("*.md")):
1542 rel = str(path.relative_to(root)).replace("\\", "/")
1543 out.append(rel)
1544 return out
1547def main() -> int:
1548 ap = argparse.ArgumentParser(description="M0 Markdown -> folio/doc converter")
1549 ap.add_argument("root", type=Path, help="corpus root (READ ONLY; never written)")
1550 ap.add_argument("--out", type=Path, required=True, help="output directory for .sgd")
1551 ap.add_argument("--audit", type=Path, required=True, help="output directory for audit JSON")
1552 ap.add_argument("--conversion-date", metavar="YYYY-MM-DD",
1553 help="date stamped on @update records the source cannot date "
1554 "(e.g. the archival stamp). An INPUT, never a clock read: "
1555 "reading the clock would break byte-repeatability.")
1556 ap.add_argument("--id-allocation", type=Path,
1557 help="persistent id allocation table (JSON). First run allocates, "
1558 "later runs REUSE -- this is what keeps repeated runs "
1559 "deterministic without the id being content-derived. Commit it.")
1560 ap.add_argument("--disambiguate-anchors", action="store_true",
1561 help="opt-in: rename repeated heading slugs (NOT the default; a "
1562 "migration must not rename what it can instead report)")
1563 args = ap.parse_args()
1565 allocator = IdAllocator(args.id_allocation)
1566 if args.conversion_date and not DATE_RE.match(args.conversion_date):
1567 print("--conversion-date must be YYYY-MM-DD", file=sys.stderr)
1568 return 2
1569 conv = Converter(disambiguate_anchors=args.disambiguate_anchors,
1570 allocator=allocator, conversion_date=args.conversion_date)
1571 args.out.mkdir(parents=True, exist_ok=True)
1572 args.audit.mkdir(parents=True, exist_ok=True)
1574 totals = {code: 0 for code in Warn.ALL}
1575 denominator = 0
1576 files = 0
1577 summary_rows = []
1579 skipped = []
1580 failures: list[dict] = []
1582 # PRE-PASS: collect every id in the corpus BEFORE minting anything. Without it,
1583 # a minted id could collide with an id that only appears in a file converted
1584 # later -- the collision would be real and the run would still look clean.
1585 for rel in discover(args.root):
1586 try:
1587 raw = (args.root / rel).read_bytes().decode("utf-8", errors="replace")
1588 except OSError:
1589 continue
1590 for m in ID_RE.finditer(raw):
1591 conv.all_known_ids.add(m.group(1))
1593 for rel in discover(args.root):
1594 try:
1595 data = (args.root / rel).read_bytes()
1596 src = Source(rel, data)
1597 except (SourceError, OSError) as exc:
1598 failures.append({"path": rel, "error": str(exc)})
1599 continue
1600 # CONVERT THE CANONICAL KINDS ONLY, and NAME every file that is skipped.
1602 # F0's `classify` is the authority and is reused rather than restated.
1603 # `README.md` classifies as `auxiliary`, and converting it minted the
1604 # document address "README" -- uppercase, which `valid-document-id?`
1605 # refuses. The parse arm caught it, which is the arm working; the fix is
1606 # to convert what F0 calls canonical and to ACCOUNT for the rest rather
1607 # than drop it quietly. A skipped file with no recorded class is exactly
1608 # the "count with no denominator" this estate refuses.
1609 if src.kind not in CONVERTIBLE_KINDS:
1610 skipped.append({"path": rel, "kind": src.kind, "owner": src.owner,
1611 "sha256": src.sha256})
1612 continue
1613 try:
1614 sgd, audit = conv.convert(src)
1615 except Exception as exc: # noqa: BLE001 -- isolate, never abort the corpus
1616 failures.append({"path": rel, "error": f"{type(exc).__name__}: {exc}"})
1617 continue
1618 files += 1
1620 target = args.out / (rel[:-3] + ".sgd")
1621 target.parent.mkdir(parents=True, exist_ok=True)
1622 target.write_text(sgd, encoding="utf-8")
1624 # `notes/a/b.md` and `notes/a__b.md` both mapped to `notes__a__b.json`, so
1625 # the second write destroyed the first record's entire evidence while the
1626 # .sgd survived. The digest makes the name injective.
1627 stem = rel[:-3].replace("/", "__")
1628 audit_path = args.audit / f"{stem}.{sha256(rel.encode())[:8]}.json"
1629 audit_path.write_text(
1630 json.dumps(audit, indent=2, sort_keys=True, ensure_ascii=False) + "\n",
1631 encoding="utf-8",
1634 for code, count in audit["warnings"]["counts"].items():
1635 totals[code] += count
1636 denominator += audit["warnings"]["denominator"]
1637 summary_rows.append({
1638 "path": rel,
1639 "document_id": audit["document_id"],
1640 "sha256": audit["source"]["sha256"],
1641 "kind": audit["source"]["kind"],
1642 "denominator": audit["warnings"]["denominator"],
1643 "visible_tasks": audit["tasks"]["visible"],
1644 "id_population": audit["identity"]["population_size"],
1645 "edges": audit["edges"]["total"],
1646 })
1648 summary = {
1649 "converter_version": CONVERTER_VERSION,
1650 "files": files,
1651 "files_discovered": files + len(skipped),
1652 "warning_counts": totals,
1653 "denominator": denominator,
1654 "denominator_check": sum(totals.values()),
1655 "files_detail": summary_rows,
1656 "failures": {
1657 "count": len(failures),
1658 "note": (
1659 "Files that could not be converted. A per-file failure is isolated "
1660 "and named; it never aborts the corpus, and the run exits non-zero "
1661 "so a gate cannot mistake a partial conversion for a complete one."
1662 ),
1663 "entries": failures,
1664 },
1665 "skipped": {
1666 "count": len(skipped),
1667 "note": (
1668 "Files F0's classify() does not call canonical. Named individually "
1669 "so the converted count carries its denominator: "
1670 "files_discovered = files + skipped.count."
1671 ),
1672 "entries": skipped,
1673 },
1675 (args.audit / "SUMMARY.json").write_text(
1676 json.dumps(summary, indent=2, sort_keys=True, ensure_ascii=False) + "\n",
1677 encoding="utf-8",
1680 allocator.save()
1682 print(
1683 f"M0_CONVERT_DONE files={files} minted={allocator.minted_this_run} "
1684 f"denominator={denominator} "
1685 f"check={sum(totals.values())} "
1686 + " ".join(f"{c}={totals[c]}" for c in Warn.ALL)
1688 # The denominator is recomputed from the per-class counts INDEPENDENTLY of the
1689 # running total. If those two ever disagree, the report is not trustworthy and
1690 # saying so loudly beats printing a number.
1691 if denominator != sum(totals.values()):
1692 print("M0_CONVERT_DENOMINATOR_MISMATCH", file=sys.stderr)
1693 return 1
1694 if failures:
1695 for f in failures:
1696 print(f"M0_CONVERT_FAILED {f['path']}: {f['error']}", file=sys.stderr)
1697 return 1
1698 return 0
1701if __name__ == "__main__":
1702 raise SystemExit(main())