tools/m0_value_census.py

1#!/usr/bin/env python3

2"""Value census: every closed-vocabulary field's CORPUS VALUES against the FROZEN SET.

4WHY THIS EXISTS

5---------------

6A key-level gap analysis is enumeration at the wrong granularity. `priority` is a key

7the frozen vocabulary HAS, so a key-level census put it in the "representable" column

8and looked fine -- while `medium`, the estate's most-used priority value (526

9occurrences), was in no frozen set at all and `normal`, the value the model provided,

10appeared zero times. Only checking VALUES against the frozen sets found it.

12Promoted to a standing obligation for M1 (ruled 2026-08-12): the differential carries

13a value-census arm, derived from the corpus rather than sampled.

15THE FROZEN SETS ARE PARSED OUT OF `model.sgl`, NEVER RESTATED HERE.

16A hardcoded copy goes stale the moment the model moves -- and it is moving right now,

17since `normal` is being renamed to `medium`. A census that restated the old set would

18report a clean result against a vocabulary that no longer exists.

20READ-ONLY. Opens nothing for writing except an optional --json report.

22 python3 tools/m0_value_census.py /home/daviwil/Ops/folio

23"""

25from __future__ import annotations

27import argparse

28import json

29import re

30import sys

31from collections import Counter

32from pathlib import Path

34sys.dont_write_bytecode = True

35sys.path.insert(0, str(Path(__file__).resolve().parent))

37from f0_inventory import META_RE, TASK_RE, classify, comment_mask # noqa: E402

38from m0_convert import Source, parse_metadata # noqa: E402

40SET_RE = re.compile(r"\(define\s+(?P<name>[a-z-]+)\s+'\((?P<values>[^)]*)\)\)")

42# Which frozen set each source-side field is measured against. A field whose record

43# has NO such slot is named here with `None`, because "there is no field" and "the

44# value is out of range" are different findings and must not be merged.

45FIELD_TO_SET = {

46 "task.priority": "task-priorities",

47 "project.status": "container-statuses",

48 "note.status": None, # note-record has no status slot at all

49 "task.status": "task-statuses", # source carries only a checkbox bit

50 # The Markdown dialect has NO construct that lowers to a decision or a question,

51 # so these populations are STRUCTURALLY zero rather than merely unobserved. They

52 # are censused anyway, because "no construct exists" is a claim that should be

53 # measured rather than asserted -- and because a later dialect change would make

54 # them non-zero silently if nothing were watching.

55 "decision.status": "decision-statuses",

56 "question.status": "question-statuses",

57}

59# Observed values are additionally checked against EVERY frozen set, so a value that

60# has no home in its own field but IS a legal value of some other set shows up as the

61# near-miss it is rather than as undifferentiated noise.

62CROSS_CHECK = ("note.status",)

65def frozen_sets(model_path: Path) -> dict[str, list[str]]:

66 """Parse the value sets out of `(folio doc model)` itself."""

67 text = model_path.read_text(encoding="utf-8")

68 out = {}

69 for m in SET_RE.finditer(text):

70 name = m.group("name")

71 if name.endswith("statuses") or name.endswith("priorities") or name.endswith("kinds"):

72 out[name] = m.group("values").split()

73 if not out:

74 raise SystemExit(f"parsed no value sets from {model_path} -- the instrument is broken")

75 return out

78def census(root: Path) -> dict:

79 observed: dict[str, Counter[str]] = {k: Counter() for k in FIELD_TO_SET}

80 examples: dict[str, list[str]] = {}

81 files = 0

83 for path in sorted(root.rglob("*.md")):

84 rel = str(path.relative_to(root)).replace("\\", "/")

85 kind, _ = classify(rel)

86 if kind not in {"inbox", "note", "project", "archived-note", "archived-project"}:

87 continue

88 try:

89 src = Source(rel, path.read_bytes())

90 except Exception: # noqa: BLE001 -- a census must not abort on one bad file

91 continue

92 files += 1

93 is_project = kind in ("project", "archived-project")

95 fm_status = src.frontmatter.get("status")

96 if isinstance(fm_status, str) and fm_status.strip():

97 field = "project.status" if is_project else "note.status"

98 observed[field][fm_status.strip()] += 1

99 examples.setdefault(f"{field}={fm_status.strip()}", []).append(rel)

100

101 masked = comment_mask(src.lines)

102 for i, line in enumerate(masked):

103 if src.in_fence(i) or i in src.frontmatter_lines or not TASK_RE.match(line):

104 continue

105 m = META_RE.search(TASK_RE.match(src.lines[i]).group("body"))

106 if not m:

107 continue

108 meta = dict(parse_metadata(m.group(1)))

109 for key, field in (("priority", "task.priority"), ("status", "task.status")):

110 v = meta.get(key)

111 if v:

112 observed[field][v] += 1

113 examples.setdefault(f"{field}={v}", []).append(f"{rel}:{i + 1}")

114

115 return {"files_scanned": files, "observed": observed, "examples": examples}

116

117

118def main() -> int:

119 ap = argparse.ArgumentParser(description=__doc__)

120 ap.add_argument("root", type=Path)

121 ap.add_argument("--model", type=Path,

122 default=Path(__file__).resolve().parent.parent / "src/folio/doc/model.sgl")

123 ap.add_argument("--json", type=Path)

124 args = ap.parse_args()

125

126 sets = frozen_sets(args.model)

127 result = census(args.root)

128

129 print(f"frozen sets parsed from {args.model.name}:")

130 for name, values in sorted(sets.items()):

131 print(f" {name:24} {values}")

132 print(f"\nfiles scanned: {result['files_scanned']}\n")

133

134 report = {"frozen_sets": sets, "files_scanned": result["files_scanned"], "fields": {}}

135 total_out = 0

136 for field, set_name in sorted(FIELD_TO_SET.items()):

137 seen = result["observed"][field]

138 frozen = sets.get(set_name, []) if set_name else []

139 in_set = {v: n for v, n in sorted(seen.items()) if v in frozen}

140 out_set = {v: n for v, n in sorted(seen.items(), key=lambda kv: (-kv[1], kv[0]))

141 if v not in frozen}

142 unused = [v for v in frozen if v not in seen]

143 total_out += sum(out_set.values())

144 report["fields"][field] = {

145 "measured_against": set_name,

146 "in_frozen_set": in_set,

147 "NOT_IN_FROZEN_SET": out_set,

148 "frozen_values_never_used": unused,

149 "population": sum(seen.values()),

150 }

151 if field in CROSS_CHECK and out_set:

152 for value in out_set:

153 homes = [n for n, vs in sorted(sets.items()) if value in vs]

154 report["fields"][field].setdefault("cross_set_homes", {})[value] = homes

155 label = set_name or "NO SUCH FIELD in the record"

156 print(f"== {field} (vs {label}) ==")

157 print(f" population {sum(seen.values())}")

158 print(f" in frozen set {in_set}")

159 print(f" NOT in frozen set {out_set}")

160 print(f" frozen but unused {unused}")

161 cross = report["fields"][field].get("cross_set_homes")

162 if cross:

163 for value, homes in sorted(cross.items()):

164 print(f" cross-check: {value!r} is a legal value of {homes or 'NO frozen set'}")

165 print()

166

167 if args.json:

168 args.json.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n")

169 print(f"M0_VALUE_CENSUS out_of_set_occurrences={total_out} files={result['files_scanned']}")

170 return 0

171

172

173if __name__ == "__main__":

174 raise SystemExit(main())