src/folio/doc/manifest.sgl

1;;; (folio doc manifest) -- the cross-file transaction manifest and its

2;;; crash recovery (F3b).

3;;;

4;;; D11: "Cross-file transactions stage and validate the complete revision set

5;;; while holding the same root lock, then retain a bounded filesystem

6;;; crash-recovery manifest until every rename completes ... After acquiring

7;;; the released lock, the next writer checks for an incomplete transaction

8;;; manifest and either completes the already-validated rename set or rolls it

9;;; back before accepting another mutation."

10;;;

11;;; THE DISCRIMINATOR IS PRESENCE, BY CONSTRUCTION. The manifest is written

12;;; ATOMICALLY (temp + rename) and ONLY AFTER every staged file has passed its

13;;; full validation; target renames begin only after the manifest exists on

14;;; disk. Therefore:

15;;;

16;;; manifest present => the rename set was ALREADY VALIDATED, and recovery

17;;; COMPLETES it (or refuses to a human where

18;;; completion cannot be proven safe);

19;;; manifest absent => no target file was ever renamed, and recovery

20;;; ROLLS BACK by sweeping this module's staged temps

21;;; and backups.

22;;;

23;;; There is deliberately NO re-validate-and-proceed path: nothing in this

24;;; module re-runs validation, because a manifest that exists IS the proof

25;;; validation happened. A recovery path that re-validated and proceeded

26;;; would be a different and much worse thing than one that finishes a

27;;; decided operation (leader ruling, 2026-08-25, on D11's "already-validated").

28;;;

29;;; WHAT THE MANIFEST IS NOT. Operational recovery metadata only -- never

30;;; authoritative knowledge state. Nothing reads a manifest to learn what a

31;;; note says; it is read once, inside the lock, to finish or undo an

32;;; interrupted publish, and then removed.

33;;;

34;;; FORMAT. One line `F3B_MANIFEST v1 <txn-id> <op-label>` then one line per

35;;; file: tab-separated `FILE <target> <pre-rev> <temp> <temp-hash> <post-rev>

36;;; <backup> <backup-hash>`, then the terminator line `COMPLETE`. A manifest

37;;; whose terminator is missing or whose lines do not parse is CORRUPT --

38;;; unreachable while POSIX rename is atomic, kept as a defensive branch --

39;;; and recovery REFUSES TO A HUMAN rather than guessing: renames may have

40;;; begun, and an unprovable state is reported, not repaired. Paths are

41;;; recorded root-relative so a root moved wholesale still recovers.

42;;;

43;;; DURABILITY, STATED HONESTLY: no fsync primitive exists on this toolchain,

44;;; so the ordering guarantees here are about rename ATOMICITY (a crashed

45;;; writer leaves whole files, never torn ones), not power-loss durability of

46;;; the metadata itself.

47(define-library (folio doc manifest)

48 (import (sigil core)

49 (sigil struct)

50 (sigil string)

51 (sigil list)

52 (sigil fs)

53 (sigil path)

54 (sigil io)

55 (folio doc revision))

56 (export manifest-file-name

57 staging-prefix backup-prefix

58 mentry mentry? mentry-target mentry-pre-rev mentry-temp

59 mentry-temp-hash mentry-post-rev mentry-backup mentry-backup-hash

60 write-transaction-manifest! clear-transaction-manifest!

61 read-transaction-manifest

62 recover-root!)

63 (begin

64

65 ;; One spelling, shared with (folio doc coordinator)'s check and the gate.

66 (define manifest-file-name ".folio-transaction-manifest")

67 (define staging-prefix ".folio-staging-")

68 (define backup-prefix ".folio-backup-")

69

70 (define-struct mentry-record

71 (target) (pre-rev) (temp) (temp-hash) (post-rev) (backup) (backup-hash))

72

73 (define (mentry target pre-rev temp temp-hash post-rev backup backup-hash)

74 (mentry-record target: target pre-rev: pre-rev temp: temp

75 temp-hash: temp-hash post-rev: post-rev

76 backup: backup backup-hash: backup-hash))

77 (define (mentry? v) (mentry-record? v))

78 (define (mentry-target e) (mentry-record-target e))

79 (define (mentry-pre-rev e) (mentry-record-pre-rev e))

80 (define (mentry-temp e) (mentry-record-temp e))

81 (define (mentry-temp-hash e) (mentry-record-temp-hash e))

82 (define (mentry-post-rev e) (mentry-record-post-rev e))

83 (define (mentry-backup e) (mentry-record-backup e))

84 (define (mentry-backup-hash e) (mentry-record-backup-hash e))

85

86 ;;; Write the manifest ATOMICALLY: the temp is fully written first, the

87 ;;; rename is the commit point. Callers must call this only after every

88 ;;; entry's staged file passed validation -- presence means validated, and

89 ;;; this module cannot check that for them, so the ORDER OF CALLS in the

90 ;;; publisher is load-bearing and its gate arm plants exactly a reordering.

91 (define (write-transaction-manifest! root txn-id op-label entries)

92 (let* ((path (path-join root manifest-file-name))

93 (temp (string-append path ".tmp"))

94 (body (apply string-append

95 (format "F3B_MANIFEST v1 ~a ~a~%" txn-id op-label)

96 (append

97 (map (lambda (e)

98 (format "FILE\t~a\t~a\t~a\t~a\t~a\t~a\t~a~%"

99 (mentry-target e) (mentry-pre-rev e)

100 (mentry-temp e) (mentry-temp-hash e)

101 (mentry-post-rev e)

102 (mentry-backup e) (mentry-backup-hash e)))

103 entries)

104 (list "COMPLETE\n")))))

105 (write-file-string temp body)

106 (rename-file temp path)

107 path))

108

109 (define (clear-transaction-manifest! root)

110 (let ((path (path-join root manifest-file-name)))

111 (when (file-exists? path) (delete-file path))))

112

113 ;;; Parse the manifest. Returns (list txn-id op-label entries) or the

114 ;;; symbol 'corrupt. Absent is the CALLER's branch (file-exists?), so a

115 ;;; read here is always of something that claims to be a manifest.

116 (define (read-transaction-manifest root)

117 (let* ((path (path-join root manifest-file-name))

118 (text (guard (e (#t #f)) (read-file-string path))))

119 (if (not (string? text))

120 'corrupt

121 (let ((lines (string-split text "\n")))

122 (if (or (null? lines)

123 (not (string-starts-with? (car lines) "F3B_MANIFEST v1 ")))

124 'corrupt

125 (let* ((head (string-split (car lines) " "))

126 (txn-id (if (>= (length head) 3) (list-ref head 2) #f))

127 (op-label (if (>= (length head) 4) (list-ref head 3) #f)))

128 (let loop ((rest (cdr lines)) (entries '()) (complete #f))

129 (cond

130 ((null? rest)

131 (if complete (list txn-id op-label (reverse entries)) 'corrupt))

132 ((equal? (car rest) "COMPLETE")

133 (loop (cdr rest) entries #t))

134 ((equal? (car rest) "")

135 (loop (cdr rest) entries complete))

136 ((string-starts-with? (car rest) "FILE\t")

137 (let ((f (string-split (car rest) "\t")))

138 (if (= (length f) 8)

139 (loop (cdr rest)

140 (cons (mentry (list-ref f 1) (list-ref f 2)

141 (list-ref f 3) (list-ref f 4)

142 (list-ref f 5) (list-ref f 6)

143 (list-ref f 7))

144 entries)

145 complete)

146 'corrupt)))

147 (else 'corrupt)))))))))

148

149 (define (hash-of-file path)

150 (guard (e (#t #f)) (revision-of-file path)))

151

152 ;; THE MANIFEST IS UNTRUSTED INPUT AT RECOVERY TIME. A crashed writer --

153 ;; or anything that can drop a file into the root -- authored its fields,

154 ;; and recovery renames and deletes paths built from them. The forward

155 ;; engine only ever writes root-relative in-root paths, but recovery must

156 ;; not TRUST that: a manifest whose target/temp/backup contains `../`

157 ;; makes recovery rename OUTSIDE the root (reviewer HIGH, 2026-08-25,

158 ;; reproduced -- `outside victim changed: #t`). Every manifest path is

159 ;; resolved through real-path containment before it is acted on, exactly

160 ;; as the forward doors resolve a request's path; anything escaping is a

161 ;; corrupt/hostile manifest and joins the refuse-to-a-human class -- the

162 ;; extent of a hostile recovery cannot be established, so nothing is

163 ;; renamed or deleted. The check is on the PARENT dir (where the rename

164 ;; lands / the file sits), matching `require-contained-path`'s domain.

165 (define (path-contained? root rel)

166 (let ((root-real (realpath root)))

167 (and root-real

168 (let* ((joined (path-join root-real rel))

169 (parent-real (realpath (path-dirname joined))))

170 (and parent-real

171 (or (equal? parent-real root-real)

172 (string-starts-with? parent-real

173 (string-append root-real "/"))))))))

174

175 (define (manifest-paths-contained? root entries)

176 (let loop ((rest entries))

177 (cond

178 ((null? rest) #t)

179 ((and (path-contained? root (mentry-target (car rest)))

180 (path-contained? root (mentry-temp (car rest)))

181 (path-contained? root (mentry-backup (car rest))))

182 (loop (cdr rest)))

183 (else #f))))

184

185 ;;; RECOVERY. Runs inside the root lock (the coordinator calls it before

186 ;;; accepting any mutation). Returns a LIST OF ACTION STRINGS -- '() when

187 ;;; there was nothing to recover -- which the coordinator threads into the

188 ;;; transaction's result so a self-heal is never silent. RAISES with a

189 ;;; human-directed message when the state cannot be proven safe to finish

190 ;;; or undo; it never guesses.

191 ;;;

192 ;;; Two branches, per the discriminator:

193 ;;;

194 ;;; COMPLETE (manifest present and parseable): every entry is finished

195 ;;; individually --

196 ;;; target already at post-rev -> already published, note it

197 ;;; target at pre-rev, temp hash matches -> rename the temp in

198 ;;; anything else -> REFUSE to a human: either

199 ;;; the temp is gone (cannot complete) or the target is at a THIRD

200 ;;; revision (an external edit raced the crash window; finishing would

201 ;;; clobber it, rolling back would clobber it, only a human knows).

202 ;;; Backups and the manifest are removed only after every entry finished.

203 ;;;

204 ;;; ROLLBACK (no manifest): nothing was ever renamed, so recovery is

205 ;;; sweeping THIS MODULE'S debris -- stale staging temps and backups from

206 ;;; a transaction that died before its commit point. Targets are not

207 ;;; touched (there is nothing to restore; they were never written).

208 (define (recover-root! root)

209 (let ((mpath (path-join root manifest-file-name)))

210 (if (file-exists? mpath)

211 (recover-complete! root)

212 (sweep-debris! root))))

213

214 (define (recover-complete! root)

215 (let ((parsed (read-transaction-manifest root)))

216 (when (eq? parsed 'corrupt)

217 (error (format

218

                   "recovery: the transaction manifest at ~a/~a does not parse. Renames may have begun and their extent cannot be established; refusing every mutation on this root until a human inspects the manifest, the .folio-backup-* files, and the targets."

219 root manifest-file-name)))

220 (let ((txn-id (list-ref parsed 0))

221 (op-label (list-ref parsed 1))

222 (entries (list-ref parsed 2)))

223 ;; Containment FIRST, before proving anything else and before

224 ;; touching any file: a path that escapes the root cannot be

225 ;; safely renamed OR rolled back, and it is the signature of a

226 ;; corrupt or hostile manifest, so it is refused to a human like

227 ;; any other unprovable state.

228 (unless (manifest-paths-contained? root entries)

229 (error (format

230

                     "recovery: the transaction manifest at ~a/~a names a path outside the root. A manifest is written only with in-root relative paths, so this one is corrupt or hostile; recovery will not rename or delete along it. A human must inspect the manifest and the .folio-backup-* files."

231 root manifest-file-name)))

232 ;; First pass: PROVE every entry is finishable before touching

233 ;; anything -- completion is all-or-refuse, not best-effort.

234 (for-each

235 (lambda (e)

236 (let* ((target (path-join root (mentry-target e)))

237 (cur (hash-of-file target)))

238 (cond

239 ((equal? cur (mentry-post-rev e)) #t)

240 ((and (equal? cur (mentry-pre-rev e))

241 (equal? (hash-of-file (path-join root (mentry-temp e)))

242 (mentry-temp-hash e)))

243 #t)

244 (else

245 (error (format

246

                             "recovery: cannot prove ~a safe to complete (target at ~a; manifest expected pre ~a or post ~a; staged temp ~a). A human must finish or undo transaction ~a (~a); backups are the .folio-backup-* files named in the manifest."

247 (mentry-target e) (or cur "unreadable")

248 (mentry-pre-rev e) (mentry-post-rev e)

249 (if (equal? (hash-of-file (path-join root (mentry-temp e)))

250 (mentry-temp-hash e))

251 "intact" "missing or altered")

252 txn-id op-label))))))

253 entries)

254 ;; Second pass: finish.

255 (let ((actions

256 (map (lambda (e)

257 (let* ((target (path-join root (mentry-target e)))

258 (cur (hash-of-file target)))

259 (if (equal? cur (mentry-post-rev e))

260

                               (format "recovery: ~a was already published by the interrupted transaction ~a" (mentry-target e) txn-id)

261 (begin

262 (rename-file (path-join root (mentry-temp e)) target)

263

                                 (format "recovery: COMPLETED the validated rename of ~a for interrupted transaction ~a (~a)" (mentry-target e) txn-id op-label)))))

264 entries)))

265 ;; Only now is the metadata safe to drop.

266 (for-each (lambda (e)

267 (let ((b (path-join root (mentry-backup e))))

268 (guard (x (#t #f)) (when (file-exists? b) (delete-file b)))))

269 entries)

270 (clear-transaction-manifest! root)

271 actions))))

272

273 ;;; Sweep staged temps and backups from a transaction that never reached

274 ;;; its commit point. Only THIS MODULE'S prefixes; every removal reported.

275 ;;;

276 ;;; DEPTH IS UNBOUNDED, matching discovery: folio slugs nest routinely

277 ;;; (topics/, archive/tasks/), so a cross-file transaction stages debris

278 ;;; beside notes two and three directories deep, and a one-level sweep

279 ;;; leaves it to accumulate silently (reviewer MEDIUM, 2026-08-25). The

280 ;;; walk skips directory symlinks -- the same cycle hazard discovery

281 ;;; refuses to descend -- so it cannot loop, and it deletes ONLY a file

282 ;;; whose real path is under the real root, so a symlinked debris-named

283 ;;; file pointing outside cannot make the sweep delete out-of-root.

284 (define (sweep-debris! root)

285 (let ((root-real (realpath root))

286 (actions '()))

287 (let walk ((dirs (list root)))

288 (unless (null? dirs)

289 (let ((dir (car dirs)))

290 (let loop ((names (guard (e (#t '())) (or (directory-list dir) '())))

291 (subdirs '()))

292 (cond

293 ((null? names)

294 (walk (append subdirs (cdr dirs))))

295 (else

296 (let ((full (path-join dir (car names))))

297 (cond

298 ((and (directory? full) (not (symlink? full)))

299 (loop (cdr names) (cons full subdirs)))

300 ((and (not (directory? full))

301 (or (string-starts-with? (car names) staging-prefix)

302 (string-starts-with? (car names) backup-prefix)))

303 (let ((parent-real (realpath (path-dirname full))))

304 (when (and root-real parent-real

305 (or (equal? parent-real root-real)

306 (string-starts-with? parent-real

307 (string-append root-real "/"))))

308 (guard (e (#t #f)) (delete-file full))

309 (set! actions

310

                                   (cons (format "recovery: ROLLED BACK (swept pre-commit debris ~a -- no manifest existed, so no target was ever renamed)"

311 full)

312 actions))))

313 (loop (cdr names) subdirs))

314 (else (loop (cdr names) subdirs))))))))))

315 (reverse actions)))))