F3b: rename-id with reference rewrite, and the cross-file engine

Message

The sharp one. folio-rename-id! renames a node id across every document that references it, all-or-nothing:

The NEW id must be completely fresh: any identifier-boundary occurrence anywhere in the root -- prose included -- refuses 'rename-target-taken.
Exactly ONE document may declare the old id; a workspace-ambiguous id refuses 'ambiguous-rename rather than picking (D9 presumes uniqueness and the estate has live duplicates).
The OWNING document's revision must match exactly; structural edits never rebase (amendment 1).
Every raw occurrence must be accounted for by a rewritable span (related/transclude node: fields, the owner's declaration) -- an occurrence in prose, code, a query spec, an uninspectable file, or a Markdown note refuses the WHOLE rename with no override: renaming around it would leave it stale, t-5e4329's defect class, and rewriting it blind is worse.
Every touched file is staged, verified through the full single-file pipeline (allowed-lost carries the old id), then GRAPH CLOSURE: zero boundary occurrences of the old id across the staged set.

publish-file-set! is the shared cross-file engine: hash-verified backups, hash-verified staged temps, the ATOMIC manifest write as the commit point, renames, manifest cleared, backups dropped. A crash before the manifest = debris the next writer sweeps; after = a validated set the next writer completes. The txn id derives from the operation's own content -- no clock, no randomness, reproducible in transcripts.

Smoked: a 3-file rename (owner + 1-site and 2-site referrers) published atomically with zero old-id residue and no debris; the five refusal classes (prose occurrence, markdown occurrence, taken target, stale owner revision, unknown id) each fire with their own reason. Toolchain note: this sigil's map is single-list; the result zip is a manual loop.

Changed

 src/folio/doc/structural.sgl | 320 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 316 insertions(+), 4 deletions(-)

Diff

src/folio/doc/structural.sglmodified

@@ -473,9 +473,321 @@

473

((pred (car rest)) (loop (cdr rest) (cons (car rest) out)))

474

(else (loop (cdr rest) out)))))

475

−476

;; Placeholders wired in the next slice; declared so the export list is

−477

;; honest from the first commit.

−478

(define (folio-rename-id! . args)

−479

(error "folio-rename-id!: landing in the next F3b slice"))

+476

;; ------------------------------------------------------------------

+477

;; THE CROSS-FILE PUBLICATION ENGINE

+478

;;

+479

;; Order is the contract (see manifest.sgl's header): stage everything,

+480

;; VERIFY everything, then backups -> temps -> MANIFEST (the atomic

+481

;; commit point) -> renames -> clear manifest -> drop backups. A crash

+482

;; before the manifest lands leaves only debris the next writer's

+483

;; recovery sweeps (nothing was renamed); a crash after it leaves a

+484

;; validated set the next writer COMPLETES. The txn id is derived from

+485

;; the operation's own content -- this toolchain deliberately exposes no

+486

;; clock or randomness to library code, and a content-derived id is

+487

;; reproducible in transcripts.

+488

;; ------------------------------------------------------------------

+489

(define (content-txn-id op-label parts)

+490

(substring (revision-hex (revision-of-bytes

+491

(string->utf8 (apply string-append op-label parts))))

+492

0 12))

+493

+494

;; staged-list entries: (list rel-path abs-path pre-bytes staged-source)

+495

;; Returns per-file (list rel pre-rev post-rev) after full publication.

+496

(define (publish-file-set! coord txn-id op-label staged-list)

+497

(let* ((root (coordinator-root coord))

+498

(entries

+499

(map (lambda (s)

+500

(let* ((rel (list-ref s 0))

+501

(abs (list-ref s 1))

+502

(pre-bytes (list-ref s 2))

+503

(staged-bytes (string->utf8 (list-ref s 3)))

+504

(dir (path-dirname abs))

+505

(base (path-basename abs))

+506

(temp-name (string-append staging-prefix base "." txn-id ".tmp"))

+507

(backup-name (string-append backup-prefix base "." txn-id))

+508

(pre-rev (revision-of-bytes pre-bytes))

+509

(post-rev (revision-of-bytes staged-bytes)))

+510

;; Backups first, temps second, both hash-verified:

+511

;; an unverified restore artifact is the thing the

+512

;; estate's sabotage-restore discipline exists for.

+513

(write-file-bytes (path-join dir backup-name) pre-bytes)

+514

(unless (equal? (revision-of-file (path-join dir backup-name)) pre-rev)

+515

(error "publish-file-set!: backup hash mismatch after write"))

+516

(write-file-bytes (path-join dir temp-name) staged-bytes)

+517

(unless (equal? (revision-of-file (path-join dir temp-name)) post-rev)

+518

(error "publish-file-set!: staged temp hash mismatch after write"))

+519

(list rel abs pre-rev post-rev

+520

(path-relative root (path-join dir temp-name))

+521

(path-relative root (path-join dir backup-name)))))

+522

staged-list)))

+523

;; THE COMMIT POINT.

+524

(write-transaction-manifest! root txn-id op-label

+525

(map (lambda (e)

+526

(mentry (list-ref e 0) (list-ref e 2)

+527

(list-ref e 4) (list-ref e 3)

+528

(list-ref e 3)

+529

(list-ref e 5) (list-ref e 2)))

+530

entries))

+531

;; Renames. From here a crash is COMPLETED by recovery.

+532

(for-each (lambda (e)

+533

(rename-file (path-join root (list-ref e 4)) (list-ref e 1)))

+534

entries)

+535

(clear-transaction-manifest! root)

+536

(for-each (lambda (e)

+537

(guard (x (#t #f))

+538

(delete-file (path-join root (list-ref e 5)))))

+539

entries)

+540

(map (lambda (e) (list (list-ref e 0) (list-ref e 2) (list-ref e 3)))

+541

entries)))

+542

+543

;; ------------------------------------------------------------------

+544

;; RENAME ID WITH REFERENCE REWRITE

+545

;; ------------------------------------------------------------------

+546

(define (folio-rename-id! coord old-id new-id expected-revision)

+547

(unless (coordinator? coord) (error "folio-rename-id!: coord must be a coordinator"))

+548

(unless (and (string? old-id) (valid-node-id? old-id))

+549

(error "folio-rename-id!: old-id must use flat node-id syntax"))

+550

(unless (and (string? new-id) (valid-node-id? new-id))

+551

(error "folio-rename-id!: new-id must use flat node-id syntax"))

+552

(when (equal? old-id new-id)

+553

(error "folio-rename-id!: old and new ids are identical"))

+554

(coordinator-transact coord

+555

(lambda (recovery-actions)

+556

(%txn-stamp-recovery

+557

(rename-under-lock coord old-id new-id expected-revision)

+558

recovery-actions))))

+559

+560

(define (rename-under-lock coord old-id new-id expected-revision)

+561

(let* ((population (root-population coord))

+562

(census (car population))

+563

(entries (cdr population)))

+564

;; 1. The NEW id must be completely fresh -- no boundary occurrence

+565

;; anywhere, prose included. Conservative by design.

+566

(let ((taken

+567

(filter (lambda (e)

+568

(and (list-ref e 3)

+569

(pair? (boundary-occurrences (utf8->string (list-ref e 3)) new-id))))

+570

entries)))

+571

(if (pair? taken)

+572

(%txn-refuse 'rename-target-taken

+573

(format "new id ~s already occurs in: ~a -- a rename must land in a fresh name"

+574

new-id (map car taken)))

+575

;; 2. Exactly ONE document declares the old id (D9 workspace

+576

;; uniqueness; the estate's live duplicates refuse here rather

+577

;; than being picked between).

+578

(let ((owners

+579

(filter (lambda (e)

+580

(and (eq? (list-ref e 2) 'sgd)

+581

(list-ref e 3)

+582

(let* ((text (utf8->string (list-ref e 3)))

+583

(hits (boundary-occurrences text old-id)))

+584

(and (pair? hits)

+585

(let ((d (%txn-bounded-descriptor

+586

coord (list-ref e 3) text

+587

(list-ref e 0)

+588

(revision-of-bytes (list-ref e 3))

+589

(format "document ~a" (list-ref e 0)))))

+590

(and (descriptor? d)

+591

(pair? (declaration-sites d (list old-id)))))))))

+592

entries)))

+593

(cond

+594

((null? owners)

+595

(%txn-refuse 'unknown-node

+596

(format "no document in this root declares node id ~s" old-id)))

+597

((pair? (cdr owners))

+598

(%txn-refuse 'ambiguous-rename

+599

                                (format "node id ~s is declared in ~a documents (~a); a workspace-ambiguous id cannot be renamed -- D9 presumes uniqueness and this coordinator refuses to pick"

+600

old-id (length owners) (map car owners))))

+601

(else

+602

(rename-with-owner coord old-id new-id expected-revision

+603

entries (car owners)))))))))

+604

+605

(define (rename-with-owner coord old-id new-id expected-revision entries owner)

+606

(let* ((owner-rel (list-ref owner 0))

+607

(owner-bytes (list-ref owner 3))

+608

(owner-rev (revision-of-bytes owner-bytes)))

+609

(cond

+610

;; 3. Structural: exact revision match on the OWNING document,

+611

;; never rebased (amendment 1).

+612

((not (equal? owner-rev expected-revision))

+613

(%txn-refuse 'structural-revision-mismatch

+614

                        (format "expected the owning document ~a at revision ~a but it is at ~a. Structural edits ALWAYS refuse on revision mismatch (D11 amendment 1); reread and re-decide."

+615

owner-rel expected-revision owner-rev)

+616

current-revision: owner-rev

+617

diff: (format "-expected-revision ~a~%+current-revision ~a~%"

+618

expected-revision owner-rev)))

+619

(else

+620

;; 4. Referrer accounting: every raw occurrence must be a

+621

;; rewritable span or a declaration; anything else -- prose,

+622

;; code, query specs, Markdown notes, uninspectable files --

+623

;; refuses the WHOLE rename, absolutely.

+624

(let* ((found (find-referrers coord entries (list old-id)))

+625

(semantic (list-ref found 0))

+626

(raw-only (list-ref found 1))

+627

(troubles (list-ref found 2)))

+628

(cond

+629

((pair? troubles)

+630

(%txn-refuse 'referrer-uninspectable

+631

(format "cannot account for every occurrence of ~s: ~a file(s) could not be inspected: ~a"

+632

old-id (length troubles) troubles)

+633

retryable: #t))

+634

((pair? raw-only)

+635

(%txn-refuse 'unrewritable-reference

+636

                              (format "~s occurs where no rewritable span exists -- ~a -- renaming would leave those stale (t-5e4329's defect class) or rewrite them blind; both refuse, with no override"

+637

old-id raw-only)))

+638

(else

+639

(stage-rename-set coord old-id new-id entries owner semantic))))))))

+640

+641

;; Stage every touched file, verify each, publish through the engine.

+642

(define (stage-rename-set coord old-id new-id entries owner semantic)

+643

(let* ((root (coordinator-root coord))

+644

;; Group semantic reference sites by file; the OWNER also gets

+645

;; its declaration site(s) rewritten.

+646

(touched-rels

+647

(let loop ((rest semantic) (out (list (list-ref owner 0))))

+648

(cond ((null? rest) (reverse out))

+649

((member (car (car rest)) out) (loop (cdr rest) out))

+650

(else (loop (cdr rest) (cons (car (car rest)) out))))))

+651

(staged-or-refusal

+652

(let loop ((rels touched-rels) (staged '()))

+653

(if (null? rels)

+654

(reverse staged)

+655

(let ((s (stage-one-rename-file coord old-id new-id entries

+656

(car rels) semantic)))

+657

(if (txn-refusal? s)

+658

s

+659

(loop (cdr rels) (cons s staged))))))))

+660

(if (txn-refusal? staged-or-refusal)

+661

staged-or-refusal

+662

;; GRAPH CLOSURE over the staged set: zero boundary occurrences

+663

;; of the old id remain anywhere it occurred.

+664

(let ((leak

+665

(filter (lambda (s)

+666

(pair? (boundary-occurrences (list-ref s 3) old-id)))

+667

staged-or-refusal)))

+668

(if (pair? leak)

+669

(%txn-refuse 'postcheck-failed

+670

(format "rename closure failed: ~s still occurs in staged ~a"

+671

old-id (map car leak)))

+672

(let* ((txn-id (content-txn-id "rename-id"

+673

(list old-id new-id

+674

(revision-of-bytes (list-ref owner 3)))))

+675

(published (publish-file-set! coord txn-id "rename-id"

+676

staged-or-refusal)))

+677

(sop-result-record

+678

op: 'rename-id

+679

files: (let zip ((ps published) (ss staged-or-refusal) (out '()))

+680

(if (or (null? ps) (null? ss))

+681

(reverse out)

+682

(let ((p (car ps)) (s (car ss)))

+683

(zip (cdr ps) (cdr ss)

+684

(cons (sop-file (list-ref p 0) (list-ref p 1) (list-ref p 2)

+685

(%txn-make-line-diff

+686

                                                            (utf8->string (list-ref (assoc-entry entries (list-ref p 0)) 3))

+687

(list-ref s 3)))

+688

out)))))

+689

node-id: new-id

+690

detail: (format "renamed ~s -> ~s across ~a file(s); ~a reference site(s) rewritten"

+691

old-id new-id (length touched-rels) (length semantic))

+692

recovery: '())))))))

+693

+694

(define (assoc-entry entries rel)

+695

(let loop ((rest entries))

+696

(cond ((null? rest) #f)

+697

((equal? (car (car rest)) rel) (car rest))

+698

(else (loop (cdr rest))))))

+699

+700

;; Stage one file of the rename: rewrite its declaration site(s) (owner)

+701

;; and reference site(s), rear-most span first so earlier offsets stay

+702

;; valid; verify with the full single-file pipeline.

+703

(define (stage-one-rename-file coord old-id new-id entries rel semantic)

+704

(let* ((entry (assoc-entry entries rel))

+705

(bytes (list-ref entry 3))

+706

(text (utf8->string bytes))

+707

(rev (revision-of-bytes bytes))

+708

(d (%txn-bounded-descriptor coord bytes text rel rev

+709

(format "document ~a" rel))))

+710

(if (txn-refusal? d)

+711

d

+712

(let* ((decl (declaration-sites d (list old-id)))

+713

(refs (let loop ((rest semantic) (out '()))

+714

(cond ((null? rest) (reverse out))

+715

((equal? (car (car rest)) rel)

+716

(loop (cdr rest) (cons (cdr (car rest)) out)))

+717

(else (loop (cdr rest) out)))))

+718

;; Spans to rewrite: each declaration node's ID FIELD (the

+719

;; positional first argument) and each reference site's

+720

;; node field. All must have spans or the file refuses.

+721

(spans

+722

(append

+723

(map (lambda (n)

+724

(let lp ((fs (inspected-node-fields n)))

+725

(cond ((null? fs) #f)

+726

((and (not (field-name (car fs)))

+727

(equal? (field-value (car fs)) old-id))

+728

(field-span (car fs)))

+729

(else (lp (cdr fs))))))

+730

decl)

+731

(map (lambda (site) (field-span (list-ref site 1)))

+732

refs))))

+733

(if (memq #f spans)

+734

(%txn-refuse 'rebase-span-invalid

+735

                               (format "a rewrite site for ~s in ~a has no recorded span; refusing a blind rewrite" old-id rel)

+736

current-revision: rev)

+737

(let* ((replacement (%txn-escape-string-literal new-id))

+738

(ordered (sort-spans-desc spans))

+739

(staged

+740

(let loop ((rest ordered) (s text))

+741

(if (null? rest)

+742

s

+743

(loop (cdr rest)

+744

(%txn-splice s (span-scalar-start (car rest))

+745

(span-scalar-end (car rest))

+746

replacement)))))

+747

;; Verify through the standard pipeline with a

+748

;; coarse byte region spanning all rewrites.

+749

(byte-lo (%txn-scalar->byte-offset text

+750

(span-scalar-start (list-ref ordered (- (length ordered) 1)))))

+751

(byte-hi (%txn-scalar->byte-offset text

+752

(span-scalar-end (car ordered))))

+753

(bad (%txn-verify-staged coord rel bytes text rev

+754

staged (descriptor-ids d)

+755

byte-lo (- byte-hi byte-lo)

+756

0

+757

(lambda (staged-d)

+758

(rename-file-check staged-d rel old-id new-id

+759

(pair? decl) rev))

+760

'postcheck-failed

+761

(descriptor-node-count d)

+762

0

+763

(list old-id))))

+764

(if bad

+765

bad

+766

(list rel (list-ref entry 1) bytes staged))))))))

+767

+768

(define (sort-spans-desc spans)

+769

;; Insertion sort, descending by scalar start; site counts are small.

+770

(let loop ((rest spans) (out '()))

+771

(if (null? rest)

+772

out

+773

(loop (cdr rest)

+774

(let ins ((xs out))

+775

(cond ((null? xs) (list (car rest)))

+776

((> (span-scalar-start (car rest)) (span-scalar-start (car xs)))

+777

(cons (car rest) xs))

+778

(else (cons (car xs) (ins (cdr xs))))))))))

+779

+780

(define (rename-file-check staged-d rel old-id new-id owner? rev)

+781

(cond

+782

((not (eq? (descriptor-id-status staged-d old-id) 'absent))

+783

(%txn-refuse 'postcheck-failed

+784

(format "~s still declared in staged ~a" old-id rel)

+785

current-revision: rev))

+786

((and owner? (not (eq? (descriptor-id-status staged-d new-id) 'unique)))

+787

(%txn-refuse 'postcheck-failed

+788

(format "~s is not uniquely declared in the staged owner ~a" new-id rel)

+789

current-revision: rev))

+790

(else #f)))

+791

792

(define (folio-move-node! . args)

793

(error "folio-move-node!: landing in the next F3b slice"))))