Cache status's index work on the notes listing, data_version and total

Message

index-note-summaries and index-orphan-names keep their result while the native notes/ listing, PRAGMA data_version (another connection's commit) and total_changes() (this connection's own writes) are unchanged. The listing keeps today's freshness: the summaries are filtered by a walk of notes/, so a note deleted outside folio still leaves status at once. A listing with a symlink the walk follows, or a failed query, is not cached.

folio-status memoizes the sorted recent notes and the orphan list on the identity of those two lists.

No index or file format change.

Changed

 CHANGELOG.md        |  5 +++++
 package.sgl         |  2 +-
 src/folio/index.sgl | 94 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++----------------------
 src/folio/query.sgl | 36 ++++++++++++++++++++++++++----------
 4 files changed, 104 insertions(+), 33 deletions(-)

Diff

CHANGELOG.mdmodified

@@ -5,6 +5,11 @@ All notable changes to **folio** are documented in this file.

The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),

and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

## [0.3.3] - Unreleased

+10

- `status` reuses its index work while nothing it reads has changed. The note summaries and orphan names, with the sorted recent notes and the orphan list derived from them, are kept while the native `notes/` listing, SQLite's `data_version` (another connection's commit) and `total_changes()` (this connection's own writes) are unchanged, so a note deleted outside folio still leaves `status` at once. A `notes/` symlink the walk follows, or a failed query, turns the reuse off.

+11

- No index or file format change: 0.3.1, 0.3.2 and 0.3.3 processes share one store and index.

+12

## [0.3.2] - Unreleased

- Skip the per-file census on idle ticks. Each 15 s tick compares one native metadata listing of `notes/` and `projects/` (size, mtime-ns and ctime-ns of every entry) and SQLite's `data_version` with the last complete census, and runs the full census only when something differs, another connection committed, or a retryable failure is pending. A listing is trusted only once its newest mtime and ctime are 2 s old, so writes inside one coarse timestamp granule cannot hide. A symlink the walk would follow (any in `projects/`, any not under a dot component in `notes/`), a metadata query that fails, or a listing line that does not parse as 7 fields disables the skip. On a 4,467-note corpus an idle tick drops from 216-611 ms of CPU to about 24 ms.

package.sglmodified

@@ -6,7 +6,7 @@

(package

name: "folio"

−9

version: "0.3.2"

version: "0.3.3"

sigil: "0.22.15"

description: "Markdown-native knowledge and project management via MCP"

url: "https://codeberg.org/sigil/folio"

src/folio/index.sglmodified

@@ -47,7 +47,9 @@

(failed-fingerprints)

(transaction-mutex)

;; Box: what the last complete census saw (see census-state), or #f.

−50

(census-state))

+50

(census-state)

+51

;; Box: (key . #(summaries orphan-names)) for status, or #f.

+52

(summary-cache))

(define (checked-sql result)

(unless result (error "Folio index SQL operation failed")))

@@ -147,7 +149,8 @@

149

fingerprints: (vector versions)

150

failed-fingerprints: (vector (dict))

151

transaction-mutex: mutex

−150

census-state: (vector #f)))))

+152

census-state: (vector #f)

+153

summary-cache: (vector #f)))))

154

155

;; ============================================================

156

;; Write-through operations

@@ -342,26 +345,70 @@

345

"SELECT s.path, s.name, s.kind FROM note_links l

346

JOIN link_sources s ON s.path = l.source WHERE l.target = ?" name))))

347

+348

;; Status reads note summaries and orphan names from the index; the

+349

;; summaries are also filtered and ordered by a walk of notes/. Both are

+350

;; reused while the notes/ listing, data_version (another connection's

+351

;; commit) and total_changes() (this connection's own writes) are all

+352

;; unchanged. A listing with a symlink the walk follows cannot vouch for

+353

;; what is behind it, so then nothing is cached. Returns the key or #f.

+354

(define (summary-cache-key idx)

+355

(guard (exn (else #f))

+356

(let* ((listing (file-tree-metadata (store-notes-dir (folio-index-store idx))))

+357

(db (folio-index-db idx))

+358

(dv (sqlite-query-row db "PRAGMA data_version"))

+359

(tc (sqlite-query-row db "SELECT total_changes() AS n")))

+360

(and listing dv tc

+361

(listing-settled? listing #f #t)

+362

(list listing (cdr (assoc 'data_version dv)) (cdr (assoc 'n tc)))))))

+363

+364

;; compute returns (cacheable? . value). The key is read first, so a cached

+365

;; value is never older than its key.

+366

(define (cached-summary idx slot compute)

+367

(let* ((key (summary-cache-key idx))

+368

(box (folio-index-summary-cache idx))

+369

(entry (vector-ref box 0))

+370

(current (and key entry (equal? (car entry) key) (cdr entry))))

+371

(if (and current (vector-ref current slot))

+372

(car (vector-ref current slot))

+373

(let ((result (compute)))

+374

(when (and key (car result))

+375

(let ((slots (or current (make-vector 2 #f))))

+376

(vector-set! slots slot (list (cdr result)))

+377

(vector-set! box 0 (cons key slots))))

+378

(cdr result)))))

+379

+380

;; Rows, and whether the query ran: sqlite-query answers '() or a partial

+381

;; result on failure, which must not be cached.

+382

(define (summary-rows idx sql)

+383

(let* ((db (folio-index-db idx))

+384

(rows (sqlite-query db sql)))

+385

(cons (equal? (sqlite-errmsg db) "not an error") rows)))

+386

387

;; Status needs names and dates, not the contents of every note.

388

(define (index-note-summaries idx)

−347

(map

−348

(lambda (row)

−349

(folio-note name: (cdr (assoc 'name row))

−350

file-path: (cdr (assoc 'path row))

−351

frontmatter: (dict last-updated: (cdr (assoc 'last_updated row)))))

−352

(order-source-rows idx

−353

(sqlite-query (folio-index-db idx)

−354

"SELECT s.path, m.name, m.last_updated FROM notes_meta m

−355

JOIN link_sources s ON s.name = m.name AND s.kind = 'note'"))))

+389

(cached-summary idx 0

+390

(lambda ()

+391

(let ((result (summary-rows idx

+392

"SELECT s.path, m.name, m.last_updated FROM notes_meta m

+393

JOIN link_sources s ON s.name = m.name AND s.kind = 'note'")))

+394

(cons (car result)

+395

(map

+396

(lambda (row)

+397

(folio-note name: (cdr (assoc 'name row))

+398

file-path: (cdr (assoc 'path row))

+399

frontmatter: (dict last-updated: (cdr (assoc 'last_updated row)))))

+400

(order-source-rows idx (cdr result))))))))

401

402

(define (index-orphan-names idx)

−358

(map (lambda (row) (cdr (assoc 'name row)))

−359

(sqlite-query (folio-index-db idx)

−360

"SELECT s.name FROM link_sources s WHERE s.kind = 'note'

−361

AND NOT EXISTS (SELECT 1 FROM note_links l WHERE l.source = s.path)

−362

AND NOT EXISTS (SELECT 1 FROM note_links l

−363

JOIN link_sources incoming ON incoming.path = l.source

−364

WHERE incoming.kind = 'note' AND l.target = s.name)")))

+403

(cached-summary idx 1

+404

(lambda ()

+405

(let ((result (summary-rows idx

+406

"SELECT s.name FROM link_sources s WHERE s.kind = 'note'

+407

AND NOT EXISTS (SELECT 1 FROM note_links l WHERE l.source = s.path)

+408

AND NOT EXISTS (SELECT 1 FROM note_links l

+409

JOIN link_sources incoming ON incoming.path = l.source

+410

WHERE incoming.kind = 'note' AND l.target = s.name)")))

+411

(cons (car result) (map (lambda (row) (cdr (assoc 'name row))) (cdr result)))))))

412

413

(define (path-signature path)

414

(format "~s" (file-stat-fingerprint path)))

@@ -579,9 +626,10 @@

626

;; or a tab cannot hide an entry; no line may be a symlink the store walk

627

;; would follow (the walk follows them, the listing does not show their

628

;; target's changes); and every mtime and ctime must be older than

−582

;; cutoff-ns. hidden-exempt? is true only for notes/, whose walk skips

−583

;; every dot component; the projects/ walk reads dotfiles too, so any

−584

;; symlink there counts.

+629

;; cutoff-ns, unless cutoff-ns is #f (the status cache reads only names,

+630

;; which no same-granule write can hide). hidden-exempt? is true only for

+631

;; notes/, whose walk skips every dot component; the projects/ walk reads

+632

;; dotfiles too, so any symlink there counts.

633

(define (listing-settled? listing cutoff-ns hidden-exempt?)

634

(let loop ((lines (string-split listing "\n")))

635

(cond

@@ -594,7 +642,9 @@

642

(not (and hidden-exempt? (any-hidden-component? (car fields))))))

643

(let ((mtime (string->number (list-ref fields 4)))

644

(ctime (string->number (list-ref fields 5))))

−597

(and mtime ctime (< mtime cutoff-ns) (< ctime cutoff-ns)))

+645

(and mtime ctime

+646

(or (not cutoff-ns)

+647

(and (< mtime cutoff-ns) (< ctime cutoff-ns)))))

648

(loop (cdr lines))))))))

649

650

(define (any-hidden-component? relative-path)

src/folio/query.sglmodified

@@ -93,7 +93,11 @@

(and due (string<? due today)))))

tasks))

(notes (if idx (index-note-summaries idx) (all-notes store)))

−96

(recent-notes (take-up-to (sort-by-last-updated notes) 5))

+96

(derived (if idx

+97

(recent-and-orphaned notes (index-orphan-names idx))

+98

(list (take-up-to (sort-by-last-updated notes) 5)

+99

(find-orphaned-notes store notes))))

+100

(recent-notes (car derived))

101

;; Stale projects: active but last-updated > 14 days ago

102

(stale-projects

103

(filter (lambda (p)

@@ -102,15 +106,7 @@

106

(and lu (days-ago? lu 14 today))))

107

active-projects))

108

;; Orphaned notes: no incoming or outgoing links

−105

(orphaned

−106

(if idx

−107

(let ((names (dict)))

−108

(for-each

−109

(lambda (name) (set! names (dict-set names name #t)))

−110

(index-orphan-names idx))

−111

(filter (lambda (note) (dict-contains? names (folio-note-name note)))

−112

notes))

−113

(find-orphaned-notes store notes))))

+109

(orphaned (cadr derived)))

110

(dict inbox-count: (length inbox-tasks)

111

overdue-count: (length overdue)

112

overdue-tasks: overdue

@@ -124,6 +120,26 @@

120

total-tasks: (length tasks)

121

total-notes: (length notes))))

122

+123

;; Sorting every summary and filtering the orphans is most of what status

+124

;; costs once the index answers are cached. The index hands back the same

+125

;; list objects while nothing changed, so the derived lists are kept for

+126

;; exactly those objects (eq?), and any change yields new ones.

+127

(define status-derived (vector #f))

+128

+129

(define (recent-and-orphaned notes orphan-names)

+130

(let ((memo (vector-ref status-derived 0)))

+131

(if (and memo (eq? (car memo) notes) (eq? (cadr memo) orphan-names))

+132

(cddr memo)

+133

(let* ((names (dict))

+134

(_ (for-each (lambda (name) (set! names (dict-set names name #t)))

+135

orphan-names))

+136

(derived (list (take-up-to (sort-by-last-updated notes) 5)

+137

(filter (lambda (note)

+138

(dict-contains? names (folio-note-name note)))

+139

notes))))

+140

(vector-set! status-derived 0 (cons notes (cons orphan-names derived)))

+141

derived))))

+142

143

;; Check if a date string is more than N days before today.

144

(define (days-ago? date-str n today-str)

145

(let ((cutoff (n-days-before-today n)))