test/test-shared-index.sgl

1(import (sigil async)

2 (sigil test)

3 (sigil time)

4 (sigil fs)

5 (sigil path)

6 (sigil sqlite)

7 (sigil io)

8 (sigil log)

9 (sigil process)

10 (sigil string)

11 (sigil mcp server)

12 (sigil mcp protocol)

13 (folio store)

14 (folio note)

15 (folio index)

16 (folio tools))

18(test-group "incremental index transactions"

19 (test "unchanged notes are not reindexed and failed replacement rolls back"

20 (call-with-temp-directory

21 (lambda (root)

22 (let* ((store (make-folio-store root))

23 (one (path-join (store-notes-dir store) "one.md"))

24 (two (path-join (store-notes-dir store) "two.md")))

25 (write-file-string one "# One\nbeforeword\n")

26 (write-file-string two "# Two\nuntouchedword\n")

27 (let* ((idx (make-folio-index store))

28 (db (folio-index-db idx)))

29 (index-reconcile! idx)

30 (assert-true

31 (sqlite-run db "UPDATE notes_meta SET indexed_at = 123 WHERE name = 'two'"))

32 (assert-true

33 (sqlite-exec db

34 "CREATE TRIGGER reject_one BEFORE INSERT ON notes_meta

35 WHEN NEW.name = 'one'

36 BEGIN SELECT RAISE(ABORT, 'injected index failure'); END"))

37 (write-file-string one "# One\nafterword\n")

38 (let ((failed #f))

39 (guard (exn (else (set! failed #t)))

40 (index-note! idx one))

41 (assert-true failed))

42 ;; Both tables still describe the old document, despite DELETEs

43 ;; having run before the deliberately failed metadata INSERT.

44 (assert-equal 1 (length (search-index idx "beforeword" #f #f)))

45 (assert-equal 0 (length (search-index idx "afterword" #f #f)))

46 (assert-true (sqlite-exec db "DROP TRIGGER reject_one"))

47 (index-reconcile! idx)

48 (assert-equal 0 (length (search-index idx "beforeword" #f #f)))

49 (assert-equal 1 (length (search-index idx "afterword" #f #f)))

50 (assert-true

51 (= 123

52 (cdr (assoc 'indexed_at

53 (car (sqlite-query db

54 "SELECT indexed_at FROM notes_meta WHERE name = 'two'"))))))

55 (sqlite-close db)))))))

57(test-group "post-write index recovery"

58 (test "a committed append succeeds despite index failure and census heals it"

59 (call-with-temp-directory

60 (lambda (root)

61 (let* ((store (make-folio-store root))

62 (path (note-create! store "one" '() "# One\nbeforeword\n"))

63 (idx (make-folio-index store))

64 (db (folio-index-db idx))

65 (server (mcp-server name: "index-recovery-test")))

66 (index-reconcile! idx)

67 (register-folio-tools! server store idx)

68 (assert-true

69 (sqlite-exec db

70 "CREATE TRIGGER reject_one BEFORE INSERT ON notes_meta

71 WHEN NEW.name = 'one'

72 BEGIN SELECT RAISE(ABORT, 'injected index failure'); END"))

73 (let ((response

74 (mcp-server-handle-message server

75 (jsonrpc-request

76 id: 1

77 method: "tools/call"

78 params: (dict name: "folio/note-edit"

79 arguments: (dict name: "one"

80 append: "afterword"))))))

81 (assert-true (jsonrpc-response? response))

82 (assert-true (string-contains? (response->json response)

83 "Appended to note")))

84 (assert-equal 2

85 (length (string-split (folio-note-body (read-note path)) "afterword")))

86 (assert-equal 0 (length (search-index idx "afterword" #f #f)))

87 (assert-true (sqlite-exec db "DROP TRIGGER reject_one"))

88 ;; No second edit or retry: unchanged bytes must be recovered.

89 (index-reconcile! idx)

90 (assert-equal 1 (length (search-index idx "afterword" #f #f)))

91 (sqlite-close db)))))

93 (test "unreadable files log once until their fingerprint changes"

94 (call-with-temp-directory

95 (lambda (root)

96 (let* ((store (make-folio-store root))

97 (path (note-create! store "unreadable" '() "# Recovered\nhealedword\n"))

98 (idx (make-folio-index store))

99 (output (open-output-string)))

100 (dynamic-wind

101 (lambda ()

102 (assert-equal 0 (process-run "chmod" "000" path))

103 (log-configure! target: output level: 'warn))

104 (lambda ()

105 (index-reconcile! idx)

106 (let ((first (get-output-string output)))

107 (assert-true (string-contains? first "Search index refresh failed"))

108 (index-reconcile! idx)

109 (index-reconcile! idx)

110 (assert-equal first (get-output-string output)))

111 (assert-equal 0 (process-run "chmod" "600" path))

112 (index-reconcile! idx)

113 (assert-equal 1 (length (search-index idx "healedword" #f #f))))

114 (lambda ()

115 (process-run "chmod" "600" path)

116 (log-configure! target: 'console level: 'info)

117 (sqlite-close (folio-index-db idx)))))))))

118

119(test-group "concurrent tasks own complete index transactions"

120 (test "a census commit yield cannot absorb another task's failed replacement"

121 (call-with-temp-directory

122 (lambda (root)

123 (let* ((store (make-folio-store root))

124 (census-file (note-create! store "census" '() "# Census\ncensusoldword\n"))

125 (request-file (note-create! store "request" '() "# Request\nrequestoldword\n"))

126 (idx (make-folio-index store))

127 (db (folio-index-db idx))

128 (reader #f)

129 (overlapped? #f)

130 (request-started? #f)

131 (request-failed? #f)

132 (census-error #f))

133 (index-reconcile! idx)

134 ;; A reader in rollback-journal mode forces the writer's COMMIT

135 ;; to yield, after BEGIN and all of its per-document statements.

136 (assert-true (sqlite-exec db "PRAGMA journal_mode=DELETE"))

137 (assert-true

138 (sqlite-exec db

139 "CREATE TRIGGER reject_request BEFORE INSERT ON notes_meta

140 WHEN NEW.name = 'request'

141 BEGIN SELECT RAISE(ABORT, 'injected concurrent failure'); END"))

142 (set! reader (sqlite-open (path-join root ".folio-index.db")))

143 (assert-true (sqlite-exec reader "BEGIN"))

144 (sqlite-query reader "SELECT name FROM notes_meta")

145 (write-file-string census-file "# Census\ncensusnewword\n")

146 (with-async

147 (go

148 (guard (exn (else (set! census-error exn)))

149 (index-reconcile! idx)))

150 (go

151 (let ((deadline (+ (current-second) 2)))

152 (let wait ()

153 (cond

154 ((sqlite-connection-locked? db)

155 (set! overlapped? #t))

156 ((< (current-second) deadline)

157 (sleep 0.001)

158 (wait)))))

159 (set! request-started? #t)

160 (write-file-string request-file "# Request\nrequestnewword\n")

161 (guard (exn (else (set! request-failed? #t)))

162 (index-note! idx request-file)))

163 (go

164 (let ((deadline (+ (current-second) 2)))

165 (let wait ()

166 (when (and (not request-started?)

167 (< (current-second) deadline))

168 (sleep 0.001)

169 (wait))))

170 ;; Let the second task enter or wait for its transaction.

171 (sleep 0.02)

172 (sqlite-exec reader "ROLLBACK")))

173 (sqlite-close reader)

174 (assert-true overlapped?)

175 (assert-false census-error)

176 (assert-true request-failed?)

177 ;; The failed writer must leave all three old rows intact, even

178 ;; when another task was waiting inside a yielding COMMIT.

179 (assert-equal 2

180 (cdr (assoc 'total

181 (car (sqlite-query db

182 "SELECT count(*) AS total FROM notes_versions v

183 JOIN notes_meta m USING(name)

184 JOIN notes_fts f USING(name)")))))

185 (assert-equal 1 (length (search-index idx "requestoldword" #f #f)))

186 (assert-equal 1 (length (search-index idx "censusnewword" #f #f)))

187 (assert-true (sqlite-exec db "DROP TRIGGER reject_request"))

188 (index-reconcile! idx)

189 (assert-equal 1 (length (search-index idx "requestnewword" #f #f)))

190 (assert-equal 0 (length (search-index idx "requestoldword" #f #f)))

191 (sqlite-close db))))))