test/f4a-estate-scale.sh

1#!/bin/sh

2# f4a-estate-scale.sh -- NOT A GATE. A scale and shape measurement of

3# `(folio graph)` against a COPY of the live Ops knowledge base.

4#

5# It is deliberately not a gate arm: its corpus is the live estate, which

6# changes daily, so every number it prints is true of one moment and no

7# assertion made against it would be reproducible. `test/gate-f4a.sh` owns the

8# reproducible claims over a frozen 12-file fixture corpus; this script answers

9# the different question of whether the fixture corpus is REPRESENTATIVE and

10# whether the thing survives two thousand real notes.

11#

12# THE LIVE ESTATE IS NEVER WRITTEN TO. Everything happens in a `/tmp` copy.

13# The safety verdict at the bottom asserts what it can actually establish -- the

14# copy is intact, and every change under $SRC during the run is LISTED so a

15# reader can see it was somebody else's note-taking -- rather than refusing to

16# speak because a live corpus changed while being read. See the long comment

17# there for why the original before/after-hash version cried wolf on every run.

18#

19# Usage: sh test/f4a-estate-scale.sh [source-folio-root]

21set -u

22repo_dir=$(cd "$(dirname "$0")/.." && pwd)

23cd "$repo_dir" || exit 1

25SRC="${1:-/home/daviwil/Ops/folio}"

26sigil="${SIGIL:-sigil}"

27export SIGIL_REDIRECTS="${SIGIL_F4A_REDIRECTS:-$repo_dir/dev-redirects.sgl}"

29[ -d "$SRC/notes" ] || { echo "SETUP-FAILED: no notes directory under $SRC"; exit 3; }

30[ -d "$repo_dir/build/dev/lib" ] || { echo "SETUP-FAILED: run \`sigil build\` first"; exit 3; }

32work=$(mktemp -d /tmp/f4a-estate.XXXXXX) || { echo "SETUP-FAILED: no temp dir"; exit 3; }

33trap 'rm -rf "$work"' EXIT INT TERM

35# Fingerprint = EVERY entry under the source -- files, directories and symlinks

36# -- with its type, size and mtime. It moves if anything is created, deleted,

37# truncated, rewritten or re-linked.

38#

39# `-type f` alone was the first version and it had a hole an adversarial review

40# demonstrated: adding a directory AND a symlink left the digest byte-identical.

41fingerprint () {

42 find "$SRC" -printf '%y %P %s %T@\n' 2>/dev/null | sort | sha256sum | cut -d' ' -f1

43}

45# A FLOOR ON THE FINGERPRINT ITSELF, and it is the load-bearing line here.

46#

47# `find` without `-printf` (non-GNU), a missing `sha256sum`, or an unreadable

48# `$SRC` all produce the EMPTY-INPUT digest e3b0c442…52b855 -- on BOTH sides.

49# Two empty digests compare equal, so the script would print "source untouched"

50# and exit 0 having measured nothing at all. That is the exact shape this whole

51# estate keeps paying for, so the fingerprint has to prove it saw something.

52EMPTY_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855

53fingerprint_entries () { find "$SRC" -printf '.\n' 2>/dev/null | wc -l | tr -d ' '; }

55before=$(fingerprint)

56entries=$(fingerprint_entries)

57if [ "$before" = "$EMPTY_SHA" ] || [ "${entries:-0}" -lt 100 ]; then

58 echo "SETUP-FAILED: the fingerprint saw $entries entries (digest $before)." >&2

59 echo " A fingerprint over nothing compares equal to itself and would certify" >&2

60 echo " 'source untouched' without having looked. Check that \$SRC is readable" >&2

61 echo " and that find supports -printf." >&2

62 exit 3

63fi

65W="$work/ws"

66mkdir -p "$W"

67copy_taken=$(date +%s)

68cp -r "$SRC/notes" "$W/notes" || { echo "SETUP-FAILED: copy failed"; exit 3; }

69# A copied index database would make the first open report `intact` over a

70# schema this build may not share.

71find "$W" -name '.folio-*.db*' -delete 2>/dev/null

73# The exit status and the PROBE_DONE marker are BOTH checked, for the reason

74# `gate-f4a.sh` gives at length: piping `sigil` into `grep` makes the pipeline's

75# status grep's, so without these every measurement below could fail and this

76# script would still exit 0 having printed zeroes.

77probe () {

78 po="$work/probe.out"

79 "$sigil" -L "$repo_dir/build/dev/lib" -L "$repo_dir/src" \

80 "$repo_dir/tools/f4a_graph_probe.sgl" "$@" > "$po" 2>&1

81 rc=$?

82 if [ "$rc" -ne 0 ] || ! grep -q '^PROBE_DONE' "$po"; then

83 echo "PROBE FAILED (rc=$rc, no completion marker): $*" >&2

84 grep -v '\[INFO\]' "$po" | tail -8 >&2

85 return 1

86 fi

87 grep -v '\[INFO\]' "$po"

88}

89field () { awk -F'\t' -v n="$2" '$1 == v { print $n }' v="$1"; }

91echo "=========================================================================="

92echo "f4a estate scale measurement -- NOT A GATE"

93echo "=========================================================================="

94echo "source : $SRC (read-only; work happens in $W)"

95echo "compiler : $("$sigil" --version 2>&1 | sed 's/\x1b\[[0-9;]*m//g' | tail -1)"

96echo "files : $(find "$W/notes" -type f | wc -l | tr -d ' ') under notes/"

97echo " .md : $(find "$W/notes" -name '*.md' | wc -l | tr -d ' ')"

98echo " .sgd : $(find "$W/notes" -name '*.sgd' | wc -l | tr -d ' ')"

99echo

100

101start=$(date +%s)

102probe rebuild "$W" ops file > "$work/rebuild.txt" || { echo "REBUILD FAILED"; cat "$work/rebuild.txt"; exit 1; }

103end=$(date +%s)

104docs=$(field DOCS 2 < "$work/rebuild.txt")

105echo "rebuild : $docs documents in $((end - start))s"

106

107probe edges "$W" ops file > "$work/edges.txt"

108probe nodes "$W" ops file > "$work/nodes.txt"

109probe diagnostics "$W" ops file > "$work/diags.txt"

110probe orphans "$W" ops file > "$work/orphans.txt"

111probe unresolved "$W" ops file > "$work/unres.txt"

112probe cycles "$W" ops file > "$work/cycles.txt"

113

114echo "nodes : $(field NODE_COUNT 2 < "$work/nodes.txt")"

115echo "edges : $(field EDGE_COUNT 2 < "$work/edges.txt")"

116echo " by kind :"

117awk -F'\t' '$1 == "EDGE" { print $3 }' "$work/edges.txt" | sort | uniq -c | sort -rn | sed 's/^/ /'

118echo "resolution :"

119awk -F'\t' '$1 == "EDGE" { print $11 }' "$work/edges.txt" | sort | uniq -c | sort -rn | sed 's/^/ /'

120echo "unresolved : $(field UNRESOLVED_COUNT 2 < "$work/unres.txt") declared edges"

121echo "orphans : $(field ORPHAN_COUNT 2 < "$work/orphans.txt") of $docs documents"

122echo "diagnostics: $(field DIAGNOSTIC_COUNT 2 < "$work/diags.txt")"

123awk -F'\t' '$1 == "DIAGNOSTIC" { print $5 }' "$work/diags.txt" | sort | uniq -c | sort -rn | sed 's/^/ /'

124echo "cycles : $(field CYCLE_COUNT 2 < "$work/cycles.txt")"

125grep '^CYCLE ' "$work/cycles.txt" | head -5 | sed 's/^/ /'

126echo

127

128# CONVERGENCE ON REAL DATA. The gate proves this over 12 curated files; this

129# asks whether it survives the real corpus's shapes.

130#

131# IT DRIVES A REAL INCREMENTAL PASS FIRST. The earlier version compared a

132# rebuild against a rebuild, which is a determinism check wearing a convergence

133# check's label -- it could not answer the question its own comment asked. An

134# adversarial review caught the mislabel. Re-indexing every source through the

135# incremental entry point and THEN rebuilding is the comparison that matters.

136echo "convergence: re-indexing incrementally over the whole corpus, then rebuilding..."

137inc_start=$(date +%s)

138inc_failed=0

139# ITERATE THE CENSUS, NOT THE FILESYSTEM. `find -type f` under a real estate

140# turns up things discovery deliberately skips -- `.wrangler/cache/pages.json`

141# and friends -- and `graph-index-document!` correctly REFUSES a path that is

142# not a discoverable source. The first version of this loop fed it every file

143# and collected raises, which was this script being wrong about the product

144# rather than the product being wrong.

145probe sources "$W" ops file | awk -F'\t' '$1 == "SOURCE" { print $2 }' | sort > "$work/sources.txt"

146census_n=$(wc -l < "$work/sources.txt" | tr -d ' ')

147

148# THE INCREMENTAL PASS IS SAMPLED, AND THE SAMPLE IS STATED.

149#

150# `graph-index-document!` runs a full census on every call -- that is what makes

151# it converge, because the two whole-corpus facts (is this slug contested, does

152# this document still exist) cannot be derived from one file. The cost is that

153# incremental indexing is O(corpus) PER CALL, so re-indexing all 2,900 notes one

154# at a time is O(n^2) and does not finish in a useful time.

155#

156# That is a real property worth knowing rather than hiding, so it is measured

157# and reported: a bounded sample with a denominator beats an unbounded pass that

158# never returns. Override with F4A_ESTATE_SAMPLE=0 for all of them.

159sample=${F4A_ESTATE_SAMPLE:-200}

160if [ "$sample" -gt 0 ] && [ "$census_n" -gt "$sample" ]; then

161 head -n "$sample" "$work/sources.txt" > "$work/sample.txt"

162 echo " census: $census_n sources; SAMPLING $sample for the incremental pass"

163 echo " (graph-index-document! runs a full census per call, so a whole-corpus"

164 echo " incremental pass is O(n^2); the rebuild above is the O(n) path)"

165else

166 cp "$work/sources.txt" "$work/sample.txt"

167 echo " census: $census_n sources; indexing ALL of them incrementally"

168fi

169while IFS= read -r f; do

170 probe index "$W" ops file "$f" > /dev/null || { inc_failed=$((inc_failed + 1)); }

171done < "$work/sample.txt"

172inc_end=$(date +%s)

173

echo "             incremental pass: $(wc -l < "$work/sample.txt" | tr -d " ") documents in $((inc_end - inc_start))s, $inc_failed failure(s)"

174probe dump "$W" ops file | grep '^R ' | cut -f2- > "$work/a.txt"

175probe rebuild "$W" ops file > /dev/null

176probe dump "$W" ops file | grep '^R ' | cut -f2- > "$work/b.txt"

177la=$(wc -l < "$work/a.txt" | tr -d ' '); lb=$(wc -l < "$work/b.txt" | tr -d ' ')

178if [ "$la" -eq 0 ] || [ "$lb" -eq 0 ]; then

179 echo " INCONCLUSIVE: a dump was empty ($la / $lb); two empty dumps compare equal"

180elif diff -q "$work/a.txt" "$work/b.txt" > /dev/null 2>&1; then

181 echo " incremental-vs-rebuild IDENTICAL over $la relation lines"

182else

183 echo " DIVERGED over $la/$lb relation lines:"

184 diff -u "$work/a.txt" "$work/b.txt" | head -20 | sed 's/^/ /'

185fi

186

187# ==========================================================================

188# THE SAFETY VERDICT

189# ==========================================================================

190#

191# WHAT THIS SCRIPT ACTUALLY CLAIMS: it never writes under $SRC. The first

192# version tried to establish that with a before/after fingerprint of the source

193# tree, and refused to report its numbers if the fingerprint moved.

194#

195# THAT GUARD FIRED ON EVERY RUN, and it was RIGHT about the fact and WRONG about

196# the conclusion. $SRC is a live knowledge base that four agents write to

197# continuously; during one two-hour run, 24 files changed -- notes, projects,

198# inbox, the FTS index -- written by two other workers, the leader, and this

199# worker's own note-taking. NONE of them was anywhere this script writes.

200#

201# The expectation was wrong, not the instrument: it assumed a QUIESCENT source,

202# and that assumption was never written down so it was never checked. A

203# before/after hash CANNOT ATTRIBUTE A CHANGE TO A WRITER, so it cannot answer

204# the question this script is actually asking.

205#

206# And a guard that cries wolf on every run is worse than no guard: a false red

207# that fires occasionally gets investigated, one that fires ALWAYS gets a

208# learned exception -- and the exception is indistinguishable from the real

209# detection it was built for. See

210# [[topics/a-wrong-expectation-is-a-false-red]].

211#

212# SO IT NOW ASSERTS WHAT IT CAN, AND REPORTS WHAT IT CANNOT:

213#

214# 1. the /tmp COPY is intact and non-empty -- a real precondition

215# 2. no path under $SRC is newer than the copy -- ATTRIBUTED, see below

216# 3. foreign writes during the window are REPORTED, not fatal

217#

218# Check 2 is the one that carries the claim. Every file this script could

219# possibly have written would have to be newer than the moment the copy was

220# taken AND lie under $SRC. Listing those explicitly turns "something changed"

221# into "here is exactly what changed, decide for yourself" -- which is the

222# print-the-list-not-the-count lesson from the same night.

223after=$(fingerprint)

224echo

225

226copy_files=$(find "$W/notes" -type f 2>/dev/null | wc -l | tr -d ' ')

227if [ "${copy_files:-0}" -lt 100 ]; then

228 echo "SETUP-FAILED: the working copy holds only $copy_files files; nothing above was measured"

229 exit 3

230fi

231echo "working copy : $copy_files files, intact"

232

233if [ "$before" = "$after" ]; then

234 echo "source : fingerprint unchanged ($before)"

235 echo

236 echo "SAFE: no writes to $SRC, and no foreign writes during the window."

237 exit 0

238fi

239

240# The fingerprint moved. Say WHAT moved rather than refusing to speak.

241echo "source : fingerprint MOVED ($before -> $after)"

242echo " $SRC is a LIVE corpus; other agents write to it continuously."

243echo

244echo "changed under $SRC during this run (newer than the copy):"

245find "$SRC" -newermt "@$copy_taken" -type f 2>/dev/null \

246 | sed "s|^$SRC/| |" | sort | head -40

247n=$(find "$SRC" -newermt "@$copy_taken" -type f 2>/dev/null | wc -l | tr -d ' ')

248echo " ($n file(s))"

249echo

250echo "FOREIGN WRITES DETECTED; THE NUMBERS ABOVE DESCRIBE THE SNAPSHOT."

251echo " This script writes only under /tmp. If every path listed above is an"

252echo " ordinary folio write (notes/, projects/, inbox.md, .folio-index.db*),"

253echo " that is another agent using the knowledge base and the measurements"

254echo " stand -- they ran against a copy taken before those writes."

255echo " If any listed path is one THIS script could have produced, stop and"

256echo " investigate: that would be the thing the check exists for."

257exit 0