test/f2b-memory-ladder.sh

1#!/bin/sh

2# f2b-memory-ladder.sh -- at what address-space cap does a given worker shape

3# stop being able to parse a manuscript?

4#

5# RUN: sh test/f2b-memory-ladder.sh [reps]

6#

7# WHY THIS IS A SEPARATE SCRIPT AND NOT A ONE-LINER

8#

9# The question looks like "walk `ulimit -v` down until it fails". Two things

10# make the naive version report the machine instead of the worker.

11#

12# 1. WHICH BOUND FIRED MUST BE RECORDED, NOT INFERRED FROM FAILURE.

13#

14# A run that fails tells you nothing on its own. It may have exhausted the

15# address-space cap -- the thing being measured -- or died for an unrelated

16# reason. If the invocation had a WALL CLOCK in it (this one deliberately does

17# not; see below), CPU contention could make a note that would have completed

18# trip the timeout instead, and a naive reading records that as "the memory cap

19# bit at this rung". That is measuring load and reporting it as a memory

20# threshold -- one instrument answering two questions, the same defect class as

21# a preflight that cannot tell "did not start" from "did not parse".

22#

23# So every rung records the EXIT STATUS and whether an out-of-memory signature

24# appeared, and prints them. A failure with no OOM signature is reported as

25# `other`, never folded into the threshold.

26#

27# NOTE this ladder invokes the worker DIRECTLY under a bare `ulimit -v`: no

28# `timeout(1)`, no `run-bounded`, therefore no wall clock at all. That is

29# deliberate -- it removes the confound rather than trying to correct for it.

30# The trade-off is that a pathological input could hang this script; the

31# fixture is a well-formed note of known size, so that is bounded in practice.

32#

33# 2. ONE SAMPLE PER RUNG IS NOISE.

34#

35# Behaviour near an address-space threshold is not deterministic -- mapping

36# layout and allocator state move it between runs. Two single probes taken

37# earlier disagreed about the bundled worker's threshold (one had it parsing

38# at 300 MB, another failing at 400 MB), which is exactly what a boundary looks

39# like when sampled once. Each rung is therefore repeated and reported as a

40# SUCCESS COUNT, so a threshold shows up as a gradient rather than a cliff and

41# the noise is visible instead of averaged away.

42#

43# THE PRECONDITION FOR THIS MEASUREMENT CLASS IS MEMORY HEADROOM, NOT CPU LOAD.

44#

45# `test/f2b-spawn-cost.sgl` refuses to run above a CPU load-average ceiling,

46# because wall-clock timing is load-sensitive: a competing job stretches every

47# sample and the number becomes unreproducible.

48#

49# THAT CEILING DOES NOT BELONG HERE, and applying it would refuse correct

50# measurements -- which is how a good check gets deleted by someone in a hurry.

51# `ulimit -v` is a PER-PROCESS ADDRESS SPACE limit. It does not move with system

52# CPU load, and this ladder has no wall clock for contention to trip. A CPU-heavy

53# neighbour makes a rung SLOWER; it does not change which rung fails.

54#

55# What WOULD invalidate this ladder is genuine MEMORY pressure: if the machine is

56# near its limit or swapping, an allocation can fail for reasons that have

57# nothing to do with the cap under test, and the threshold moves. So that is what

58# is checked, and it refuses rather than warns -- same discipline as the timing

59# harness, different quantity, because the confound is different.

60#

61# The CPU load is still RECORDED at both ends. An attributable measurement states

62# its conditions even when those conditions do not affect it; "load was 4.6 and

63# that does not apply to this measurement class because the cap is per-process"

64# is a stronger record than silence.

65#

66# /proc is read through a subprocess: `read-file-string` and anything else that

67# sizes a buffer from `stat` gets "" from procfs, which stats as size 0.

68mem_floor_kb="${F2B_MEM_FLOOR_KB:-8388608}" # 8 GB available, default

69avail_kb=$(awk '/^MemAvailable:/{print $2}' /proc/meminfo 2>/dev/null)

70swap_used_kb=$(awk '/^SwapTotal:/{t=$2} /^SwapFree:/{f=$2} END{print (t-f)+0}' /proc/meminfo 2>/dev/null)

72if [ -z "$avail_kb" ]; then

  echo "SETUP-FAILED: could not read MemAvailable from /proc/meminfo; memory headroom is an input to this measurement and it is unavailable" >&2

74 exit 3

75fi

76if [ "$avail_kb" -lt "$mem_floor_kb" ]; then

77 echo "SETUP-FAILED: MemAvailable ${avail_kb} kB is below the floor ${mem_floor_kb} kB." >&2

78 echo " Under memory pressure an allocation can fail for reasons unrelated to the cap under test," >&2

79 echo " so the threshold this ladder reports would be the machine's, not the worker's." >&2

80 exit 3

81fi

82if [ "$swap_used_kb" -gt 0 ]; then

83 echo "NOTE: ${swap_used_kb} kB of swap is in use; thresholds near the boundary may be less stable." >&2

84fi

86set -u

87repo_dir=$(cd "$(dirname "$0")/.." && pwd)

88cd "$repo_dir" || exit 1

90sigil="${SIGIL:-sigil}"

92# ABSOLUTE path, resolved once.

93#

94# A bare name is resolved through PATH at exec time, and the first version of this

95# ladder ALSO wrapped every invocation in `env -i PATH=/usr/bin:/bin`. `sigil` is

96# not on that PATH, so EVERY dev rung exited 127 at EVERY cap -- including caps

97# where the bundled arm parsed fine. Read naively that says "the dev shape cannot

98# parse this note at any address-space limit", which is a harness artifact wearing

99# a measurement's clothes.

100#

101# It was caught by the ok/oom/other split: the failures landed in `other` with no

102# OOM signature, rather than being counted as the threshold. That split existed

103# only because the leader pushed for failure attribution -- without it this would

104# have been reported as a memory profile.

105sigil_abs=$(command -v "$sigil" 2>/dev/null) \

106 || { echo "SETUP-FAILED: cannot resolve '$sigil' to an absolute path" >&2; exit 3; }

107redirects="$repo_dir/dev-redirects.sgl"

108export SIGIL_REDIRECTS="$redirects"

109reps="${1:-3}"

110

111bundled="$repo_dir/build/dev/bin/folio-doc-worker"

112[ -e "$bundled" ] || { echo "SETUP-FAILED: $bundled missing; run \`sigil build\` first" >&2; exit 3; }

113

114work=$(mktemp -d /tmp/f2b-memladder.XXXXXX) || exit 3

115trap 'rm -rf "$work"' EXIT INT TERM

116

117note="$work/mem.sgd"

118{ printf '#lang sigil/doc\n@note["topics/m" title: "M"]\n\n@p{'

119 head -c 1000000 /dev/zero | tr '\0' 'z'

120 printf '}\n\n@end note\n'; } > "$note"

121

122echo "=========================================================================="

123echo "f2b memory ladder"

124echo " note : $(stat -c%s "$note") bytes"

125echo " reps per rung : $reps"

126echo " sigil (CLI) : $("$sigil" --version 2>&1 | sed 's/\x1b\[[0-9;]*m//g')"

127echo " ../sigil : $(git -C "$repo_dir/../sigil" rev-parse --short HEAD 2>/dev/null || echo UNKNOWN)"

128echo " ../sigil-doc : $(git -C "$repo_dir/../sigil-doc" rev-parse --short HEAD 2>/dev/null || echo UNKNOWN)"

129echo " MemAvailable : $avail_kb kB (floor $mem_floor_kb -- THE gating precondition here)"

130echo " swap in use : $swap_used_kb kB"

131echo " CPU load start : $(cut -d' ' -f1-3 /proc/loadavg) (RECORDED, NOT GATED -- see header)"

132echo " NO wall clock in this path: bare \`ulimit -v\`, no timeout(1), no run-bounded"

133echo "=========================================================================="

134

135# $1 = label, $2 = cap kB, $3.. = argv

136run_rung () {

137 label=$1; cap=$2; shift 2

138 ok=0; oom=0; noexec=0; other=0; statuses=""

139 i=0

140 while [ "$i" -lt "$reps" ]; do

141 i=$((i + 1))

142 out="$work/rung.out"

143 # No `env -i` here. Proving the bundled worker needs nothing on PATH is the

144 # GATE's job (its bundled preflight); this ladder measures ADDRESS SPACE, and

145 # stripping the environment only guaranteed the dev arm could not exec at all.

146 : > "$out" # truncate: a rep that produces no output must not inherit the last rep's

147 ( ulimit -v "$cap"; "$@" "$note" > "$out" 2>&1 )

148 rc=$?

149 statuses="$statuses $rc"

150 # FOUR OUTCOMES, because three of them are failures that mean different things.

151 #

152 # ok parsed successfully under this cap

153 # oom the allocator refused -- THIS is the threshold being measured

154 # noexec the program could not be started at all (exit 127 or 126). Under a

155 # cap the kernel cannot map a 31 MB binary, and `env`/the shell

156 # reports that as "not found" -- indistinguishable by status from a

157 # genuinely missing program. Either way NO PARSE WAS ATTEMPTED, so

158 # folding it into `oom` would report "the cap bit during parsing"

159 # when nothing ever ran.

160 # other anything else, never folded into the threshold.

161 if [ "$rc" -eq 0 ] && grep -q '^PARSED yes' "$out"; then

162 ok=$((ok + 1))

163 elif grep -qiE 'out of memory|cannot allocate|memory exhausted' "$out"; then

164 oom=$((oom + 1))

165 elif [ "$rc" -eq 127 ] || [ "$rc" -eq 126 ]; then

166 noexec=$((noexec + 1))

167 else

168 other=$((other + 1))

169 fi

170 done

171 printf ' %-9s cap %-8s ok=%s/%s oom=%s noexec=%s other=%s exits:%s\n' \

172 "$label" "$cap" "$ok" "$reps" "$oom" "$noexec" "$other" "$statuses"

173}

174

175for cap in 600000 500000 450000 400000 350000 300000 250000 200000; do

176 run_rung bundled "$cap" "$bundled"

177 run_rung dev "$cap" "$sigil_abs" -L build/dev/lib -L src tools/f2b_inspect_worker.sgl

178done

179

180echo " CPU load end : $(cut -d" " -f1-3 /proc/loadavg)"

181echo " MemAvailable end: $(awk "/^MemAvailable:/{print \$2}" /proc/meminfo) kB"

182echo " ../sigil at end : $(git -C "$repo_dir/../sigil" rev-parse --short HEAD 2>/dev/null || echo UNKNOWN)"

183echo " ../sigil-doc end: $(git -C "$repo_dir/../sigil-doc" rev-parse --short HEAD 2>/dev/null || echo UNKNOWN)"

184echo "MEMLADDER-DONE"