test/test-cli-contract.sh

1#!/usr/bin/env bash

2# CLI contract regression test.

3#

4# Two defects, both measured against folio 0.2.10 before the fix:

5#

6# 1. `folio <anything>` silently ignored the unknown argument and started the

7# MCP server anyway, EXITING 0. So `folio migrate-buried` -- the first

8# thing anyone types after reading about buried-id recovery -- appeared to

9# run and do nothing, and a typo in a supervisor config was invisible.

10# A silent success for an operation that did not happen.

11#

12# 2. `folio --help` required FOLIO_ROOT and exited 1 without printing help.

13# Help is what you reach for when you do not yet know how to configure the

14# thing, so requiring configuration to read it is backwards.

15#

16# The third and fourth arms are the ones that keep this fix from being worse

17# than the bug: `serve` and the real flags MUST keep working. `serve` was never

18# a subcommand -- the old loop ignored it -- but every MCP config in the estate

19# passes it, so refusing it would take the fleet's folio down.

20#

21# Exit 0 on pass, 1 on any failure, 2 if the binary could not be found.

22set -u

24FOLIO=${FOLIO:-$(dirname "$0")/../build/release/bin/folio}

25if [ ! -x "$FOLIO" ]; then

26 FOLIO=$(dirname "$0")/../build/dev/bin/folio

27fi

28if [ ! -x "$FOLIO" ]; then

29 echo "ERROR: no folio binary at build/release/bin/folio or build/dev/bin/folio" >&2

30 exit 2

31fi

33ROOT=$(mktemp -d -t folio-cli.XXXXXX)

34# Unconditional, not reached-if-we-get-there: an early exit must still clean up.

35trap 'rm -rf "$ROOT"' EXIT

36printf '# Inbox\n\n- [ ] a task {id: t-cli1}\n' > "$ROOT/inbox.md"

37mkdir -p "$ROOT/projects"

39FAILED=0

40pass() { echo " PASS $1"; }

41fail() { echo " FAIL $1"; FAILED=1; }

43# --- 1. an unknown argument must be REFUSED, not ignored -------------------

44# Guard with `timeout`: the defect's failure mode is that the server STARTS,

45# which would otherwise hang here forever rather than failing.

46OUT=$(timeout 10 env FOLIO_ROOT="$ROOT" "$FOLIO" migrate-buried < /dev/null 2>&1)

47RC=$?

48if [ "$RC" -eq 0 ]; then

49 fail "unknown argument accepted (exit 0) -- server started instead of refusing"

50elif [ "$RC" -eq 124 ]; then

51 fail "unknown argument accepted -- server started and had to be timed out"

52elif ! printf '%s' "$OUT" | grep -q 'no subcommands'; then

53 fail "unknown argument refused (exit $RC) but the message does not explain why"

54else

55 pass "unknown argument refused with a non-zero exit and an explanation"

56fi

58# The message must name what IS valid, or the refusal just moves the problem.

59if printf '%s' "$OUT" | grep -q -- '--web-port' && printf '%s' "$OUT" | grep -q 'USAGE'; then

60 pass "refusal prints usage naming the valid arguments"

61else

62 fail "refusal does not name the valid arguments"

63fi

65# Discoverability: `folio migrate-buried` is what people will type, so the

66# error has to say where recovery actually lives.

67if printf '%s' "$OUT" | grep -q 'folio/migrate-buried'; then

68 pass "refusal points at the MCP tool for recovery"

69else

70 fail "refusal does not mention where migrate-buried actually lives"

71fi

73# --- 2. --help must work with NO environment ------------------------------

74OUT=$(env -u FOLIO_ROOT "$FOLIO" --help 2>&1)

75RC=$?

76if [ "$RC" -ne 0 ]; then

77 fail "--help exited $RC with no FOLIO_ROOT (must not need configuration)"

78elif ! printf '%s' "$OUT" | grep -q 'USAGE'; then

79 fail "--help exited 0 but printed no usage"

80else

81 pass "--help works with no environment"

82fi

84OUT=$(env -u FOLIO_ROOT "$FOLIO" -h 2>&1)

85if [ $? -ne 0 ] || ! printf '%s' "$OUT" | grep -q 'USAGE'; then

86 fail "-h does not behave like --help"

87else

88 pass "-h works with no environment"

89fi

91# --- 3. --version keeps working with no environment (positive control) -----

92# If this ever fails, the harness is broken rather than the feature: --version

93# worked before the fix and is not what the fix touched.

94OUT=$(env -u FOLIO_ROOT "$FOLIO" --version 2>&1)

95if [ $? -ne 0 ] || ! printf '%s' "$OUT" | grep -q '^folio '; then

96 fail "--version regressed (positive control -- suspect the harness)"

97else

98 pass "--version still works with no environment"

99fi

100

101# --- 4. FLEET-VECTOR ARMS. DO NOT PRUNE THESE AS REDUNDANT. ---------------

102#

103# The arms in this section and in section 8 exercise REAL invocations taken

104# from live configs, not synthetic defect cases. They look redundant next to

105# the defect arms above. They are the opposite of redundant, and here is the

106# measured reason:

107#

108# While hardening this CLI, the port validator was written with `exact?`,

109# which is UNBOUND in sigil. It COMPILED CLEAN and failed at RUNTIME on

110# every --web-port invocation -- i.e. on launch-server.sh's live vector.

111# Every defect-case arm still passed. Only running a real fleet vector

112# caught it.

113#

114# A build that succeeds is not a code path that has run. These arms are the

115# only thing here that executes the paths the estate actually uses.

116#

117# --- THE FLEET MUST NOT BREAK ---------------------------------------------

118# `serve` plus the logging flags are what every MCP config passes. Measured

119# 2026-08-11: 15 live processes use `serve --log FILE --log-level LEVEL`.

120# With stdin closed the server reads EOF and shuts down cleanly, so ACCEPTED

121# looks like exit 0 with no usage text; REFUSED looks like exit 1 with usage.

122OUT=$(timeout 10 env FOLIO_ROOT="$ROOT" "$FOLIO" serve --log "$ROOT/f.log" --log-level trace \

123 < /dev/null 2>&1)

124RC=$?

125if [ "$RC" -eq 0 ] && ! printf '%s' "$OUT" | grep -q 'USAGE'; then

126 pass "serve + --log + --log-level accepted (clean startup and shutdown)"

127else

128 fail "serve + logging flags rejected (exit $RC) -- this would break every MCP config"

129fi

130

131# NOTE ON THE ASSERTION ABOVE. The first version of this arm expected exit 124

132# (timeout), reasoning that a server which starts never exits. That was wrong and

133# it failed on BOTH the pre-fix and post-fix binaries: with stdin closed the MCP

134# server reads EOF and shuts down cleanly, exiting 0. Caught by running the arm

135# against the pre-fix binary and reading the failure rather than assuming a red

136# there meant the arm was working. This arm is a REGRESSION GUARD, not a defect

137# detector -- it must pass on both binaries, because `serve` worked before the

138# fix and must keep working after it.

139

140# A value-flag with its value missing must be refused, not silently dropped.

141OUT=$(timeout 10 env FOLIO_ROOT="$ROOT" "$FOLIO" --web-port < /dev/null 2>&1)

142RC=$?

143if [ "$RC" -ne 0 ] && [ "$RC" -ne 124 ]; then

144 pass "--web-port with no value refused"

145else

146 fail "--web-port with no value not refused (exit $RC)"

147fi

148

149# --- 5. --version / --help MUST NOT BYPASS VALIDATION ----------------------

150# Both are `member` scans over the whole command line. When they ran BEFORE

151# validation, any unknown argument was waved through if one of them appeared

152# anywhere: `folio migrate-buried --version` printed the version and exited 0 --

153# the exact silent success this validation exists to kill, reintroduced by the

154# ordering of the fix for it. Found by adversarial review, uncovered until now.

155for COMBO in "migrate-buried --version" "migrate-buried -h" "--version migrate-buried"; do

156 # shellcheck disable=SC2086

157 OUT=$(timeout 10 env FOLIO_ROOT="$ROOT" "$FOLIO" $COMBO < /dev/null 2>&1)

158 RC=$?

159 if [ "$RC" -eq 0 ]; then

160 fail "'folio $COMBO' exited 0 -- unknown argument bypassed validation"

161 else

162 pass "'folio $COMBO' refused"

163 fi

164done

165

166# --- 6. VALIDATION MUST AGREE WITH THE PARSERS THAT DRIVE BEHAVIOUR --------

167# `folio --log --web` was accepted as "--log with value --web"; find-web-port

168# then saw --web and started an HTTP server nobody asked for, and the logger

169# wrote to a file named `--web`. Exit 0, no diagnostic. A validator that blesses

170# an invocation the program then reads differently is worse than none.

171# Assert a REFUSAL, not merely a non-zero exit: without the fix this invocation

172# starts a web server and is killed by `timeout` (124), which a bare non-zero

173# check would score as a pass for entirely the wrong reason.

174OUT=$(timeout 10 env FOLIO_ROOT="$ROOT" "$FOLIO" --log --web < /dev/null 2>&1)

175RC=$?

176if [ "$RC" -ne 0 ] && [ "$RC" -ne 124 ] && printf '%s' "$OUT" | grep -q 'requires a value'; then

177 pass "a value-flag swallowing an option is refused"

178elif [ "$RC" -eq 124 ]; then

179 fail "'--log --web' started a server (timed out) -- validator disagrees with find-web-port"

180else

181 fail "'--log --web' not refused with an explanation (exit $RC)"

182fi

183

184# Run this probe with cwd INSIDE the throwaway root, so a file created by the

185# defect lands there and is cleaned up with it. The first version checked `./`

186# and reported a stale `--web` left in the repo by an earlier run against an

187# older binary -- a true finding about the wrong build, which is its own kind of

188# false green. Scope a side-effect check to the run that could cause it.

189( cd "$ROOT" && timeout 10 env FOLIO_ROOT="$ROOT" "$FOLIO" --log --web < /dev/null > /dev/null 2>&1 )

190if [ -e "$ROOT/--web" ]; then

191 fail "a file named '--web' was created -- the option was consumed as a value"

192else

193 pass "no file created from an option mistaken for a value"

194fi

195

196# An unknown log level used to be accepted and silently demoted to info.

197OUT=$(timeout 10 env FOLIO_ROOT="$ROOT" "$FOLIO" --log-level tarce < /dev/null 2>&1)

198if [ $? -eq 0 ]; then

199 fail "unknown --log-level accepted (silently demoted rather than refused)"

200else

201 pass "unknown --log-level refused"

202fi

203

204# A non-numeric port used to pass validation and fail only AFTER the index was

205# opened -- refusing after acting is not refusing.

206OUT=$(timeout 10 env FOLIO_ROOT="$ROOT" "$FOLIO" --web-port abc < /dev/null 2>&1)

207if [ $? -eq 0 ]; then

208 fail "--web-port abc accepted"

209else

210 pass "--web-port with a non-numeric value refused"

211fi

212

213# --- 7. STREAMS. Redirected INDEPENDENTLY, because 2>&1 cannot see this ----

214# Every other arm captures with 2>&1 and is therefore blind to which stream

215# anything went to. A stub printing help to stderr and the refusal to stdout

216# passed all of them. Usage on stdout would pollute the MCP stdio channel.

217if [ -n "$(env -u FOLIO_ROOT "$FOLIO" --help 2>/dev/null)" ]; then

218 pass "--help prints to stdout"

219else

220 fail "--help prints nothing to stdout"

221fi

222

223if [ -z "$(env -u FOLIO_ROOT "$FOLIO" --help 2>&1 1>/dev/null)" ]; then

224 pass "--help prints nothing to stderr"

225else

226 fail "--help writes to stderr (would pollute the MCP stdio channel)"

227fi

228

229if [ -n "$(timeout 10 env FOLIO_ROOT="$ROOT" "$FOLIO" bogus < /dev/null 2>&1 1>/dev/null)" ]; then

230 pass "refusal prints to stderr"

231else

232 fail "refusal prints nothing to stderr"

233fi

234

235if [ -z "$(timeout 10 env FOLIO_ROOT="$ROOT" "$FOLIO" bogus < /dev/null 2>/dev/null)" ]; then

236 pass "refusal prints nothing to stdout"

237else

238 fail "refusal writes usage to stdout"

239fi

240

241# --- 8. THE OTHER LIVE FLEET VECTOR: --web-port WITH a value --------------

242# `serve --web-port 3737 --log F --log-level trace` is what launch-server.sh

243# passes. The earlier --web-port arm only covers the MISSING-value case.

244OUT=$(timeout 10 env FOLIO_ROOT="$ROOT" "$FOLIO" serve --web-port 3737 \

245 --log "$ROOT/w.log" --log-level trace < /dev/null 2>&1)

246RC=$?

247if [ "$RC" -eq 0 ] && ! printf '%s' "$OUT" | grep -q 'USAGE'; then

248 pass "serve + --web-port VALUE + logging flags accepted (live fleet vector)"

249else

250 fail "serve + --web-port 3737 rejected (exit $RC) -- breaks launch-server.sh"

251fi

252

253# --- 9. --stdio is a compatibility no-op, and must stay one ---------------

254# Many MCP clients pass it. folio speaks stdio anyway, so it must be accepted

255# and ignored: refusing it breaks such a client for no safety gain.

256OUT=$(timeout 10 env FOLIO_ROOT="$ROOT" "$FOLIO" serve --stdio < /dev/null 2>&1)

257RC=$?

258if [ "$RC" -eq 0 ] && ! printf '%s' "$OUT" | grep -q 'USAGE'; then

259 pass "--stdio accepted and ignored"

260else

261 fail "--stdio rejected (exit $RC) -- breaks MCP clients that pass it"

262fi

263

264# --- 10. --flag=value must be REFUSED WITH THE CORRECTION -----------------

265# folio cannot honour this spelling: log-configure-from-args! matches with

266# (equal? ...) and find-web-port with (string=? ...), neither splitting on '='.

267# Accepting it would mean the flag is silently ignored by the code that

268# consumes it -- a validator/consumer divergence, the defect this file exists

269# to prevent. So it must refuse, and the message must say how to write it.

270for PAIR in "--log-level=trace|--log-level trace" "--web-port=8080|--web-port 8080"; do

271 ARG=${PAIR%%|*}; WANT=${PAIR##*|}

272 OUT=$(timeout 10 env FOLIO_ROOT="$ROOT" "$FOLIO" "$ARG" < /dev/null 2>&1)

273 RC=$?

274 if [ "$RC" -eq 0 ]; then

275 fail "'$ARG' accepted -- it would be silently ignored downstream"

276 elif printf '%s' "$OUT" | grep -qF -- "$WANT"; then

277 pass "'$ARG' refused, and the message gives the correct spelling"

278 else

279 fail "'$ARG' refused but the message does not show '$WANT'"

280 fi

281done

282

283if [ "$FAILED" -eq 0 ]; then

284 echo "CLI-CONTRACT-OK"

285 exit 0

286fi

287echo "CLI-CONTRACT-FAILED"

288exit 1