AtlatestRepositorysigil-doc
1
#!/usr/bin/env bash2
# Sweep src/ for imports this package does not declare a dependency for.3
#4
# WHY THIS EXISTS. `package.sgl` declared only `sigil-sxml` while5
# `src/sigil/doc/verify.sgl` imported `(sigil shell)`, `(sigil package)` and6
# `(sigil crypto)`. None of that is visible from inside this repo: the implicit7
# `sigil-stdlib` peer is fetched as the WHOLE sigil monorepo checkout, so every8
# sibling workspace package's sources sit on the module search path and an9
# undeclared import compiles clean here. It only fails in a CONSUMER, at load10
# time, once the module is asked for and no one ever built it -- which is how11
# press got `module-import: library not found: (sigil shell)` from a sigil-doc12
# that built and tested green.13
#14
# WHAT IT REPORTS, per (file, imported external module):15
# PACKAGE the package that defines the module, derived by searching the16
# resolved dependency trees for its `define-library` form and then17
# reading the name out of the nearest enclosing package.sgl -- not a18
# hand-written table, and not the dependency DIRECTORY name, which19
# is the repo's (`codeberg-sigil-sigil-crypto-<sha>`) and names no20
# package any manifest could declare.21
# DECLARED whether package.sgl declares that package. `sigil-stdlib` is the22
# implicit peer of the `sigil:` range and is never declared.23
# USED which of that module's exported names occur as whole tokens in the24
# importing file's body, so an import that is merely DEAD is25
# distinguishable from one that is load-bearing but undeclared. The26
# two want opposite fixes: delete the import, or declare the dep.27
#28
# HOW THE BODY IS DERIVED. Not by skipping a fixed number of header lines -- an29
# earlier version did that and it was wrong twice over. It cut at the FIRST30
# `(export` line, so the rest of a multi-line export list was scanned as body31
# and a symbol this file merely RE-EXPORTS scored as a use of an imported32
# module: a false USED, which masks precisely the "dead import" verdict this33
# tool exists to produce. And the cutoff was computed by an awk that printed two34
# numbers (awk runs END even after `exit`), which turned `NR > skip` into a35
# STRING comparison and silently blanked most of every file.36
#37
# Instead the source is reduced, character by character, to a token stream with38
# comments, string literals, and the whole of every `(import ...)` and39
# `(export ...)` form removed by paren balance. Order-independent, length-40
# independent, and an import can never count as a use of itself.41
#42
# GUARDS, because each of these has produced a confident wrong answer here:43
# * A module whose export list cannot be extracted reports UNSCANNABLE, never44
# "unused". `(sigil core)` exports via `%module-export!` rather than an45
# `(export ...)` form and scored NONE everywhere until the extractor learned46
# that form.47
# * The scanner is proved to fire AND proved to stand down before any verdict48
# is printed -- a positive control on a symbol used late in a long file, and49
# a NEGATIVE control on a symbol that is textually present but only inside a50
# comment and a string. Sabotage shows an instrument can fire; only the51
# negative control shows it is not simply matching everything.52
#53
# KNOWN LIMITS, stated rather than left for someone to discover: block comments54
# (`#| ... |#`) and datum comments (`#;`) are not understood, and only55
# two-element lowercase `(sigil <name>)` module heads are swept. The count of56
# import forms NOT swept is printed, so the population is never silently57
# narrowed.58
set -euo pipefail60
repo_dir="$(cd "$(dirname "$0")/.." && pwd)"61
cd "$repo_dir"62
deps_root="${SIGIL_DEPS_ROOT:-$HOME/.sigil/deps}"64
for tool in awk grep sed sort find; do65
command -v "$tool" >/dev/null || { echo "SETUP-FAILED: $tool is required" >&2; exit 2; }66
done67
test -f package.sgl || { echo "SETUP-FAILED: no package.sgl at $repo_dir" >&2; exit 2; }68
test -d "$deps_root" || { echo "SETUP-FAILED: no resolved dependency trees at $deps_root" >&269
echo " run 'sigil deps install' first" >&2; exit 2; }71
work=$(mktemp -d "${TMPDIR:-/tmp}/sigil-doc-import-sweep.XXXXXX")72
trap 'rm -rf "$work"' EXIT INT TERM74
# ---------------------------------------------------------------- extractors75
cat >"$work/imports.awk" <<'AWK'76
# Print each (import ...) form as one line, by paren balance from the "(import"77
# token to its matching close. Line comments are stripped first.78
{79
line = $0; sub(/;.*$/, "", line)80
if (!inimp) { idx = index(line, "(import"); if (idx == 0) next81
line = substr(line, idx); inimp = 1; depth = 0; buf = "" }82
n = length(line); out = ""83
for (i = 1; i <= n; i++) {84
c = substr(line, i, 1); out = out c85
if (c == "(") depth++86
else if (c == ")") { depth--87
if (depth == 0) { printf "%s\n", buf out; inimp = 0; buf = ""; next } }88
}89
buf = buf out " "90
}91
AWK93
cat >"$work/exports.awk" <<'AWK'94
# Emit exported symbols from a module source. Handles both forms in use:95
# (export a b c ...) -- define-library modules96
# (%module-export! 'name) -- bootstrap modules such as (sigil core)97
{98
line = $0; sub(/;.*$/, "", line)99
if (match(line, /%module-export![ \t]*'[^ \t()]+/)) {100
s = substr(line, RSTART, RLENGTH); sub(/.*'/, "", s); print s }101
if (!ine) { i = index(line, "(export"); if (i == 0) next102
line = substr(line, i); ine = 1; d = 0 }103
n = length(line); out = ""104
for (i = 1; i <= n; i++) {105
c = substr(line, i, 1); out = out c106
if (c == "(") d++107
else if (c == ")") { d--; if (d == 0) { print out; ine = 0; next } } }108
print out109
}110
AWK112
cat >"$work/body.awk" <<'AWK'113
# Reduce a Sigil source to its scannable BODY: everything except comments,114
# string literals, and the whole of every (import ...) / (export ...) form.115
# Emits one whitespace-separated token per line.116
#117
# String state persists across lines, so a multi-line literal is removed rather118
# than tokenised -- the per-line version left its contents in the stream.119
# Strings are recognised BEFORE comments, so a `;` inside a literal does not120
# truncate the line; the earlier order silently discarded whatever code followed121
# such a string, and this source has sixteen lines of that shape.122
{123
line = $0; n = length(line); out = ""; i = 1124
while (i <= n) {125
c = substr(line, i, 1)126
if (instr) {127
if (c == "\\") { i += 2; continue }128
if (c == "\"") instr = 0129
i++; continue }130
if (c == "\"") { instr = 1; i++; continue }131
if (c == ";") break # rest of line is a comment132
if (blank > 0) {133
if (c == "(") blank++134
else if (c == ")") blank--135
i++; continue }136
if (substr(line, i, 7) == "(import" || substr(line, i, 7) == "(export") {137
blank = 1; i += 7; continue }138
out = out c; i++139
}140
gsub(/[()\[\]'`,]/, " ", out)141
k = split(out, a, " ")142
for (j = 1; j <= k; j++) if (a[j] != "") print a[j]143
}144
AWK146
# ------------------------------------------------- module -> defining package147
declare -A MOD_SRC MOD_PKG148
index_module() {149
local mod="$1" src dir150
[ -n "${MOD_SRC[$mod]:-}" ] && return 0151
src=$(grep -rl --include='*.sgl' -E "^\(define-library \(sigil $mod\)" "$deps_root" 2>/dev/null | head -1 || true)152
# Bootstrap modules have no define-library form -- (sigil core) builds itself153
# with %make-module/%module-export!. Fall back to the canonical path so it154
# resolves to a package rather than reading UNRESOLVED, which must stay the155
# signature of a genuinely missing module.156
[ -n "$src" ] || src=$(find "$deps_root" -path "*/src/sigil/$mod.sgl" 2>/dev/null | head -1 || true)157
[ -n "$src" ] || { MOD_SRC[$mod]=""; MOD_PKG[$mod]="UNRESOLVED"; return 0; }158
MOD_SRC[$mod]="$src"159
dir="${src%/src/*}"160
if [ -f "$dir/package.sgl" ]; then161
MOD_PKG[$mod]=$(grep -oE 'name:[[:space:]]*"[^"]+"' "$dir/package.sgl" | head -1 | sed -E 's/.*"(.*)"/\1/' || true)162
else163
MOD_PKG[$mod]="UNKNOWN-PACKAGE"164
fi165
[ -n "${MOD_PKG[$mod]}" ] || MOD_PKG[$mod]="UNKNOWN-PACKAGE"166
}168
exports_of() {169
local mod="$1"170
index_module "$mod"171
[ -n "${MOD_SRC[$mod]}" ] || return 1172
awk -f "$work/exports.awk" "${MOD_SRC[$mod]}" \173
| tr '()' ' ' | tr -s ' \n' '\n' | grep -vE '^$|^export$' | sort -u174
}176
body_of() { # cached; the reduction is the same for every module in a file177
local file="$1" key cache178
key=$(printf '%s' "$file" | tr '/' '_')179
cache="$work/body.$key"180
[ -f "$cache" ] || awk -f "$work/body.awk" "$file" | sort -u >"$cache"181
printf '%s' "$cache"182
}184
uses_of() {185
local file="$1" mod="$2" syms cache186
syms=$(exports_of "$mod" || true)187
[ -n "$syms" ] || { echo "UNSCANNABLE"; return; }188
cache=$(body_of "$file")189
# The symbol list goes through a FILE, not `awk -v`. `-v` performs escape190
# processing, so a symbol containing a backslash arrives mangled and can never191
# match -- a silent partial want-set, which reads as an unused import.192
printf '%s\n' "$syms" >"$work/syms.txt"193
awk 'NR == FNR { want[$0] = 1; next } ($0 in want) { print; found = 1 }194
END { if (!found) print "NONE" }' "$work/syms.txt" "$cache" | sort | tr '\n' ' '195
}197
declared() {198
local pkg="$1"199
[ "$pkg" = "sigil-stdlib" ] && { echo "implicit"; return; }200
# Comment lines are excluded: a manifest comment saying why we do NOT depend201
# on a package must not score as declaring it.202
if sed -e 's/;.*$//' package.sgl | grep -qF -- "\"$pkg\""; then echo "yes"203
elif sed -e 's/;.*$//' package.sgl | grep -qF -- "/$pkg\""; then echo "yes"204
else echo "NO"; fi205
}207
abbrev() {208
awk -v max="${SWEEP_MAX_SYMS:-6}" '{ n = NF209
if (n <= max) { print; exit }210
out = ""; for (i = 1; i <= max; i++) out = out $i " "211
printf "%s(+%d more)\n", out, n - max }'212
}214
# ------------------------------------------------------------------- controls215
# Nothing below is printed until the scanner is shown to fire AND to stand down.216
# The first version of this control asked only that `define-struct` be seen in a217
# parser file. It passed while the scanner was skipping most of every source,218
# because that symbol happened to sit on a line the broken comparison let219
# through -- certifying a table in which every genuinely used module read NONE.220
ctl_file="src/sigil/doc/verify.sgl"221
test -f "$ctl_file" || { echo "SETUP-FAILED: control file $ctl_file is gone; re-point the controls" >&2; exit 2; }222
echo "usage scanner controls:"224
# POSITIVE. `sha256` is a (sigil crypto) export genuinely used in this file.225
pos=" $(uses_of "$ctl_file" crypto) "226
case "$pos" in *" sha256 "*) ;; *)227
echo "SETUP-FAILED: the scanner did not see sha256 in $ctl_file via (sigil crypto)." >&2228
echo " It reported:$pos -- every absence verdict below would be vacuous." >&2; exit 2 ;; esac229
# REACH. Asserting that the control symbol occurs LATE in the source does not230
# test the scanner at all -- `sha256` also occurs at line 35, so a scanner231
# truncated to the first 100 lines still passes on it. That is what the previous232
# version of this control did, and a deliberate truncation walked straight233
# through it.234
#235
# So bind reach directly: take an export of the control module that does NOT236
# occur anywhere in the control file, append a use of it as the LAST line of a237
# copy, and require the scanner to report it. Anything that stops early, reads238
# only a header, or compares line numbers as strings fails this and cannot fail239
# it accidentally.240
reach_sym=$(exports_of crypto | while read -r s; do241
grep -qF -- "$s" "$ctl_file" || { echo "$s"; break; }; done || true)242
test -n "$reach_sym" || {243
echo "SETUP-FAILED: every (sigil crypto) export already appears in $ctl_file, so no" >&2244
echo " unambiguous reach probe can be built. Re-point the reach control." >&2; exit 2; }245
cp "$ctl_file" "$work/reach-probe.sgl"246
printf '\n(define (sweep-reach-probe) (%s 1))\n' "$reach_sym" >>"$work/reach-probe.sgl"247
probe_lines=$(wc -l < "$work/reach-probe.sgl")248
reach=" $(uses_of "$work/reach-probe.sgl" crypto) "249
case "$reach" in *" $reach_sym "*) ;; *)250
echo "SETUP-FAILED: the scanner did not see '$reach_sym' on line ~$probe_lines of the" >&2251
echo " reach probe. It does not read to the end of a file; every verdict below" >&2252
echo " would be scoped to however far it does read. It reported:$reach" >&2; exit 2 ;; esac253
echo " REACH saw '$reach_sym' planted at line $probe_lines of a $probe_lines-line probe"254
echo " POSITIVE saw sha256 in $ctl_file via (sigil crypto)"256
# NEGATIVE. `&&` is a (sigil shell) export and IS textually present in this file257
# -- inside an embedded bash string and a prose comment. The scanner must not258
# report it. Without this, a scanner that stripped nothing would pass the259
# positive control and call every import used.260
grep -qF -- '&&' "$ctl_file" || {261
echo "SETUP-FAILED: '&&' is no longer textually present in $ctl_file, so the negative" >&2262
echo " control cannot fail and proves nothing. Re-point it." >&2; exit 2; }263
neg=" $(uses_of "$ctl_file" shell) "264
case "$neg" in *" && "*)265
echo "SETUP-FAILED: the scanner reported '&&' as a use, but every occurrence in" >&2266
echo " $ctl_file is inside a string literal or a comment. Stripping is not working." >&2267
exit 2 ;; esac268
echo " NEGATIVE '&&' is present in the source but correctly not reported as a use"269
echo271
# ----------------------------------------------------------------------- sweep272
files=$(find src -name '*.sgl' | sort)273
test -n "$files" || { echo "SETUP-FAILED: no sources under src/" >&2; exit 2; }274
printf '%-44s %-16s %-14s %-9s %s\n' FILE MODULE PACKAGE DECLARED USED-EXPORTS275
undeclared_used=0; undeclared_dead=0; scanned=0; unswept=0276
for f in $files; do277
awk -f "$work/imports.awk" "$f" >"$work/forms.txt"278
# Only the FIRST import form is this library's own; later ones are generated279
# source -- a quoted prelude, a string handed to a subprocess.280
head -1 "$work/forms.txt" >"$work/first.txt"281
mods=$(grep -oE '\(sigil [a-z0-9-]+\)' "$work/first.txt" | sed -E 's/\(sigil (.*)\)/\1/' | sort -u || true)282
heads=$(grep -oE '\([a-z][a-z0-9 -]*\)' "$work/first.txt" | sort -u | wc -l)283
swept=$(printf '%s\n' "$mods" | grep -c . || true)284
unswept=$(( unswept + heads - swept ))285
for m in $mods; do286
index_module "$m"287
pkg="${MOD_PKG[$m]}"; d=$(declared "$pkg"); u=$(uses_of "$f" "$m")288
scanned=$(( scanned + 1 ))289
printf '%-44s %-16s %-14s %-9s %s\n' "$f" "(sigil $m)" "$pkg" "$d" "$(printf '%s' "$u" | abbrev)"290
if [ "$d" = "NO" ]; then291
case "$u" in292
NONE*) undeclared_dead=$(( undeclared_dead + 1 )) ;;293
UNSCANNABLE*) : ;;294
*) undeclared_used=$(( undeclared_used + 1 )) ;;295
esac296
fi297
done298
done299
echo300
echo "scanned $scanned (file, external module) pairs across $(printf '%s\n' "$files" | wc -l) source files"301
echo "module heads in those import forms NOT swept (own modules, or not two-element"302
echo "lowercase (sigil <name>)): $unswept"303
echo "undeclared and USED (declare the package): $undeclared_used"304
echo "undeclared and UNUSED (delete the import): $undeclared_dead"305
echo "IMPORT_SWEEP_COMPLETE"