AtlatestRepositorysigil-doc

sigil-doc / tree / toolsimport-sweep.sh

1#!/usr/bin/env bash
2# Sweep src/ for imports this package does not declare a dependency for.
3#
4# WHY THIS EXISTS. `package.sgl` declared only `sigil-sxml` while
5# `src/sigil/doc/verify.sgl` imported `(sigil shell)`, `(sigil package)` and
6# `(sigil crypto)`. None of that is visible from inside this repo: the implicit
7# `sigil-stdlib` peer is fetched as the WHOLE sigil monorepo checkout, so every
8# sibling workspace package's sources sit on the module search path and an
9# undeclared import compiles clean here. It only fails in a CONSUMER, at load
10# time, once the module is asked for and no one ever built it -- which is how
11# press got `module-import: library not found: (sigil shell)` from a sigil-doc
12# that built and tested green.
14# WHAT IT REPORTS, per (file, imported external module):
15# PACKAGE the package that defines the module, derived by searching the
16# resolved dependency trees for its `define-library` form and then
17# reading the name out of the nearest enclosing package.sgl -- not a
18# hand-written table, and not the dependency DIRECTORY name, which
19# is the repo's (`codeberg-sigil-sigil-crypto-<sha>`) and names no
20# package any manifest could declare.
21# DECLARED whether package.sgl declares that package. `sigil-stdlib` is the
22# implicit peer of the `sigil:` range and is never declared.
23# USED which of that module's exported names occur as whole tokens in the
24# importing file's body, so an import that is merely DEAD is
25# distinguishable from one that is load-bearing but undeclared. The
26# two want opposite fixes: delete the import, or declare the dep.
28# HOW THE BODY IS DERIVED. Not by skipping a fixed number of header lines -- an
29# earlier version did that and it was wrong twice over. It cut at the FIRST
30# `(export` line, so the rest of a multi-line export list was scanned as body
31# and a symbol this file merely RE-EXPORTS scored as a use of an imported
32# module: a false USED, which masks precisely the "dead import" verdict this
33# tool exists to produce. And the cutoff was computed by an awk that printed two
34# numbers (awk runs END even after `exit`), which turned `NR > skip` into a
35# STRING comparison and silently blanked most of every file.
37# Instead the source is reduced, character by character, to a token stream with
38# comments, string literals, and the whole of every `(import ...)` and
39# `(export ...)` form removed by paren balance. Order-independent, length-
40# independent, and an import can never count as a use of itself.
42# GUARDS, because each of these has produced a confident wrong answer here:
43# * A module whose export list cannot be extracted reports UNSCANNABLE, never
44# "unused". `(sigil core)` exports via `%module-export!` rather than an
45# `(export ...)` form and scored NONE everywhere until the extractor learned
46# that form.
47# * The scanner is proved to fire AND proved to stand down before any verdict
48# is printed -- a positive control on a symbol used late in a long file, and
49# a NEGATIVE control on a symbol that is textually present but only inside a
50# comment and a string. Sabotage shows an instrument can fire; only the
51# negative control shows it is not simply matching everything.
53# KNOWN LIMITS, stated rather than left for someone to discover: block comments
54# (`#| ... |#`) and datum comments (`#;`) are not understood, and only
55# two-element lowercase `(sigil <name>)` module heads are swept. The count of
56# import forms NOT swept is printed, so the population is never silently
57# narrowed.
58set -euo pipefail
60repo_dir="$(cd "$(dirname "$0")/.." && pwd)"
61cd "$repo_dir"
62deps_root="${SIGIL_DEPS_ROOT:-$HOME/.sigil/deps}"
64for tool in awk grep sed sort find; do
65 command -v "$tool" >/dev/null || { echo "SETUP-FAILED: $tool is required" >&2; exit 2; }
66done
67test -f package.sgl || { echo "SETUP-FAILED: no package.sgl at $repo_dir" >&2; exit 2; }
68test -d "$deps_root" || { echo "SETUP-FAILED: no resolved dependency trees at $deps_root" >&2
69 echo " run 'sigil deps install' first" >&2; exit 2; }
71work=$(mktemp -d "${TMPDIR:-/tmp}/sigil-doc-import-sweep.XXXXXX")
72trap 'rm -rf "$work"' EXIT INT TERM
74# ---------------------------------------------------------------- extractors
75cat >"$work/imports.awk" <<'AWK'
76# Print each (import ...) form as one line, by paren balance from the "(import"
77# token to its matching close. Line comments are stripped first.
79 line = $0; sub(/;.*$/, "", line)
80 if (!inimp) { idx = index(line, "(import"); if (idx == 0) next
81 line = substr(line, idx); inimp = 1; depth = 0; buf = "" }
82 n = length(line); out = ""
83 for (i = 1; i <= n; i++) {
84 c = substr(line, i, 1); out = out c
85 if (c == "(") depth++
86 else if (c == ")") { depth--
87 if (depth == 0) { printf "%s\n", buf out; inimp = 0; buf = ""; next } }
88 }
89 buf = buf out " "
91AWK
93cat >"$work/exports.awk" <<'AWK'
94# Emit exported symbols from a module source. Handles both forms in use:
95# (export a b c ...) -- define-library modules
96# (%module-export! 'name) -- bootstrap modules such as (sigil core)
98 line = $0; sub(/;.*$/, "", line)
99 if (match(line, /%module-export![ \t]*'[^ \t()]+/)) {
100 s = substr(line, RSTART, RLENGTH); sub(/.*'/, "", s); print s }
101 if (!ine) { i = index(line, "(export"); if (i == 0) next
102 line = substr(line, i); ine = 1; d = 0 }
103 n = length(line); out = ""
104 for (i = 1; i <= n; i++) {
105 c = substr(line, i, 1); out = out c
106 if (c == "(") d++
107 else if (c == ")") { d--; if (d == 0) { print out; ine = 0; next } } }
108 print out
110AWK
112cat >"$work/body.awk" <<'AWK'
113# Reduce a Sigil source to its scannable BODY: everything except comments,
114# string literals, and the whole of every (import ...) / (export ...) form.
115# Emits one whitespace-separated token per line.
117# String state persists across lines, so a multi-line literal is removed rather
118# than tokenised -- the per-line version left its contents in the stream.
119# Strings are recognised BEFORE comments, so a `;` inside a literal does not
120# truncate the line; the earlier order silently discarded whatever code followed
121# such a string, and this source has sixteen lines of that shape.
123 line = $0; n = length(line); out = ""; i = 1
124 while (i <= n) {
125 c = substr(line, i, 1)
126 if (instr) {
127 if (c == "\\") { i += 2; continue }
128 if (c == "\"") instr = 0
129 i++; continue }
130 if (c == "\"") { instr = 1; i++; continue }
131 if (c == ";") break # rest of line is a comment
132 if (blank > 0) {
133 if (c == "(") blank++
134 else if (c == ")") blank--
135 i++; continue }
136 if (substr(line, i, 7) == "(import" || substr(line, i, 7) == "(export") {
137 blank = 1; i += 7; continue }
138 out = out c; i++
139 }
140 gsub(/[()\[\]'`,]/, " ", out)
141 k = split(out, a, " ")
142 for (j = 1; j <= k; j++) if (a[j] != "") print a[j]
144AWK
146# ------------------------------------------------- module -> defining package
147declare -A MOD_SRC MOD_PKG
148index_module() {
149 local mod="$1" src dir
150 [ -n "${MOD_SRC[$mod]:-}" ] && return 0
151 src=$(grep -rl --include='*.sgl' -E "^\(define-library \(sigil $mod\)" "$deps_root" 2>/dev/null | head -1 || true)
152 # Bootstrap modules have no define-library form -- (sigil core) builds itself
153 # with %make-module/%module-export!. Fall back to the canonical path so it
154 # resolves to a package rather than reading UNRESOLVED, which must stay the
155 # signature of a genuinely missing module.
156 [ -n "$src" ] || src=$(find "$deps_root" -path "*/src/sigil/$mod.sgl" 2>/dev/null | head -1 || true)
157 [ -n "$src" ] || { MOD_SRC[$mod]=""; MOD_PKG[$mod]="UNRESOLVED"; return 0; }
158 MOD_SRC[$mod]="$src"
159 dir="${src%/src/*}"
160 if [ -f "$dir/package.sgl" ]; then
161 MOD_PKG[$mod]=$(grep -oE 'name:[[:space:]]*"[^"]+"' "$dir/package.sgl" | head -1 | sed -E 's/.*"(.*)"/\1/' || true)
162 else
163 MOD_PKG[$mod]="UNKNOWN-PACKAGE"
164 fi
165 [ -n "${MOD_PKG[$mod]}" ] || MOD_PKG[$mod]="UNKNOWN-PACKAGE"
168exports_of() {
169 local mod="$1"
170 index_module "$mod"
171 [ -n "${MOD_SRC[$mod]}" ] || return 1
172 awk -f "$work/exports.awk" "${MOD_SRC[$mod]}" \
173 | tr '()' ' ' | tr -s ' \n' '\n' | grep -vE '^$|^export$' | sort -u
176body_of() { # cached; the reduction is the same for every module in a file
177 local file="$1" key cache
178 key=$(printf '%s' "$file" | tr '/' '_')
179 cache="$work/body.$key"
180 [ -f "$cache" ] || awk -f "$work/body.awk" "$file" | sort -u >"$cache"
181 printf '%s' "$cache"
184uses_of() {
185 local file="$1" mod="$2" syms cache
186 syms=$(exports_of "$mod" || true)
187 [ -n "$syms" ] || { echo "UNSCANNABLE"; return; }
188 cache=$(body_of "$file")
189 # The symbol list goes through a FILE, not `awk -v`. `-v` performs escape
190 # processing, so a symbol containing a backslash arrives mangled and can never
191 # match -- a silent partial want-set, which reads as an unused import.
192 printf '%s\n' "$syms" >"$work/syms.txt"
193 awk 'NR == FNR { want[$0] = 1; next } ($0 in want) { print; found = 1 }
194 END { if (!found) print "NONE" }' "$work/syms.txt" "$cache" | sort | tr '\n' ' '
197declared() {
198 local pkg="$1"
199 [ "$pkg" = "sigil-stdlib" ] && { echo "implicit"; return; }
200 # Comment lines are excluded: a manifest comment saying why we do NOT depend
201 # on a package must not score as declaring it.
202 if sed -e 's/;.*$//' package.sgl | grep -qF -- "\"$pkg\""; then echo "yes"
203 elif sed -e 's/;.*$//' package.sgl | grep -qF -- "/$pkg\""; then echo "yes"
204 else echo "NO"; fi
207abbrev() {
208 awk -v max="${SWEEP_MAX_SYMS:-6}" '{ n = NF
209 if (n <= max) { print; exit }
210 out = ""; for (i = 1; i <= max; i++) out = out $i " "
211 printf "%s(+%d more)\n", out, n - max }'
214# ------------------------------------------------------------------- controls
215# Nothing below is printed until the scanner is shown to fire AND to stand down.
216# The first version of this control asked only that `define-struct` be seen in a
217# parser file. It passed while the scanner was skipping most of every source,
218# because that symbol happened to sit on a line the broken comparison let
219# through -- certifying a table in which every genuinely used module read NONE.
220ctl_file="src/sigil/doc/verify.sgl"
221test -f "$ctl_file" || { echo "SETUP-FAILED: control file $ctl_file is gone; re-point the controls" >&2; exit 2; }
222echo "usage scanner controls:"
224# POSITIVE. `sha256` is a (sigil crypto) export genuinely used in this file.
225pos=" $(uses_of "$ctl_file" crypto) "
226case "$pos" in *" sha256 "*) ;; *)
227 echo "SETUP-FAILED: the scanner did not see sha256 in $ctl_file via (sigil crypto)." >&2
228 echo " It reported:$pos -- every absence verdict below would be vacuous." >&2; exit 2 ;; esac
229# REACH. Asserting that the control symbol occurs LATE in the source does not
230# test the scanner at all -- `sha256` also occurs at line 35, so a scanner
231# truncated to the first 100 lines still passes on it. That is what the previous
232# version of this control did, and a deliberate truncation walked straight
233# through it.
235# So bind reach directly: take an export of the control module that does NOT
236# occur anywhere in the control file, append a use of it as the LAST line of a
237# copy, and require the scanner to report it. Anything that stops early, reads
238# only a header, or compares line numbers as strings fails this and cannot fail
239# it accidentally.
240reach_sym=$(exports_of crypto | while read -r s; do
241 grep -qF -- "$s" "$ctl_file" || { echo "$s"; break; }; done || true)
242test -n "$reach_sym" || {
243 echo "SETUP-FAILED: every (sigil crypto) export already appears in $ctl_file, so no" >&2
244 echo " unambiguous reach probe can be built. Re-point the reach control." >&2; exit 2; }
245cp "$ctl_file" "$work/reach-probe.sgl"
246printf '\n(define (sweep-reach-probe) (%s 1))\n' "$reach_sym" >>"$work/reach-probe.sgl"
247probe_lines=$(wc -l < "$work/reach-probe.sgl")
248reach=" $(uses_of "$work/reach-probe.sgl" crypto) "
249case "$reach" in *" $reach_sym "*) ;; *)
250 echo "SETUP-FAILED: the scanner did not see '$reach_sym' on line ~$probe_lines of the" >&2
251 echo " reach probe. It does not read to the end of a file; every verdict below" >&2
252 echo " would be scoped to however far it does read. It reported:$reach" >&2; exit 2 ;; esac
253echo " REACH saw '$reach_sym' planted at line $probe_lines of a $probe_lines-line probe"
254echo " POSITIVE saw sha256 in $ctl_file via (sigil crypto)"
256# NEGATIVE. `&&` is a (sigil shell) export and IS textually present in this file
257# -- inside an embedded bash string and a prose comment. The scanner must not
258# report it. Without this, a scanner that stripped nothing would pass the
259# positive control and call every import used.
260grep -qF -- '&&' "$ctl_file" || {
261 echo "SETUP-FAILED: '&&' is no longer textually present in $ctl_file, so the negative" >&2
262 echo " control cannot fail and proves nothing. Re-point it." >&2; exit 2; }
263neg=" $(uses_of "$ctl_file" shell) "
264case "$neg" in *" && "*)
265 echo "SETUP-FAILED: the scanner reported '&&' as a use, but every occurrence in" >&2
266 echo " $ctl_file is inside a string literal or a comment. Stripping is not working." >&2
267 exit 2 ;; esac
268echo " NEGATIVE '&&' is present in the source but correctly not reported as a use"
269echo
271# ----------------------------------------------------------------------- sweep
272files=$(find src -name '*.sgl' | sort)
273test -n "$files" || { echo "SETUP-FAILED: no sources under src/" >&2; exit 2; }
274printf '%-44s %-16s %-14s %-9s %s\n' FILE MODULE PACKAGE DECLARED USED-EXPORTS
275undeclared_used=0; undeclared_dead=0; scanned=0; unswept=0
276for f in $files; do
277 awk -f "$work/imports.awk" "$f" >"$work/forms.txt"
278 # Only the FIRST import form is this library's own; later ones are generated
279 # source -- a quoted prelude, a string handed to a subprocess.
280 head -1 "$work/forms.txt" >"$work/first.txt"
281 mods=$(grep -oE '\(sigil [a-z0-9-]+\)' "$work/first.txt" | sed -E 's/\(sigil (.*)\)/\1/' | sort -u || true)
282 heads=$(grep -oE '\([a-z][a-z0-9 -]*\)' "$work/first.txt" | sort -u | wc -l)
283 swept=$(printf '%s\n' "$mods" | grep -c . || true)
284 unswept=$(( unswept + heads - swept ))
285 for m in $mods; do
286 index_module "$m"
287 pkg="${MOD_PKG[$m]}"; d=$(declared "$pkg"); u=$(uses_of "$f" "$m")
288 scanned=$(( scanned + 1 ))
289 printf '%-44s %-16s %-14s %-9s %s\n' "$f" "(sigil $m)" "$pkg" "$d" "$(printf '%s' "$u" | abbrev)"
290 if [ "$d" = "NO" ]; then
291 case "$u" in
292 NONE*) undeclared_dead=$(( undeclared_dead + 1 )) ;;
293 UNSCANNABLE*) : ;;
294 *) undeclared_used=$(( undeclared_used + 1 )) ;;
295 esac
296 fi
297 done
298done
299echo
300echo "scanned $scanned (file, external module) pairs across $(printf '%s\n' "$files" | wc -l) source files"
301echo "module heads in those import forms NOT swept (own modules, or not two-element"
302echo "lowercase (sigil <name>)): $unswept"
303echo "undeclared and USED (declare the package): $undeclared_used"
304echo "undeclared and UNUSED (delete the import): $undeclared_dead"
305echo "IMPORT_SWEEP_COMPLETE"