Bryan Johnson
e08f030df5
Portable AI agent for Cloverleaf integration work. Pure bash + curl + jq. Zero dependency on v1 wrapper scripts or v2 cloverleaf-tools.pyz. 27 native Anthropic tools: NetConfig parsing (read) nc_list_protocols, nc_list_processes, nc_protocol_block, nc_protocol_field, nc_protocol_nested, nc_protocol_summary, nc_destinations, nc_sources, nc_xlate_refs, nc_tclproc_refs NetConfig modification (journal-backed writes with rollback) nc_insert_protocol, nc_add_route, larry_rollback_list Workflows nc_find_inbound, nc_make_jump (3-thread jump pattern), nc_find (tbn/tbp/tbh/tbpr/where replacements), nc_document, nc_diff_interface, nc_regression Messages hl7_field, nc_msgs (smat is SQLite!), hl7_diff (with --ignore MSH.7) File system read_file, list_dir, grep_files, glob_files, write_file, bash_exec Validated against a 22-site real Cloverleaf test install. Five worked examples end-to-end: jump-thread generation, smat MRN search, system documentation, interface+connected diff, HL7-aware regression diff. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
333 lines
15 KiB
Bash
Executable File
333 lines
15 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# nc-regression.sh — Example 6 orchestrator: end-to-end regression testing
|
|
# between two Cloverleaf environments.
|
|
#
|
|
# Phases:
|
|
# 1. discover → list inbound threads in scope (uses nc-find-inbound)
|
|
# 2. sample → grab N messages per inbound from env-A smatdb → input .msgs files
|
|
# 3. route-A → run route_test on env-A for each inbound → captured outputs/env-a/...
|
|
# 4. route-B → run route_test on env-B with same inputs → captured outputs/env-b/...
|
|
# 5. diff → hl7-diff every paired output file with --ignore MSH.7 → per-pair report
|
|
# 6. summary → one master regression-summary.md compiling everything
|
|
#
|
|
# Phases 3 and 4 require Cloverleaf's route_test command on each box. The
|
|
# command is parameterized via --route-test-cmd with placeholders:
|
|
# {THREAD} → the inbound thread name
|
|
# {INPUT} → absolute path to the .msgs input file
|
|
# {OUTPUT_DIR} → absolute path where output files should land
|
|
# {HCIROOT} → the env's HCIROOT
|
|
# {HCISITE} → the env's HCISITE
|
|
# Default: not set — you must pass it once for your shop's invocation pattern.
|
|
#
|
|
# A common pattern: a wrapper script that sources the Cloverleaf profile and
|
|
# runs `<thread> route_test <INPUT>`, with output redirected to OUTPUT_DIR.
|
|
# Example template you might pass:
|
|
# --route-test-cmd 'cd {HCIROOT}/{HCISITE} && . ./.profile && {THREAD} route_test {INPUT} && cp *.out.* {OUTPUT_DIR}/'
|
|
#
|
|
# Usage:
|
|
# nc-regression.sh --scope <SCOPE>
|
|
# --count N
|
|
# --env-a HCIROOT_A --site-a SITENAME
|
|
# --env-b HCIROOT_B --site-b SITENAME
|
|
# --out DIR
|
|
# --route-test-cmd 'TEMPLATE'
|
|
# [--ignore "FIELDS"]
|
|
# [--include-fields "FIELDS"]
|
|
# [--phase 1|2|3|4|5|6|all]
|
|
# [--dry-run]
|
|
# [--inbound-mode tcp-listen|icl-or-file|all]
|
|
# [--env-b-host HOST] [--env-b-user USER] # for scp of inputs
|
|
#
|
|
# Scope formats:
|
|
# thread:NAME one specific thread
|
|
# threads:N1,N2,N3 comma-separated list
|
|
# site every inbound in the configured site
|
|
# server every inbound in every site under HCIROOT
|
|
set -o pipefail
|
|
|
|
NC_SELF="$0"
|
|
LIB_DIR="$(cd "$(dirname "$NC_SELF")" && pwd)"
|
|
NCP="$LIB_DIR/nc-parse.sh"
|
|
NCI="$LIB_DIR/nc-inbound.sh"
|
|
NCM="$LIB_DIR/nc-msgs.sh"
|
|
HL7DIFF="$LIB_DIR/hl7-diff.sh"
|
|
|
|
die() { printf 'nc-regression: %s\n' "$*" >&2; exit 1; }
|
|
say() { printf 'nc-regression: %s\n' "$*" >&2; }
|
|
|
|
SCOPE=""
|
|
COUNT=10
|
|
ENV_A=""
|
|
SITE_A=""
|
|
ENV_B=""
|
|
SITE_B=""
|
|
OUT=""
|
|
ROUTE_TEST_CMD=""
|
|
IGNORE="MSH.7"
|
|
INCLUDE=""
|
|
PHASE="all"
|
|
DRY_RUN=0
|
|
INBOUND_MODE="all"
|
|
ENV_B_HOST=""
|
|
ENV_B_USER=""
|
|
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
--scope) shift; SCOPE="$1" ;;
|
|
--count) shift; COUNT="$1" ;;
|
|
--env-a) shift; ENV_A="$1" ;;
|
|
--site-a) shift; SITE_A="$1" ;;
|
|
--env-b) shift; ENV_B="$1" ;;
|
|
--site-b) shift; SITE_B="$1" ;;
|
|
--out) shift; OUT="$1" ;;
|
|
--route-test-cmd) shift; ROUTE_TEST_CMD="$1" ;;
|
|
--ignore) shift; IGNORE="$1" ;;
|
|
--include-fields) shift; INCLUDE="$1" ;;
|
|
--phase) shift; PHASE="$1" ;;
|
|
--dry-run) DRY_RUN=1 ;;
|
|
--inbound-mode) shift; INBOUND_MODE="$1" ;;
|
|
--env-b-host) shift; ENV_B_HOST="$1" ;;
|
|
--env-b-user) shift; ENV_B_USER="$1" ;;
|
|
-h|--help) sed -n '2,55p' "$NC_SELF"; exit 0 ;;
|
|
-*) die "unknown flag: $1" ;;
|
|
*) die "extra arg: $1" ;;
|
|
esac
|
|
shift
|
|
done
|
|
|
|
[ -n "$SCOPE" ] || die "missing --scope (thread:NAME | threads:N1,N2 | site | server)"
|
|
[ -n "$ENV_A" ] || die "missing --env-a HCIROOT_A"
|
|
[ -n "$ENV_B" ] || die "missing --env-b HCIROOT_B"
|
|
[ -n "$OUT" ] || die "missing --out DIR"
|
|
[ -d "$ENV_A" ] || die "env-a is not a directory: $ENV_A"
|
|
case "$PHASE" in 1|2|3|4|5|6|all) ;; *) die "bad --phase" ;; esac
|
|
[ "$DRY_RUN" = "1" ] || [ -n "$ROUTE_TEST_CMD" ] || say "WARNING: --route-test-cmd is unset; phases 3 and 4 will be skipped (you can run them manually using the generated input files)"
|
|
|
|
mkdir -p "$OUT" "$OUT/inputs" "$OUT/outputs/env-a" "$OUT/outputs/env-b" "$OUT/diff" 2>/dev/null
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Phase 1: discover inbound threads in scope
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
discover_inbounds() {
|
|
case "$SCOPE" in
|
|
thread:*) echo "${SCOPE#thread:}" ;;
|
|
threads:*) echo "${SCOPE#threads:}" | tr ',' '\n' ;;
|
|
site)
|
|
[ -n "$SITE_A" ] || die "scope=site requires --site-a"
|
|
"$NCI" "$ENV_A/$SITE_A/NetConfig" --mode "$INBOUND_MODE" --format tsv \
|
|
| awk -F'\t' 'NR>1 {print $1}'
|
|
;;
|
|
server)
|
|
while IFS= read -r nc; do
|
|
"$NCI" "$nc" --mode "$INBOUND_MODE" --format tsv \
|
|
| awk -F'\t' 'NR>1 {print $1}'
|
|
done < <(find "$ENV_A" -maxdepth 2 -name NetConfig -type f 2>/dev/null)
|
|
;;
|
|
*) die "bad --scope: $SCOPE" ;;
|
|
esac
|
|
}
|
|
|
|
phase_1() {
|
|
say "=== PHASE 1: discover inbounds ==="
|
|
local inbounds; inbounds=$(discover_inbounds | sort -u | grep -v '^$')
|
|
if [ -z "$inbounds" ]; then
|
|
say "no inbounds found in scope $SCOPE"
|
|
return 1
|
|
fi
|
|
printf '%s\n' "$inbounds" > "$OUT/inbounds.txt"
|
|
local count; count=$(printf '%s\n' "$inbounds" | wc -l | tr -d ' ')
|
|
say "discovered $count inbound thread(s) → $OUT/inbounds.txt"
|
|
printf '%s\n' "$inbounds" | sed 's/^/ - /'
|
|
}
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Phase 2: sample N messages per inbound from env-A's smatdbs
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
phase_2() {
|
|
say "=== PHASE 2: sample $COUNT messages per inbound from env-A ==="
|
|
[ -f "$OUT/inbounds.txt" ] || { say "no inbounds file from phase 1; running phase 1 first"; phase_1 || return 1; }
|
|
local thread sitedir
|
|
local count=0
|
|
while IFS= read -r thread; do
|
|
[ -z "$thread" ] && continue
|
|
# Locate the site for this thread
|
|
sitedir=""
|
|
if [ -n "$SITE_A" ]; then
|
|
sitedir="$ENV_A/$SITE_A"
|
|
else
|
|
# Search across all sites under ENV_A for the smatdb
|
|
sitedir=$(find "$ENV_A" -maxdepth 4 -name "${thread}.smatdb" -type f 2>/dev/null | head -1 | xargs -I{} dirname {} | sed "s#/exec/processes/.*##")
|
|
[ -z "$sitedir" ] && { say " skip $thread: smatdb not found under $ENV_A"; continue; }
|
|
fi
|
|
local input="$OUT/inputs/${thread}.msgs"
|
|
if [ "$DRY_RUN" = "1" ]; then
|
|
say " [dry-run] would sample $COUNT msgs from $thread → $input"
|
|
else
|
|
HCISITEDIR="$sitedir" "$NCM" "$thread" --limit "$COUNT" --format raw > "$input" 2>/dev/null
|
|
local got; got=$(tr -cd $'\x1c' < "$input" | wc -c | tr -d ' ')
|
|
say " sampled $thread → $input ($got messages)"
|
|
fi
|
|
count=$((count+1))
|
|
done < "$OUT/inbounds.txt"
|
|
say "phase 2 done: $count thread(s) processed"
|
|
}
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Phase 3 / 4: execute route_test on each env
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
render_cmd() {
|
|
local tmpl="$1" thread="$2" input="$3" outdir="$4" hciroot="$5" hcisite="$6"
|
|
local cmd="$tmpl"
|
|
cmd="${cmd//\{THREAD\}/$thread}"
|
|
cmd="${cmd//\{INPUT\}/$input}"
|
|
cmd="${cmd//\{OUTPUT_DIR\}/$outdir}"
|
|
cmd="${cmd//\{HCIROOT\}/$hciroot}"
|
|
cmd="${cmd//\{HCISITE\}/$hcisite}"
|
|
printf '%s' "$cmd"
|
|
}
|
|
|
|
phase_routes() {
|
|
local label="$1" hciroot="$2" hcisite="$3"
|
|
say "=== PHASE ${label}: route_test on env-${label} ==="
|
|
if [ -z "$ROUTE_TEST_CMD" ]; then
|
|
say "no --route-test-cmd; skipping phase ${label}"
|
|
say "to run manually, use the input files at $OUT/inputs/*.msgs"
|
|
return 0
|
|
fi
|
|
local thread
|
|
while IFS= read -r thread; do
|
|
[ -z "$thread" ] && continue
|
|
local input="$OUT/inputs/${thread}.msgs"
|
|
local outdir="$OUT/outputs/env-${label}/${thread}"
|
|
[ -f "$input" ] || { say " skip $thread: no input file"; continue; }
|
|
mkdir -p "$outdir"
|
|
local cmd; cmd=$(render_cmd "$ROUTE_TEST_CMD" "$thread" "$input" "$outdir" "$hciroot" "$hcisite")
|
|
if [ "$DRY_RUN" = "1" ]; then
|
|
say " [dry-run] $thread:"
|
|
say " \$ $cmd"
|
|
else
|
|
say " $thread:"
|
|
say " \$ $cmd"
|
|
bash -c "$cmd" 2>&1 | sed 's/^/ /' || say " (route_test exit non-zero — continuing)"
|
|
fi
|
|
done < "$OUT/inbounds.txt"
|
|
}
|
|
|
|
phase_3() { phase_routes "a" "$ENV_A" "$SITE_A"; }
|
|
|
|
phase_4() {
|
|
# Phase 4: copy inputs to env-b host (if remote), then route_test on env-B.
|
|
if [ -n "$ENV_B_HOST" ]; then
|
|
say "copying input files to ${ENV_B_USER:-$USER}@${ENV_B_HOST}:${OUT}/inputs/"
|
|
if [ "$DRY_RUN" = "1" ]; then
|
|
say " [dry-run] scp -r $OUT/inputs/ ${ENV_B_USER:-$USER}@${ENV_B_HOST}:${OUT}/"
|
|
else
|
|
ssh "${ENV_B_USER:-$USER}@${ENV_B_HOST}" "mkdir -p $OUT/inputs $OUT/outputs/env-b" || true
|
|
scp -r "$OUT/inputs/" "${ENV_B_USER:-$USER}@${ENV_B_HOST}:${OUT}/" || say "scp failed; you'll need to copy manually"
|
|
fi
|
|
fi
|
|
phase_routes "b" "$ENV_B" "$SITE_B"
|
|
}
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Phase 5: diff outputs pair-by-pair
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
phase_5() {
|
|
say "=== PHASE 5: diff env-a vs env-b outputs ==="
|
|
local diff_index="$OUT/diff/_index.md"
|
|
{
|
|
printf '# Regression diff index\n\n'
|
|
printf '- env-A: `%s`\n- env-B: `%s`\n- scope: `%s`\n- count: %s msgs per inbound\n- ignore: `%s`\n%s\n\n' \
|
|
"$ENV_A" "$ENV_B" "$SCOPE" "$COUNT" "$IGNORE" \
|
|
"$([ -n "$INCLUDE" ] && printf -- '- include-only: `%s`' "$INCLUDE")"
|
|
printf '| thread | dest | diffs | report |\n|---|---|---|---|\n'
|
|
} > "$diff_index"
|
|
|
|
local total_diff=0 total_pairs=0
|
|
local thread destfile destname
|
|
while IFS= read -r thread; do
|
|
[ -z "$thread" ] && continue
|
|
local a_dir="$OUT/outputs/env-a/${thread}"
|
|
local b_dir="$OUT/outputs/env-b/${thread}"
|
|
[ -d "$a_dir" ] || { say " skip $thread: no env-a outputs"; continue; }
|
|
[ -d "$b_dir" ] || { say " skip $thread: no env-b outputs"; continue; }
|
|
# Pair up by filename
|
|
while IFS= read -r destfile; do
|
|
destname=$(basename "$destfile")
|
|
local b_pair="$b_dir/$destname"
|
|
total_pairs=$((total_pairs+1))
|
|
if [ ! -f "$b_pair" ]; then
|
|
echo "| \`$thread\` | \`$destname\` | (missing on env-b) | — |" >> "$diff_index"
|
|
continue
|
|
fi
|
|
local report="$OUT/diff/${thread}.${destname}.md"
|
|
local count
|
|
if [ "$DRY_RUN" = "1" ]; then
|
|
echo " [dry-run] would diff $destfile vs $b_pair → $report"
|
|
echo "| \`$thread\` | \`$destname\` | [dry-run] | — |" >> "$diff_index"
|
|
continue
|
|
fi
|
|
local diff_args=(--ignore "$IGNORE")
|
|
[ -n "$INCLUDE" ] && diff_args+=(--include-fields "$INCLUDE")
|
|
count=$("$HL7DIFF" "${diff_args[@]}" --format count "$destfile" "$b_pair" 2>/dev/null || echo "?")
|
|
{
|
|
printf '# Diff: %s → %s\n\n- env-A: `%s`\n- env-B: `%s`\n\n' "$thread" "$destname" "$destfile" "$b_pair"
|
|
"$HL7DIFF" "${diff_args[@]}" --format text "$destfile" "$b_pair" 2>/dev/null || true
|
|
} > "$report"
|
|
echo "| \`$thread\` | \`$destname\` | $count | [report](./$(basename "$report")) |" >> "$diff_index"
|
|
total_diff=$((total_diff + count))
|
|
say " $thread → $destname: $count diff(s)"
|
|
done < <(find "$a_dir" -maxdepth 1 -type f 2>/dev/null)
|
|
done < "$OUT/inbounds.txt"
|
|
|
|
{
|
|
printf '\n## Summary\n\n- pairs compared: %s\n- total field differences (post-ignore): %s\n' "$total_pairs" "$total_diff"
|
|
} >> "$diff_index"
|
|
say "phase 5 done: $total_pairs pairs compared, $total_diff total diffs"
|
|
say "index: $diff_index"
|
|
}
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Phase 6: master summary
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
phase_6() {
|
|
say "=== PHASE 6: master regression-summary.md ==="
|
|
local summary="$OUT/regression-summary.md"
|
|
{
|
|
printf '# Regression test summary\n\n'
|
|
printf 'Generated: %s\n\n' "$(date -Iseconds 2>/dev/null || date)"
|
|
printf '## Configuration\n\n'
|
|
printf '- scope: `%s`\n' "$SCOPE"
|
|
printf '- count: %s messages per inbound\n' "$COUNT"
|
|
printf '- env-A: `%s` (site=%s)\n' "$ENV_A" "${SITE_A:-auto}"
|
|
printf '- env-B: `%s` (site=%s)\n' "$ENV_B" "${SITE_B:-auto}"
|
|
printf '- ignore: `%s`\n' "$IGNORE"
|
|
[ -n "$INCLUDE" ] && printf '- include-only: `%s`\n' "$INCLUDE"
|
|
printf '\n## Inbounds tested\n\n'
|
|
[ -f "$OUT/inbounds.txt" ] && awk '{print "- `" $0 "`"}' "$OUT/inbounds.txt"
|
|
printf '\n## Inputs\n\n'
|
|
find "$OUT/inputs" -maxdepth 1 -type f 2>/dev/null \
|
|
| awk '{print "- `" $0 "`"}' || true
|
|
printf '\n## Output directories\n\n'
|
|
printf -- '- env-A: `%s`\n' "$OUT/outputs/env-a"
|
|
printf -- '- env-B: `%s`\n' "$OUT/outputs/env-b"
|
|
printf '\n## Diff details\n\n'
|
|
printf 'See [diff/_index.md](./diff/_index.md) for the per-pair table.\n'
|
|
} > "$summary"
|
|
say "summary: $summary"
|
|
}
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Dispatch
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
case "$PHASE" in
|
|
1) phase_1 ;;
|
|
2) phase_1 && phase_2 ;;
|
|
3) phase_3 ;;
|
|
4) phase_4 ;;
|
|
5) phase_5 ;;
|
|
6) phase_6 ;;
|
|
all) phase_1 && phase_2 && phase_3 && phase_4 && phase_5 && phase_6 ;;
|
|
esac
|
|
say "regression run done. Output root: $OUT"
|