From 1709655a9c73e74ea03a948acf3edbf6ce42bf75 Mon Sep 17 00:00:00 2001 From: Bryan Johnson Date: Wed, 27 May 2026 15:52:58 -0700 Subject: [PATCH] v0.6.8: cross-env Cloverleaf workflows over SSH ControlMaster MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the gap between v0.6.7's ssh_exec/ssh_status primitives and the local nc_* tools, so Bryan's two motivating workflows compose cleanly: 1. "Compare the ADT site NetConfig on qa to dev" 2. "Grab smat files from dev and bring to qa for regression testing" ssh_pull, ssh_push (lib/ssh-helper.sh + larry.sh): scp via the existing ControlMaster socket — no second auth, no second TCP handshake. Master-not-open and missing-remote-file paths fail with explicit messages ("open the master with /ssh-setup first"). Pull caches to /tmp/larry-pulls/.. when local_path is omitted, so repeat pulls of the same remote file are idempotent. Validates byte counts post-transfer to catch partial transfers. ssh_pull_smat (lib/ssh-helper.sh + larry.sh): Cloverleaf-aware smatdb pull. Full mode scp's the entire .smatdb; sampled mode (days_back=N) runs sqlite3 server-side via ssh_exec to extract up to 1000 recent messages as TSV with base64-encoded MessageContent blobs (verified end-to-end with a synthetic smatdb fixture matching nc-msgs.sh's smat_msgs schema). Avoids transferring multi-GB archives when only N samples are needed. nc_diff_interface tool (newly wired): Promotes lib/nc-diff-interface.sh into the LLM-callable tool surface. Used by the new /nc-diff-env slash command for workflow #1. nc_regression cross-env (lib/nc-regression.sh + larry.sh): source_ssh_alias / target_ssh_alias args. Phase 1 (discovery) and Phase 2 (sample) run via ssh_exec + ssh_pull / ssh_pull_smat against the source alias. Phase 3/4 (route_test) push inputs over and pull outputs back via ssh_push / ssh_pull. Phases 5/6 (diff + summary) stay local. Reports reference the SSH alias names rather than raw user@host strings. /nc-diff-env and /nc-regression-env slash commands (larry.sh): Templated prompts to Larry-the-LLM that explicitly cite the motivating workflows, call out ssh_status / ssh_pull / nc_diff_interface and the nc_regression cross-env fields. Registered in _LARRY_SLASH_CMDS + _LARRY_SLASH_CMDS_DESC + /help per v0.6.7 patterns. Bug fix unearthed during cross-env work: lib/nc-regression.sh phase_5 / phase_6 used printf 'FORMAT' where FORMAT begins with '- '. bash 3.2 (macOS default) reads the leading '-' as a bad option and emits nothing — silently dropping the entire "Configuration" section of regression-summary.md. Switched the affected lines to printf -- 'FORMAT' so the format string is unambiguous. Tool/slash surface deltas vs v0.6.7: Tools: 31 → 35 (+ssh_pull, +ssh_push, +ssh_pull_smat, +nc_diff_interface) Slash commands: 34 → 36 (+/nc-diff-env, +/nc-regression-env) Updated tool descriptions for read_file, grep_files, nc_msgs to point at ssh_pull / ssh_pull_smat as the cross-env pre-step so Larry-the-LLM picks the right chain on the first attempt. Co-Authored-By: Claude Opus 4.7 --- VERSION | 2 +- larry.sh | 215 +++++++++++++++++++++++++++++++++++++--- lib/nc-regression.sh | 228 +++++++++++++++++++++++++++++++++++-------- lib/ssh-helper.sh | 203 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 594 insertions(+), 54 deletions(-) diff --git a/VERSION b/VERSION index 2228cad..fae59ca 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.6.7 +0.6.8 diff --git a/larry.sh b/larry.sh index f648dd3..98fdc43 100755 --- a/larry.sh +++ b/larry.sh @@ -43,7 +43,7 @@ set -o pipefail # ───────────────────────────────────────────────────────────────────────────── # Config # ───────────────────────────────────────────────────────────────────────────── -LARRY_VERSION="0.6.7" +LARRY_VERSION="0.6.8" LARRY_HOME="${LARRY_HOME:-$HOME/.larry}" LARRY_BASE_URL="${LARRY_BASE_URL:-https://raw.githubusercontent.com/bojj27/cloverleaf-larry/main}" LARRY_UPDATE_URL="${LARRY_UPDATE_URL:-${LARRY_BASE_URL}/larry.sh}" @@ -745,14 +745,19 @@ tool_nc_add_route() { tool_nc_regression() { local scope="$1" count="$2" env_a="$3" site_a="$4" env_b="$5" site_b="$6" out_dir="$7" local route_cmd="${8:-}" ignore="${9:-MSH.7}" phase="${10:-all}" dry_run="${11:-0}" + local source_ssh_alias="${12:-}" target_ssh_alias="${13:-}" _lib_err_if_missing || return local args=(--scope "$scope" --count "$count" --env-a "$env_a" --env-b "$env_b" --out "$out_dir" \ --ignore "$ignore" --phase "$phase") - [ -n "$site_a" ] && args+=(--site-a "$site_a") - [ -n "$site_b" ] && args+=(--site-b "$site_b") - [ -n "$route_cmd" ] && args+=(--route-test-cmd "$route_cmd") - [ "$dry_run" = "1" ] && args+=(--dry-run) - "$LARRY_LIB_DIR/nc-regression.sh" "${args[@]}" 2>&1 + [ -n "$site_a" ] && args+=(--site-a "$site_a") + [ -n "$site_b" ] && args+=(--site-b "$site_b") + [ -n "$route_cmd" ] && args+=(--route-test-cmd "$route_cmd") + [ "$dry_run" = "1" ] && args+=(--dry-run) + [ -n "$source_ssh_alias" ] && args+=(--source-ssh-alias "$source_ssh_alias") + [ -n "$target_ssh_alias" ] && args+=(--target-ssh-alias "$target_ssh_alias") + # Pass our resolved lib dir so the regression script can reach ssh-helper.sh + # without re-resolving from its own $0. + LARRY_LIB_DIR="$LARRY_LIB_DIR" "$LARRY_LIB_DIR/nc-regression.sh" "${args[@]}" 2>&1 } tool_hl7_diff() { @@ -1195,6 +1200,59 @@ tool_ssh_status() { "$helper" status 2>&1 } +# ── v0.6.8: cross-env file transfer over the open ControlMaster ──────────── +# ssh_pull pulls a remote file → local; ssh_push pushes local → remote. Both +# multiplex via the existing master socket (set up by /ssh-setup ALIAS) — no +# second auth, no second TCP handshake. +tool_ssh_pull() { + local alias="$1" remote="$2" local_path="${3:-}" + local helper="$LARRY_LIB_DIR/ssh-helper.sh" + [ -x "$helper" ] || { echo "ERROR: ssh-helper.sh not installed"; return 1; } + [ -n "$alias" ] && [ -n "$remote" ] || { echo "ERROR: ssh_pull needs alias and remote_path"; return 1; } + local out rc + if [ -n "$local_path" ]; then + out=$("$helper" pull "$alias" "$remote" "$local_path" 2>&1); rc=$? + else + out=$("$helper" pull "$alias" "$remote" 2>&1); rc=$? + fi + printf '%s\n[ssh_pull: exit rc=%d]\n' "$out" "$rc" +} + +tool_ssh_push() { + local alias="$1" local_path="$2" remote="$3" + local helper="$LARRY_LIB_DIR/ssh-helper.sh" + [ -x "$helper" ] || { echo "ERROR: ssh-helper.sh not installed"; return 1; } + [ -n "$alias" ] && [ -n "$local_path" ] && [ -n "$remote" ] \ + || { echo "ERROR: ssh_push needs alias, local_path, and remote_path"; return 1; } + local out rc + out=$("$helper" push "$alias" "$local_path" "$remote" 2>&1); rc=$? + printf '%s\n[ssh_push: exit rc=%d]\n' "$out" "$rc" +} + +tool_ssh_pull_smat() { + local alias="$1" site="$2" thread="$3" days_back="${4:-}" + local helper="$LARRY_LIB_DIR/ssh-helper.sh" + [ -x "$helper" ] || { echo "ERROR: ssh-helper.sh not installed"; return 1; } + [ -n "$alias" ] && [ -n "$site" ] && [ -n "$thread" ] \ + || { echo "ERROR: ssh_pull_smat needs alias, site, thread"; return 1; } + local out rc + if [ -n "$days_back" ]; then + out=$("$helper" pull-smat "$alias" "$site" "$thread" "$days_back" 2>&1); rc=$? + else + out=$("$helper" pull-smat "$alias" "$site" "$thread" 2>&1); rc=$? + fi + # Cap returned bytes — sampled-mode b64 blobs can be sizable. Hard ceiling + # at ~400 KB so tool result stays in a reasonable bound; truncation is + # explicit so Larry-the-LLM can react and re-pull with smaller days_back. + local bytes; bytes=$(printf '%s' "$out" | wc -c | tr -d ' ') + if [ "$bytes" -gt 409600 ]; then + out=$(printf '%s' "$out" | head -c 409600) + printf '%s\n[ssh_pull_smat: output truncated at 400 KB; re-run with smaller days_back. exit rc=%d]\n' "$out" "$rc" + else + printf '%s\n[ssh_pull_smat: exit rc=%d]\n' "$out" "$rc" + fi +} + tool_lesson_record() { local text="$1" topic="${2:-}" site="${3:-${HCISITE:-}}" severity="${4:-info}" _lib_err_if_missing || return @@ -1234,6 +1292,20 @@ tool_nc_document() { "$LARRY_LIB_DIR/nc-document.sh" "${args[@]}" 2>&1 } +tool_nc_diff_interface() { + local interface="$1" left="$2" right="$3" out_path="${4:-}" include_tables="${5:-0}" + local left_label="${6:-}" right_label="${7:-}" depth="${8:-1}" + _lib_err_if_missing || return + [ -n "$interface" ] && [ -n "$left" ] && [ -n "$right" ] \ + || { echo "ERROR: nc_diff_interface needs interface, left, right"; return 1; } + local args=(--interface "$interface" --left "$left" --right "$right" --depth "$depth") + [ -n "$out_path" ] && args+=(--out "$out_path") + [ "$include_tables" = "1" ] && args+=(--include-tables) + [ -n "$left_label" ] && args+=(--left-label "$left_label") + [ -n "$right_label" ] && args+=(--right-label "$right_label") + "$LARRY_LIB_DIR/nc-diff-interface.sh" "${args[@]}" 2>&1 +} + tool_bash_exec() { local cmd="$1" printf '\n%s══ bash_exec ══%s\n' "$C_YELLOW" "$C_RESET" >&2 @@ -1288,14 +1360,22 @@ execute_tool() { nc_insert_protocol) tool_nc_insert_protocol "$(J '.netconfig')" "$(J '.block')" "$(J '.mode // "end"')" "$(J '.anchor // ""')" ;; nc_add_route) tool_nc_add_route "$(J '.netconfig')" "$(J '.protocol_name')" "$(J '.route')" ;; hl7_diff) tool_hl7_diff "$(J '.left')" "$(J '.right')" "$(J '.ignore // "MSH.7"')" "$(J '.include // ""')" "$(J '.format // "text"')" ;; + nc_diff_interface) tool_nc_diff_interface "$(J '.interface')" "$(J '.left')" "$(J '.right')" "$(J '.out // ""')" \ + "$(J '.include_tables // 0' | sed "s/false/0/;s/true/1/")" \ + "$(J '.left_label // ""')" "$(J '.right_label // ""')" \ + "$(J '.depth // 1')" ;; nc_regression) tool_nc_regression "$(J '.scope')" "$(J '.count // 10')" "$(J '.env_a')" "$(J '.site_a // ""')" \ "$(J '.env_b')" "$(J '.site_b // ""')" "$(J '.out')" \ "$(J '.route_test_cmd // ""')" "$(J '.ignore // "MSH.7"')" \ - "$(J '.phase // "all"')" "$(J '.dry_run // 0' | sed "s/false/0/;s/true/1/")" ;; + "$(J '.phase // "all"')" "$(J '.dry_run // 0' | sed "s/false/0/;s/true/1/")" \ + "$(J '.source_ssh_alias // ""')" "$(J '.target_ssh_alias // ""')" ;; lesson_record) tool_lesson_record "$(J '.text')" "$(J '.topic // ""')" "$(J '.site // ""')" "$(J '.severity // "info"')" ;; hl7_sanitize) tool_hl7_sanitize "$(J '.input_path')" "$(J '.strict // 0' | sed "s/false/0/;s/true/1/")" ;; ssh_exec) tool_ssh_exec "$(J '.alias')" "$(J '.command')" "$(J '.max_lines // 500')" ;; ssh_status) tool_ssh_status ;; + ssh_pull) tool_ssh_pull "$(J '.alias')" "$(J '.remote_path')" "$(J '.local_path // ""')" ;; + ssh_push) tool_ssh_push "$(J '.alias')" "$(J '.local_path')" "$(J '.remote_path')" ;; + ssh_pull_smat) tool_ssh_pull_smat "$(J '.alias')" "$(J '.site')" "$(J '.thread')" "$(J '.days_back // ""')" ;; larry_rollback_list) tool_larry_rollback_list "$(J '.session // ""')" ;; *) echo "ERROR: unknown tool: $name" ;; esac @@ -1306,9 +1386,9 @@ execute_tool() { # ───────────────────────────────────────────────────────────────────────────── TOOLS_JSON=$(cat <<'TOOLS_END' [ - {"name":"read_file","description":"Read a single regular file. Returns content with line numbers. Max 250KB; use grep_files for larger.","input_schema":{"type":"object","properties":{"path":{"type":"string","description":"Path to file (absolute or relative to cwd)."}},"required":["path"]}}, + {"name":"read_file","description":"Read a single LOCAL regular file. Returns content with line numbers. Max 250KB; use grep_files for larger. For files on a remote SSH-aliased host, use ssh_pull first to fetch the file locally, then read the returned local path.","input_schema":{"type":"object","properties":{"path":{"type":"string","description":"Path to file (absolute or relative to cwd)."}},"required":["path"]}}, {"name":"list_dir","description":"List a directory (ls -la). Use to map a Cloverleaf site_root.","input_schema":{"type":"object","properties":{"path":{"type":"string","description":"Directory path. Defaults to current dir."}},"required":["path"]}}, - {"name":"grep_files","description":"Recursive grep across files. Use for finding TCL procs, UPOC declarations, segment references, etc. Returns up to 300 matching lines with file:line:content.","input_schema":{"type":"object","properties":{"pattern":{"type":"string","description":"Regex pattern (grep -E style)."},"path":{"type":"string","description":"Starting directory."}},"required":["pattern","path"]}}, + {"name":"grep_files","description":"Recursive grep across LOCAL files only. Use for finding TCL procs, UPOC declarations, segment references, etc. Returns up to 300 matching lines with file:line:content. To grep remote files, use ssh_exec with grep, or ssh_pull the file first.","input_schema":{"type":"object","properties":{"pattern":{"type":"string","description":"Regex pattern (grep -E style)."},"path":{"type":"string","description":"Starting directory."}},"required":["pattern","path"]}}, {"name":"glob_files","description":"Find files by name pattern. Up to 300 paths.","input_schema":{"type":"object","properties":{"pattern":{"type":"string","description":"Shell glob like *.tcl or *Inbound*"},"path":{"type":"string","description":"Starting directory."}},"required":["pattern","path"]}}, {"name":"write_file","description":"Write content to a path. ALWAYS prompts Bryan for Y/N before writing. Shows a unified diff if file exists, or a preview if new.","input_schema":{"type":"object","properties":{"path":{"type":"string"},"content":{"type":"string"}},"required":["path","content"]}}, {"name":"bash_exec","description":"Run a shell command. ALWAYS prompts Bryan for Y/N before running. Output capped at 500 lines.","input_schema":{"type":"object","properties":{"command":{"type":"string","description":"Single command line, passed to bash -c."}},"required":["command"]}}, @@ -1327,7 +1407,7 @@ TOOLS_JSON=$(cat <<'TOOLS_END' {"name":"nc_sources","description":"List every protocol that has a DATAXLATE DEST routing to the named thread. The inverse of nc_destinations. Use this to find what feeds a given thread.","input_schema":{"type":"object","properties":{"netconfig":{"type":"string"},"name":{"type":"string","description":"Target thread name."}},"required":["netconfig","name"]}}, {"name":"nc_tclproc_refs","description":"List every TCL proc name referenced from a protocol block (or from the whole NetConfig if name is omitted). Pulls from DATAFORMAT.PROC, PREPROCS.PROCS, POSTPROCS.PROCS, etc. Unique sorted.","input_schema":{"type":"object","properties":{"netconfig":{"type":"string"},"name":{"type":"string","description":"Optional. Scope to one protocol."}},"required":["netconfig"]}}, {"name":"hl7_field","description":"Extract a specific HL7 v2 field from a message. field_path = SEG[.FIELD[.COMPONENT[.SUBCOMPONENT]]]. Examples: PID.3 (MRN), PID.18 (account number), MSH.7 (timestamp), MSH.9.2 (event code, like A08), PID.5 (patient name with components). Multiple repetitions are returned one per line. Native v3, no v1/v2 dependency.","input_schema":{"type":"object","properties":{"message":{"type":"string","description":"Raw HL7 message text. Segments separated by \\r."},"field_path":{"type":"string","description":"Field path like PID.3 or MSH.9.2"}},"required":["message","field_path"]}}, - {"name":"nc_msgs","description":"Query Cloverleaf smat (SQLite!) databases for messages from a thread. Filters: time range, exact HL7 field match. Native v3 — reads smatdb directly with sqlite3 -ascii, no hcidbdump/dbExtract needed. Format text shows messages line-by-line with metadata; count returns just the count; json returns structured data.","input_schema":{"type":"object","properties":{"thread":{"type":"string","description":"Thread name. The .smatdb file under $HCISITEDIR/exec/processes/*/.smatdb is auto-located unless db is given."},"after":{"type":"string","description":"Time-after filter. Accepts \"3 days ago\", \"2026-05-20 14:30:00\", \"2026-05-20\", or a unix timestamp."},"before":{"type":"string","description":"Time-before filter, same formats as after."},"field":{"type":"string","description":"HL7 field path for exact-match filter, e.g. PID.18 or MSH.10."},"value":{"type":"string","description":"Value the field must equal. Use with field. Repeatable filters not supported via this single tool call — chain calls if you need multi-field AND."},"limit":{"type":"integer","description":"Max messages to return. Default 10."},"format":{"type":"string","enum":["text","json","count","raw"],"description":"text = human-readable with metadata; count = just the number; json = structured; raw = raw bytes separated by 0x1c."},"sitedir":{"type":"string","description":"Override $HCISITEDIR for thread-to-db location."},"db":{"type":"string","description":"Explicit .smatdb path; overrides auto-locate."}},"required":["thread"]}}, + {"name":"nc_msgs","description":"Query Cloverleaf smat (SQLite!) databases for messages from a thread. Filters: time range, exact HL7 field match. Native v3 — reads smatdb directly with sqlite3 -ascii, no hcidbdump/dbExtract needed. Format text shows messages line-by-line with metadata; count returns just the count; json returns structured data. Operates on LOCAL smatdbs; for a remote env's smatdb, use ssh_pull_smat first (sampled mode is cheaper than pulling the whole DB).","input_schema":{"type":"object","properties":{"thread":{"type":"string","description":"Thread name. The .smatdb file under $HCISITEDIR/exec/processes/*/.smatdb is auto-located unless db is given."},"after":{"type":"string","description":"Time-after filter. Accepts \"3 days ago\", \"2026-05-20 14:30:00\", \"2026-05-20\", or a unix timestamp."},"before":{"type":"string","description":"Time-before filter, same formats as after."},"field":{"type":"string","description":"HL7 field path for exact-match filter, e.g. PID.18 or MSH.10."},"value":{"type":"string","description":"Value the field must equal. Use with field. Repeatable filters not supported via this single tool call — chain calls if you need multi-field AND."},"limit":{"type":"integer","description":"Max messages to return. Default 10."},"format":{"type":"string","enum":["text","json","count","raw"],"description":"text = human-readable with metadata; count = just the number; json = structured; raw = raw bytes separated by 0x1c."},"sitedir":{"type":"string","description":"Override $HCISITEDIR for thread-to-db location."},"db":{"type":"string","description":"Explicit .smatdb path; overrides auto-locate."}},"required":["thread"]}}, {"name":"nc_document","description":"Generate a complete markdown knowledge entry for a Cloverleaf subsystem identified by a name pattern. Walks every NetConfig under $HCIROOT, gathers config + sources + destinations + xlates + tclprocs for every matching thread, composes a markdown doc with placeholder context sections (Vendor POC, Internal Owner, Status, Escalation, Open items, Notes). Returns the doc text and (if out is given) writes it to that path.","input_schema":{"type":"object","properties":{"name":{"type":"string","description":"Case-insensitive substring/regex to match protocol names. e.g. 'codametrix', 'epic_adt', '3M'."},"out":{"type":"string","description":"Optional output file path. Convention: $LARRY_HOME/knowledge/.md."},"hciroot":{"type":"string","description":"Override $HCIROOT for the NetConfig scan."},"title":{"type":"string","description":"Doc title. Default derived from name."},"status":{"type":"string","description":"System status fill-in (production/test/decommissioning/...)."},"poc_internal":{"type":"string","description":"Internal owner fill-in."},"poc_vendor":{"type":"string","description":"Vendor POC fill-in."},"escalation":{"type":"string","description":"Escalation path fill-in."},"open_items":{"type":"string","description":"Open items / known issues fill-in. Can be multi-line, will be inserted as-is."},"notes":{"type":"string","description":"Freeform notes fill-in."}},"required":["name"]}}, {"name":"nc_find","description":"Cross-site thread search. Native v3 replacement for v1 tbn/tbp/tbh/tbpr/where. Walks every NetConfig under $HCIROOT and returns matching threads with site, port, host, process, direction, file, line. Modes: name=partial name match (like tbn); port=exact port (like tbp); host=substring on host (like tbh); process=substring on PROCESSNAME (like tbpr); where=exact name match across all sites (like the v1 ` where`); xlate=threads referencing a specific .xlt; tclproc=threads referencing a specific TCL proc.","input_schema":{"type":"object","properties":{"mode":{"type":"string","enum":["name","port","host","process","where","xlate","tclproc"],"description":"Search mode."},"query":{"type":"string","description":"Query value: partial name, port number, host substring, process name, exact thread name, xlate filename, or tclproc name."},"format":{"type":"string","enum":["table","tsv","jsonl"],"description":"Output format. Default table."},"hciroot":{"type":"string","description":"Override $HCIROOT."}},"required":["mode","query"]}}, @@ -1345,7 +1425,15 @@ TOOLS_JSON=$(cat <<'TOOLS_END' {"name":"hl7_diff","description":"HL7-aware diff between two message files (or multi-message dumps). Compares segment-by-segment, field-by-field, with component and subcomponent precision. Ignores configured fields (default MSH.7 timestamp) so timestamp-only diffs do not show up as noise. Use for regression testing between environments (e.g. test vs prod route-test outputs).","input_schema":{"type":"object","properties":{"left":{"type":"string","description":"Path to left HL7 file."},"right":{"type":"string","description":"Path to right HL7 file."},"ignore":{"type":"string","description":"Comma-separated list of fields to ignore (e.g. MSH.7,MSH.10,EVN.6). Default MSH.7."},"include":{"type":"string","description":"If set, ONLY these fields are compared (overrides ignore for that set)."},"format":{"type":"string","enum":["text","tsv","count"],"description":"text=human-readable diff, tsv=machine-parseable, count=just the difference count."}},"required":["left","right"]}}, - {"name":"nc_regression","description":"End-to-end regression testing between two Cloverleaf environments. 6 phases: discover inbounds in scope, sample N messages per inbound from env-A smatdbs, run route_test on env-A, run route_test on env-B with same inputs, hl7_diff every paired output file, compile summary report. Phases 3/4 require the Cloverleaf route_test command; pass it via route_test_cmd with placeholders {THREAD} {INPUT} {OUTPUT_DIR} {HCIROOT} {HCISITE}. If route_test_cmd is empty, phases 3/4 are skipped and you can run them manually using the generated input files.","input_schema":{"type":"object","properties":{"scope":{"type":"string","description":"thread:NAME | threads:N1,N2 | site (needs site_a) | server (all sites)"},"count":{"type":"integer","description":"Messages to sample per inbound. Default 10."},"env_a":{"type":"string","description":"HCIROOT of env-A (the test/source env)."},"site_a":{"type":"string","description":"Site name on env-A. Required if scope=site."},"env_b":{"type":"string","description":"HCIROOT of env-B (the prod/target env)."},"site_b":{"type":"string","description":"Site name on env-B."},"out":{"type":"string","description":"Output root directory for inputs, outputs, diffs, and summary."},"route_test_cmd":{"type":"string","description":"Command template for invoking route_test. Use {THREAD} {INPUT} {OUTPUT_DIR} {HCIROOT} {HCISITE} as placeholders."},"ignore":{"type":"string","description":"hl7_diff ignore list. Default MSH.7."},"phase":{"type":"string","enum":["1","2","3","4","5","6","all"],"description":"Run a specific phase or all. Default all."},"dry_run":{"type":"integer","description":"1 = print what would happen, do not execute. Default 0."}},"required":["scope","env_a","env_b","out"]}} + {"name":"nc_regression","description":"End-to-end regression testing between two Cloverleaf environments. 6 phases: discover inbounds in scope, sample N messages per inbound from env-A smatdbs, run route_test on env-A, run route_test on env-B with same inputs, hl7_diff every paired output file, compile summary report. Phases 3/4 require the Cloverleaf route_test command; pass it via route_test_cmd with placeholders {THREAD} {INPUT} {OUTPUT_DIR} {HCIROOT} {HCISITE}. If route_test_cmd is empty, phases 3/4 are skipped and you can run them manually using the generated input files. For cross-env regression testing across SSH-aliased hosts, set source_ssh_alias and target_ssh_alias to existing SSH aliases (run ssh_status to list them first). When set, phases 1–4 run remotely via ssh_exec + ssh_pull/ssh_push; phases 5–6 stay local. env_a / env_b remain the HCIROOT paths AS SEEN ON THE REMOTE for that alias.","input_schema":{"type":"object","properties":{"scope":{"type":"string","description":"thread:NAME | threads:N1,N2 | site (needs site_a) | server (all sites)"},"count":{"type":"integer","description":"Messages to sample per inbound. Default 10."},"env_a":{"type":"string","description":"HCIROOT of env-A (the test/source env). If source_ssh_alias is set, this is the remote-side path."},"site_a":{"type":"string","description":"Site name on env-A. Required if scope=site."},"env_b":{"type":"string","description":"HCIROOT of env-B (the prod/target env). If target_ssh_alias is set, this is the remote-side path."},"site_b":{"type":"string","description":"Site name on env-B."},"out":{"type":"string","description":"LOCAL output root directory for inputs, outputs, diffs, and summary."},"route_test_cmd":{"type":"string","description":"Command template for invoking route_test. Use {THREAD} {INPUT} {OUTPUT_DIR} {HCIROOT} {HCISITE} as placeholders."},"ignore":{"type":"string","description":"hl7_diff ignore list. Default MSH.7."},"phase":{"type":"string","enum":["1","2","3","4","5","6","all"],"description":"Run a specific phase or all. Default all."},"dry_run":{"type":"integer","description":"1 = print what would happen, do not execute. Default 0."},"source_ssh_alias":{"type":"string","description":"SSH alias for the env-A (source) host. When set, phases 1–3 run remotely. Master must be open (ssh_status). Default empty = local."},"target_ssh_alias":{"type":"string","description":"SSH alias for the env-B (target) host. When set, phase 4 runs remotely. Master must be open. Default empty = local."}},"required":["scope","env_a","env_b","out"]}}, + + {"name":"ssh_pull","description":"Pull a file from a remote SSH-aliased host to a local path via the existing ControlMaster (no second auth, no second TCP handshake). Use this BEFORE calling any local tool (read_file, nc_diff_interface, grep_files, hl7_diff, etc.) when the source file lives on a remote host. The local path returned by this tool is stable for re-use within and across turns — pulling the same remote_path again returns the same local_path. If local_path is omitted, a deterministic temp path /tmp/larry-pulls/.. is used. Verifies the master is open first; if not, fails with a clear message ('open the master with /ssh-setup first'). Validates the transferred size matches the remote stat.","input_schema":{"type":"object","properties":{"alias":{"type":"string","description":"SSH alias (see ssh_status). Master must be open."},"remote_path":{"type":"string","description":"Absolute path on the remote host."},"local_path":{"type":"string","description":"Optional explicit local destination. If omitted, a deterministic /tmp/larry-pulls/.. path is used and printed in the tool result."}},"required":["alias","remote_path"]}}, + + {"name":"ssh_push","description":"Push a local file to a remote SSH-aliased host via the existing ControlMaster. Use for sending small input bundles to a remote env (e.g. regression-test input messages, a sanitized HL7 file to feed into route_test). Same multiplexing + error handling as ssh_pull. Validates remote-side size matches local size post-transfer.","input_schema":{"type":"object","properties":{"alias":{"type":"string","description":"SSH alias (see ssh_status). Master must be open."},"local_path":{"type":"string","description":"Absolute local path to the file to send."},"remote_path":{"type":"string","description":"Absolute remote destination path."}},"required":["alias","local_path","remote_path"]}}, + + {"name":"ssh_pull_smat","description":"Pull a Cloverleaf thread's smat archive (or recent messages from it) from a remote env. Two modes: (1) Full pull — omit days_back; the entire .smatdb file is scp'd locally; returns the local path. Fine for small archives. (2) Sampled — pass days_back=N; runs sqlite3 server-side to pull just messages from the last N days as TSV with base64-encoded blobs (unix_tsdirectiontypesourcedestmessage_blob_b64). Capped at 1000 rows; the trailer line reports truncated=yes/no. Avoids transferring multi-GB smatdbs when only N samples are needed. Uses ssh_exec under the hood to find the .smatdb path (the file lives at $HCISITEDIR/exec/processes/*/.smatdb on the remote, where * is a process name that varies by site).","input_schema":{"type":"object","properties":{"alias":{"type":"string","description":"SSH alias (see ssh_status). Master must be open."},"site":{"type":"string","description":"Cloverleaf HCISITE name on the remote — used to resolve $HCISITEDIR=$HCIROOT/."},"thread":{"type":"string","description":"Thread name (e.g. IB_ADT_muxS). The .smatdb is auto-located via find on the remote."},"days_back":{"type":"integer","description":"Optional. If set, sampled mode: only messages from the last N days are returned, base64-encoded, capped at 1000 rows. Omit for full-file pull."}},"required":["alias","site","thread"]}}, + + {"name":"nc_diff_interface","description":"Diff one Cloverleaf interface across two NetConfigs. Compares the protocol block plus referenced xlates, tclprocs, and (optionally) tables. Operates on LOCAL NetConfig paths. If a NetConfig file is on a remote host, first use ssh_pull to fetch it locally (and the related Xlate/, tclprocs/, tables/ dirs alongside), then pass the local paths here. The site root is dirname(NetConfig); related artifacts (Xlate/, tclprocs/, tables/) must be alongside that file.","input_schema":{"type":"object","properties":{"interface":{"type":"string","description":"Protocol/thread name to diff. e.g. ADTto_3m."},"left":{"type":"string","description":"Local path to the LEFT NetConfig file (e.g. dev)."},"right":{"type":"string","description":"Local path to the RIGHT NetConfig file (e.g. qa)."},"out":{"type":"string","description":"Optional output path for the markdown report. Default stdout."},"include_tables":{"type":"integer","description":"1 = also diff referenced tables. Default 0."},"left_label":{"type":"string","description":"Display label for left side (default A)."},"right_label":{"type":"string","description":"Display label for right side (default B)."},"depth":{"type":"integer","description":"Hops out from the named interface to also diff. Default 1."}},"required":["interface","left","right"]}} ] TOOLS_END ) @@ -1857,6 +1945,13 @@ Slash commands: /ssh run command on the remote (you-driven, ad-hoc) Larry can also run things there via the ssh_exec tool. + Cross-environment Cloverleaf shortcuts (v0.6.8): + /nc-diff-env [pattern] diff NetConfigs across two SSH-aliased envs + (e.g. /nc-diff-env qa dev ADT) + /nc-regression-env [scope] + 6-phase regression across SSH-aliased envs + (e.g. /nc-regression-env dev qa server) + PHI inline syntax in any prompt: @@VALUE EASY: wrap PHI in @@. Spaceless = no end delim. e.g. @@12345 @@SMITH^JOHN @@V789 @@ -1974,6 +2069,8 @@ _LARRY_SLASH_CMDS=( /copy /cost /show-last-tool + /nc-diff-env + /nc-regression-env ) # _LARRY_SLASH_CMDS_DESC — one-line descriptions for each slash command. @@ -2018,6 +2115,8 @@ _LARRY_SLASH_CMDS_DESC=( [/copy]="copy last assistant response to clipboard" [/cost]="show running token + dollar cost for the session" [/show-last-tool]="print full last tool call + result for debugging" + [/nc-diff-env]=" [pattern] diff NetConfigs across two SSH-aliased envs" + [/nc-regression-env]=" [scope] 6-phase regression across SSH-aliased envs" ) # __larry_complete_slash — bound to TAB via `bind -x` (see _install_readline_tab). @@ -2511,6 +2610,100 @@ main_loop() { if [ ! -f "$f" ]; then err "no such file: $f"; continue; fi input="$(cat "$f")" larry_say "loaded $(wc -l < "$f" | tr -d ' ') lines from $f as your next message" ;; + # v0.6.8: cross-env convenience commands. These templatize a prompt and + # hand it to Larry-the-LLM to execute via the existing tools (no new + # control flow). The prompt cites the motivating workflow so the model + # picks the right tool chain unambiguously. + /nc-diff-env*) + local rest; rest=$(_slash_args "/nc-diff-env" "$input") + if [ -z "$rest" ]; then + err "usage: /nc-diff-env [pattern]"; continue + fi + # Tokenize positional args: env_a, env_b, optional pattern. + local _ea _eb _pat + _ea="${rest%% *}"; rest="${rest#"$_ea"}"; rest="${rest# }" + _eb="${rest%% *}"; rest="${rest#"$_eb"}"; rest="${rest# }" + _pat="$rest" + if [ -z "$_ea" ] || [ -z "$_eb" ]; then + err "usage: /nc-diff-env [pattern]"; continue + fi + input=$(cat <}. + +Plan and execute: +1. Run ssh_status to confirm both aliases have an open ControlMaster. If + either is closed, stop and tell me to run /ssh-setup . +2. Use ssh_exec to locate the NetConfig paths on each env (e.g. + find \$HCIROOT -maxdepth 3 -name NetConfig -type f), or ask me for the + site name if HCIROOT isn't exported on the remote. +3. ssh_pull each NetConfig locally. Also pull the matching Xlate/, tclprocs/, + tables/ directories alongside if you intend to diff referenced artifacts. +4. Use nc_diff_interface with --interface set per protocol, --left and --right + pointing at the two local NetConfigs. If a pattern was given, restrict the + set of protocols to those matching $_pat (use nc_list_protocols + a filter). +5. Report each difference with file-path references back to the source envs + (alias:remote_path so I can copy-paste back into ssh). + +Be terse. One section per protocol. Aggregate identical diffs. +EOF +) + larry_say "/nc-diff-env: templated prompt prepared for $_ea vs $_eb${_pat:+ pattern=$_pat}" + ;; + /nc-regression-env*) + local rest; rest=$(_slash_args "/nc-regression-env" "$input") + if [ -z "$rest" ]; then + err "usage: /nc-regression-env [scope]"; continue + fi + local _src _tgt _scope + _src="${rest%% *}"; rest="${rest#"$_src"}"; rest="${rest# }" + _tgt="${rest%% *}"; rest="${rest#"$_tgt"}"; rest="${rest# }" + _scope="${rest:-server}" + if [ -z "$_src" ] || [ -z "$_tgt" ]; then + err "usage: /nc-regression-env [scope]"; continue + fi + local _ts; _ts=$(date +%Y%m%d-%H%M%S) + local _out="$LARRY_HOME/regression/$_ts" + input=$(cat <. +2. Discover the remote HCIROOT for each alias (ssh_exec 'echo \$HCIROOT'). If + not exported, ask me. Same for HCISITE if scope=site. +3. Call nc_regression with: + - scope = "$_scope" + - source_ssh_alias = "$_src" + - target_ssh_alias = "$_tgt" + - env_a = + - env_b = + - out = "$_out" + - count = 10 (messages sampled per inbound) + - route_test_cmd = use the existing default if I haven't given you one; + otherwise prompt me with a one-liner template I should approve. + - phase = "all" +4. After the run, read the compiled report at $_out/regression-summary.md + and read $_out/diff/_index.md, then summarize: + - threads tested, + - pairs compared, + - total field differences post-ignore, + - any threads where one env had outputs the other didn't. +5. Reference the SSH alias names ($_src and $_tgt) in your summary, not + raw user@host strings. +EOF +) + larry_say "/nc-regression-env: templated prompt prepared for $_src → $_tgt (scope=$_scope, out=$_out)" + ;; /*) err "unknown command: $input (try /help)"; continue ;; esac diff --git a/lib/nc-regression.sh b/lib/nc-regression.sh index abaa60e..a0d8f00 100755 --- a/lib/nc-regression.sh +++ b/lib/nc-regression.sh @@ -72,6 +72,11 @@ ENV_B_HOST="" ENV_B_USER="" BUNDLE_OUT="" # after env-A phases, tar up the artifacts here BUNDLE_IN="" # at start, untar a bundle here as the env-A artifacts +# v0.6.8: cross-env via ssh-helper.sh ControlMaster aliases. When set, phases +# 1–4 do their reads/writes against the named remote alias; phases 5–6 always +# run locally because all the artifacts live in $OUT (local). +SOURCE_SSH_ALIAS="" +TARGET_SSH_ALIAS="" while [ $# -gt 0 ]; do case "$1" in @@ -92,6 +97,8 @@ while [ $# -gt 0 ]; do --env-b-user) shift; ENV_B_USER="$1" ;; --bundle-out) shift; BUNDLE_OUT="$1" ;; --bundle-in) shift; BUNDLE_IN="$1" ;; + --source-ssh-alias) shift; SOURCE_SSH_ALIAS="$1" ;; + --target-ssh-alias) shift; TARGET_SSH_ALIAS="$1" ;; -h|--help) sed -n '2,55p' "$NC_SELF"; exit 0 ;; -*) die "unknown flag: $1" ;; *) die "extra arg: $1" ;; @@ -99,13 +106,36 @@ while [ $# -gt 0 ]; do shift done +# Resolve ssh-helper.sh for cross-env support. Caller can pre-export +# LARRY_LIB_DIR; otherwise we look next to ourselves. +SSH_HELPER="${LARRY_LIB_DIR:-$LIB_DIR}/ssh-helper.sh" +[ -x "$SSH_HELPER" ] || SSH_HELPER="$LIB_DIR/ssh-helper.sh" + +# Helper: run a command on a remote alias if non-empty, else locally. +# `$1`=alias (empty=local); rest=command (single string). +_run_on() { + local alias="$1"; shift + if [ -z "$alias" ]; then + bash -c "$*" + else + [ -x "$SSH_HELPER" ] || die "ssh-helper.sh not found at $SSH_HELPER (needed for --source/target-ssh-alias)" + "$SSH_HELPER" exec "$alias" "$*" + fi +} + [ -n "$OUT" ] || die "missing --out DIR" # When --bundle-in is given, we don't need scope/env-a/etc. — the bundle has them. if [ -z "$BUNDLE_IN" ]; then [ -n "$SCOPE" ] || die "missing --scope (thread:NAME | threads:N1,N2 | site | server)" [ -n "$ENV_A" ] || die "missing --env-a HCIROOT_A" [ -n "$ENV_B" ] || die "missing --env-b HCIROOT_B" - [ -d "$ENV_A" ] || die "env-a is not a directory: $ENV_A" + # ENV_A directory check is only meaningful when SOURCE is local; same for ENV_B. + if [ -z "$SOURCE_SSH_ALIAS" ]; then + [ -d "$ENV_A" ] || die "env-a is not a directory: $ENV_A (and --source-ssh-alias unset)" + fi +fi +if [ -n "$SOURCE_SSH_ALIAS" ] || [ -n "$TARGET_SSH_ALIAS" ]; then + [ -x "$SSH_HELPER" ] || die "ssh-helper.sh required for cross-env mode but not found at $SSH_HELPER" fi case "$PHASE" in 1|2|3|4|5|6|all|env-a|env-b) ;; *) die "bad --phase (use 1|2|3|4|5|6|all|env-a|env-b)" ;; esac [ "$DRY_RUN" = "1" ] || [ -n "$ROUTE_TEST_CMD" ] || say "WARNING: --route-test-cmd is unset; phases 3 and 4 will be skipped (you can run them manually using the generated input files)" @@ -137,14 +167,38 @@ discover_inbounds() { threads:*) echo "${SCOPE#threads:}" | tr ',' '\n' ;; site) [ -n "$SITE_A" ] || die "scope=site requires --site-a" - "$NCI" "$ENV_A/$SITE_A/NetConfig" --mode "$INBOUND_MODE" --format tsv \ - | awk -F'\t' 'NR>1 {print $1}' + if [ -n "$SOURCE_SSH_ALIAS" ]; then + # Pull the remote NetConfig locally first, then parse with our local NCI. + local remote_nc="$ENV_A/$SITE_A/NetConfig" + local local_nc; local_nc=$("$SSH_HELPER" pull "$SOURCE_SSH_ALIAS" "$remote_nc" 2>/dev/null | tail -1) + [ -n "$local_nc" ] && [ -f "$local_nc" ] || die "could not pull remote NetConfig $remote_nc from $SOURCE_SSH_ALIAS" + "$NCI" "$local_nc" --mode "$INBOUND_MODE" --format tsv \ + | awk -F'\t' 'NR>1 {print $1}' + else + "$NCI" "$ENV_A/$SITE_A/NetConfig" --mode "$INBOUND_MODE" --format tsv \ + | awk -F'\t' 'NR>1 {print $1}' + fi ;; server) - while IFS= read -r nc; do - "$NCI" "$nc" --mode "$INBOUND_MODE" --format tsv \ - | awk -F'\t' 'NR>1 {print $1}' - done < <(find "$ENV_A" -maxdepth 2 -name NetConfig -type f 2>/dev/null) + if [ -n "$SOURCE_SSH_ALIAS" ]; then + # find NetConfigs remotely, then pull each and parse locally. + local ncs + ncs=$("$SSH_HELPER" exec "$SOURCE_SSH_ALIAS" "find $ENV_A -maxdepth 2 -name NetConfig -type f 2>/dev/null" 2>/dev/null \ + | grep -v '^\[ssh_exec:' ) + local nc local_nc + while IFS= read -r nc; do + [ -z "$nc" ] && continue + local_nc=$("$SSH_HELPER" pull "$SOURCE_SSH_ALIAS" "$nc" 2>/dev/null | tail -1) + [ -n "$local_nc" ] && [ -f "$local_nc" ] || continue + "$NCI" "$local_nc" --mode "$INBOUND_MODE" --format tsv \ + | awk -F'\t' 'NR>1 {print $1}' + done <<< "$ncs" + else + while IFS= read -r nc; do + "$NCI" "$nc" --mode "$INBOUND_MODE" --format tsv \ + | awk -F'\t' 'NR>1 {print $1}' + done < <(find "$ENV_A" -maxdepth 2 -name NetConfig -type f 2>/dev/null) + fi ;; *) die "bad --scope: $SCOPE" ;; esac @@ -167,28 +221,74 @@ phase_1() { # Phase 2: sample N messages per inbound from env-A's smatdbs # ───────────────────────────────────────────────────────────────────────────── phase_2() { - say "=== PHASE 2: sample $COUNT messages per inbound from env-A ===" + say "=== PHASE 2: sample $COUNT messages per inbound from env-A${SOURCE_SSH_ALIAS:+ via ssh:$SOURCE_SSH_ALIAS} ===" [ -f "$OUT/inbounds.txt" ] || { say "no inbounds file from phase 1; running phase 1 first"; phase_1 || return 1; } local thread sitedir local count=0 while IFS= read -r thread; do [ -z "$thread" ] && continue - # Locate the site for this thread - sitedir="" - if [ -n "$SITE_A" ]; then - sitedir="$ENV_A/$SITE_A" - else - # Search across all sites under ENV_A for the smatdb - sitedir=$(find "$ENV_A" -maxdepth 4 -name "${thread}.smatdb" -type f 2>/dev/null | head -1 | xargs -I{} dirname {} | sed "s#/exec/processes/.*##") - [ -z "$sitedir" ] && { say " skip $thread: smatdb not found under $ENV_A"; continue; } - fi local input="$OUT/inputs/${thread}.msgs" - if [ "$DRY_RUN" = "1" ]; then - say " [dry-run] would sample $COUNT msgs from $thread → $input" + + if [ -n "$SOURCE_SSH_ALIAS" ]; then + # Remote sampling: use ssh_pull_smat in sampled mode if SITE_A is known. + # ssh_pull_smat needs a site name; if --site-a is unset we attempt a best- + # effort discovery via remote find for the .smatdb path. Note: ssh-helper's + # pull-smat requires the site arg, so we fall back to a remote find + + # full pull when SITE_A is unset. + if [ "$DRY_RUN" = "1" ]; then + say " [dry-run] would pull-smat $thread (sampled days=14) from $SOURCE_SSH_ALIAS → $input" + else + local site_for_smat="${SITE_A:-}" + if [ -z "$site_for_smat" ]; then + # Try to locate the .smatdb path remotely and infer the site. + local remote_smatdb + remote_smatdb=$("$SSH_HELPER" exec "$SOURCE_SSH_ALIAS" \ + "find $ENV_A -maxdepth 5 -name ${thread}.smatdb -type f 2>/dev/null | head -1" 2>/dev/null \ + | grep -v '^\[ssh_exec:' | head -1) + if [ -n "$remote_smatdb" ]; then + # site = first dir component under $ENV_A + site_for_smat=$(printf '%s' "$remote_smatdb" | sed -e "s#^${ENV_A}/##" -e 's#/.*##') + fi + fi + if [ -n "$site_for_smat" ]; then + # Pull recent (last 14d) messages as TSV+b64; decode locally into the + # input file separated by 0x1c (matching nc-msgs --format raw output). + local sample_tsv; sample_tsv=$(mktemp) + "$SSH_HELPER" pull-smat "$SOURCE_SSH_ALIAS" "$site_for_smat" "$thread" 14 > "$sample_tsv" 2>/dev/null + # decode b64 column 6, separate messages with 0x1c + : > "$input" + local got=0 + while IFS=$'\t' read -r unix_ts dir typ src dst b64; do + [ -z "$b64" ] && continue + printf '%s' "$b64" | base64 -d >> "$input" 2>/dev/null && { + printf '\x1c' >> "$input" + got=$((got+1)) + [ "$got" -ge "$COUNT" ] && break + } + done < "$sample_tsv" + rm -f "$sample_tsv" + say " sampled $thread (remote, site=$site_for_smat) → $input ($got messages)" + else + say " skip $thread: could not infer remote site (set --site-a)" + continue + fi + fi else - HCISITEDIR="$sitedir" "$NCM" "$thread" --limit "$COUNT" --format raw > "$input" 2>/dev/null - local got; got=$(tr -cd $'\x1c' < "$input" | wc -c | tr -d ' ') - say " sampled $thread → $input ($got messages)" + # Local sampling (original behaviour). + sitedir="" + if [ -n "$SITE_A" ]; then + sitedir="$ENV_A/$SITE_A" + else + sitedir=$(find "$ENV_A" -maxdepth 4 -name "${thread}.smatdb" -type f 2>/dev/null | head -1 | xargs -I{} dirname {} | sed "s#/exec/processes/.*##") + [ -z "$sitedir" ] && { say " skip $thread: smatdb not found under $ENV_A"; continue; } + fi + if [ "$DRY_RUN" = "1" ]; then + say " [dry-run] would sample $COUNT msgs from $thread → $input" + else + HCISITEDIR="$sitedir" "$NCM" "$thread" --limit "$COUNT" --format raw > "$input" 2>/dev/null + local got; got=$(tr -cd $'\x1c' < "$input" | wc -c | tr -d ' ') + say " sampled $thread → $input ($got messages)" + fi fi count=$((count+1)) done < "$OUT/inbounds.txt" @@ -210,8 +310,12 @@ render_cmd() { } phase_routes() { - local label="$1" hciroot="$2" hcisite="$3" - say "=== PHASE ${label}: route_test on env-${label} ===" + local label="$1" hciroot="$2" hcisite="$3" ssh_alias="${4:-}" + if [ -n "$ssh_alias" ]; then + say "=== PHASE ${label}: route_test on env-${label} via ssh:${ssh_alias} ===" + else + say "=== PHASE ${label}: route_test on env-${label} (local) ===" + fi if [ -z "$ROUTE_TEST_CMD" ]; then say "no --route-test-cmd; skipping phase ${label}" say "to run manually, use the input files at $OUT/inputs/*.msgs" @@ -224,23 +328,55 @@ phase_routes() { local outdir="$OUT/outputs/env-${label}/${thread}" [ -f "$input" ] || { say " skip $thread: no input file"; continue; } mkdir -p "$outdir" - local cmd; cmd=$(render_cmd "$ROUTE_TEST_CMD" "$thread" "$input" "$outdir" "$hciroot" "$hcisite") - if [ "$DRY_RUN" = "1" ]; then - say " [dry-run] $thread:" + + if [ -n "$ssh_alias" ]; then + # Cross-env: push the input to a deterministic remote staging path, + # render the route_test cmd with REMOTE paths, run via ssh_exec, then + # ssh_pull the output files back into the local outdir. + local remote_input="/tmp/larry-regress/inputs/${thread}.msgs" + local remote_outdir="/tmp/larry-regress/outputs/env-${label}/${thread}" + local cmd; cmd=$(render_cmd "$ROUTE_TEST_CMD" "$thread" "$remote_input" "$remote_outdir" "$hciroot" "$hcisite") + if [ "$DRY_RUN" = "1" ]; then + say " [dry-run] $thread (remote):" + say " ssh_push $input → $ssh_alias:$remote_input" + say " ssh_exec $ssh_alias: $cmd" + say " ssh_pull $ssh_alias:$remote_outdir/* → $outdir" + continue + fi + say " $thread (remote on $ssh_alias):" + "$SSH_HELPER" exec "$ssh_alias" "mkdir -p $remote_outdir $(dirname "$remote_input")" >/dev/null 2>&1 || true + "$SSH_HELPER" push "$ssh_alias" "$input" "$remote_input" >/dev/null 2>&1 || { say " push failed for $thread"; continue; } say " \$ $cmd" + "$SSH_HELPER" exec "$ssh_alias" "$cmd" 2>&1 | sed 's/^/ /' || say " (route_test exit non-zero — continuing)" + # Pull every file from remote_outdir back to local outdir. + local out_listing + out_listing=$("$SSH_HELPER" exec "$ssh_alias" "find $remote_outdir -maxdepth 1 -type f 2>/dev/null" 2>/dev/null | grep -v '^\[ssh_exec:') + local rf + while IFS= read -r rf; do + [ -z "$rf" ] && continue + "$SSH_HELPER" pull "$ssh_alias" "$rf" "$outdir/$(basename "$rf")" >/dev/null 2>&1 || say " pull failed: $rf" + done <<< "$out_listing" else - say " $thread:" - say " \$ $cmd" - bash -c "$cmd" 2>&1 | sed 's/^/ /' || say " (route_test exit non-zero — continuing)" + local cmd; cmd=$(render_cmd "$ROUTE_TEST_CMD" "$thread" "$input" "$outdir" "$hciroot" "$hcisite") + if [ "$DRY_RUN" = "1" ]; then + say " [dry-run] $thread:" + say " \$ $cmd" + else + say " $thread:" + say " \$ $cmd" + bash -c "$cmd" 2>&1 | sed 's/^/ /' || say " (route_test exit non-zero — continuing)" + fi fi done < "$OUT/inbounds.txt" } -phase_3() { phase_routes "a" "$ENV_A" "$SITE_A"; } +phase_3() { phase_routes "a" "$ENV_A" "$SITE_A" "$SOURCE_SSH_ALIAS"; } phase_4() { - # Phase 4: copy inputs to env-b host (if remote), then route_test on env-B. - if [ -n "$ENV_B_HOST" ]; then + # Phase 4: if target is remote via ssh_alias, phase_routes handles the push. + # Legacy --env-b-host path (raw scp) is preserved for environments where the + # ssh-helper master isn't open. + if [ -z "$TARGET_SSH_ALIAS" ] && [ -n "$ENV_B_HOST" ]; then say "copying input files to ${ENV_B_USER:-$USER}@${ENV_B_HOST}:${OUT}/inputs/" if [ "$DRY_RUN" = "1" ]; then say " [dry-run] scp -r $OUT/inputs/ ${ENV_B_USER:-$USER}@${ENV_B_HOST}:${OUT}/" @@ -249,7 +385,7 @@ phase_4() { scp -r "$OUT/inputs/" "${ENV_B_USER:-$USER}@${ENV_B_HOST}:${OUT}/" || say "scp failed; you'll need to copy manually" fi fi - phase_routes "b" "$ENV_B" "$SITE_B" + phase_routes "b" "$ENV_B" "$SITE_B" "$TARGET_SSH_ALIAS" } # ───────────────────────────────────────────────────────────────────────────── @@ -260,8 +396,13 @@ phase_5() { local diff_index="$OUT/diff/_index.md" { printf '# Regression diff index\n\n' - printf '- env-A: `%s`\n- env-B: `%s`\n- scope: `%s`\n- count: %s msgs per inbound\n- ignore: `%s`\n%s\n\n' \ - "$ENV_A" "$ENV_B" "$SCOPE" "$COUNT" "$IGNORE" \ + # NOTE: format string starts with '- ', so use printf '--' separator — + # otherwise bash 3.2's printf (macOS default) reads the leading '-' as a + # bad option and emits nothing. This was a latent bug pre-v0.6.8. + printf -- '- env-A: `%s`%s\n- env-B: `%s`%s\n- scope: `%s`\n- count: %s msgs per inbound\n- ignore: `%s`\n%s\n\n' \ + "$ENV_A" "$([ -n "$SOURCE_SSH_ALIAS" ] && printf ' (via ssh:%s)' "$SOURCE_SSH_ALIAS")" \ + "$ENV_B" "$([ -n "$TARGET_SSH_ALIAS" ] && printf ' (via ssh:%s)' "$TARGET_SSH_ALIAS")" \ + "$SCOPE" "$COUNT" "$IGNORE" \ "$([ -n "$INCLUDE" ] && printf -- '- include-only: `%s`' "$INCLUDE")" printf '| thread | dest | diffs | report |\n|---|---|---|---|\n' } > "$diff_index" @@ -320,12 +461,15 @@ phase_6() { printf '# Regression test summary\n\n' printf 'Generated: %s\n\n' "$(date -Iseconds 2>/dev/null || date)" printf '## Configuration\n\n' - printf '- scope: `%s`\n' "$SCOPE" - printf '- count: %s messages per inbound\n' "$COUNT" - printf '- env-A: `%s` (site=%s)\n' "$ENV_A" "${SITE_A:-auto}" - printf '- env-B: `%s` (site=%s)\n' "$ENV_B" "${SITE_B:-auto}" - printf '- ignore: `%s`\n' "$IGNORE" - [ -n "$INCLUDE" ] && printf '- include-only: `%s`\n' "$INCLUDE" + # printf '--' guard for leading '-' format strings (bash 3.2 / macOS). + printf -- '- scope: `%s`\n' "$SCOPE" + printf -- '- count: %s messages per inbound\n' "$COUNT" + printf -- '- env-A: `%s` (site=%s)%s\n' "$ENV_A" "${SITE_A:-auto}" \ + "$([ -n "$SOURCE_SSH_ALIAS" ] && printf ' via ssh:%s' "$SOURCE_SSH_ALIAS")" + printf -- '- env-B: `%s` (site=%s)%s\n' "$ENV_B" "${SITE_B:-auto}" \ + "$([ -n "$TARGET_SSH_ALIAS" ] && printf ' via ssh:%s' "$TARGET_SSH_ALIAS")" + printf -- '- ignore: `%s`\n' "$IGNORE" + [ -n "$INCLUDE" ] && printf -- '- include-only: `%s`\n' "$INCLUDE" printf '\n## Inbounds tested\n\n' [ -f "$OUT/inbounds.txt" ] && awk '{print "- `" $0 "`"}' "$OUT/inbounds.txt" printf '\n## Inputs\n\n' diff --git a/lib/ssh-helper.sh b/lib/ssh-helper.sh index d4e7ea2..a066f91 100755 --- a/lib/ssh-helper.sh +++ b/lib/ssh-helper.sh @@ -24,6 +24,11 @@ # close close ControlMaster # status [alias] show open masters / cred presence # exec run command via master (returns output) +# pull [local] scp remote → local via existing master +# push scp local → remote via existing master +# pull-smat [days_back] +# pull a thread's smatdb (full) or sample +# recent messages from it (sampled, TSV b64) # help print this help set -u @@ -239,6 +244,201 @@ cmd_exec() { ssh -S "$sock" -p "$port" -o BatchMode=yes "$addr" "$cmd" } +# ── v0.6.8: scp helpers that multiplex via the existing ControlMaster ──────── +# We use ssh's ControlPath/ControlMaster=no for scp (scp reads ssh-style options +# via -o), so the file transfer rides the open master and needs no second auth. +# Resolve ADDR/PORT/SOCK for an alias; die if master not open. Sets globals: +# _RH_ADDR _RH_PORT _RH_SOCK +_resolve_open_master() { + local alias="$1" + local addr_port; addr_port=$(read_host_addr "$alias") + [ -n "$addr_port" ] || die "no such alias: $alias" + _RH_ADDR=$(printf '%s' "$addr_port" | cut -f1) + _RH_PORT=$(printf '%s' "$addr_port" | cut -f2) + _RH_SOCK="$SSH_SOCKETS_DIR/$alias.sock" + if [ ! -S "$_RH_SOCK" ] || ! ssh -S "$_RH_SOCK" -O check -p "$_RH_PORT" "$_RH_ADDR" 2>/dev/null; then + die "no open master for $alias — open it with /ssh-setup $alias first" + fi +} + +# Deterministic local cache path for ssh_pull. +# /tmp/larry-pulls/.. +_pull_cache_path() { + local alias="$1" remote="$2" + local base; base=$(basename -- "$remote" 2>/dev/null) + [ -z "$base" ] && base="file" + # 8-char hex hash of full remote path. We try the most common hashers in + # turn; on a stripped box without any, fall back to a length+checksum proxy + # so the path is still deterministic per . + local hash="" + if command -v shasum >/dev/null 2>&1; then + hash=$(printf '%s' "$remote" | shasum -a 1 2>/dev/null | cut -c1-8) + elif command -v sha1sum >/dev/null 2>&1; then + hash=$(printf '%s' "$remote" | sha1sum 2>/dev/null | cut -c1-8) + elif command -v md5sum >/dev/null 2>&1; then + hash=$(printf '%s' "$remote" | md5sum 2>/dev/null | cut -c1-8) + else + hash=$(printf '%s' "$remote" | cksum 2>/dev/null | awk '{printf "%08x", $1}' | cut -c1-8) + fi + [ -z "$hash" ] && hash="00000000" + mkdir -p /tmp/larry-pulls 2>/dev/null + printf '/tmp/larry-pulls/%s.%s.%s' "$alias" "$base" "$hash" +} + +cmd_pull() { + local alias="${1:-}" remote="${2:-}" local_path="${3:-}" + [ -n "$alias" ] && [ -n "$remote" ] || die "usage: pull [local_path]" + _resolve_open_master "$alias" + [ -z "$local_path" ] && local_path=$(_pull_cache_path "$alias" "$remote") + mkdir -p "$(dirname "$local_path")" 2>/dev/null + + # Get remote file size up-front for a partial-transfer sanity check. + local remote_size="" + remote_size=$(ssh -S "$_RH_SOCK" -p "$_RH_PORT" -o BatchMode=yes "$_RH_ADDR" \ + "wc -c < $(printf '%q' "$remote") 2>/dev/null" 2>/dev/null | tr -d ' ') + if [ -z "$remote_size" ] || ! [[ "$remote_size" =~ ^[0-9]+$ ]]; then + die "remote file not found or not readable: $remote" + fi + + # scp via the existing master: -o ControlPath=... -o ControlMaster=no + local scp_err; scp_err=$(mktemp 2>/dev/null || echo "/tmp/larry-scp.err.$$") + if scp -q \ + -o "ControlPath=$_RH_SOCK" \ + -o "ControlMaster=no" \ + -o "BatchMode=yes" \ + -P "$_RH_PORT" \ + "$_RH_ADDR:$remote" "$local_path" 2>"$scp_err"; then + local got; got=$(wc -c < "$local_path" 2>/dev/null | tr -d ' ') + if [ "$got" != "$remote_size" ]; then + rm -f "$scp_err" + die "partial transfer: remote=$remote_size bytes, local=$got bytes ($local_path)" + fi + rm -f "$scp_err" + ok "pulled $alias:$remote → $local_path ($got bytes)" + # Print only the local path on the final line so callers (tool layer) can + # capture it deterministically with `tail -1` or similar. + printf '%s\n' "$local_path" + return 0 + fi + local rc=$? + printf 'ssh-helper: scp pull failed (rc=%d):\n' "$rc" >&2 + cat "$scp_err" >&2 2>/dev/null + rm -f "$scp_err" + return 1 +} + +cmd_push() { + local alias="${1:-}" local_path="${2:-}" remote="${3:-}" + [ -n "$alias" ] && [ -n "$local_path" ] && [ -n "$remote" ] \ + || die "usage: push " + [ -f "$local_path" ] || die "local file not found: $local_path" + _resolve_open_master "$alias" + + local local_size; local_size=$(wc -c < "$local_path" 2>/dev/null | tr -d ' ') + local scp_err; scp_err=$(mktemp 2>/dev/null || echo "/tmp/larry-scp.err.$$") + if scp -q \ + -o "ControlPath=$_RH_SOCK" \ + -o "ControlMaster=no" \ + -o "BatchMode=yes" \ + -P "$_RH_PORT" \ + "$local_path" "$_RH_ADDR:$remote" 2>"$scp_err"; then + # Validate via remote wc -c. + local got + got=$(ssh -S "$_RH_SOCK" -p "$_RH_PORT" -o BatchMode=yes "$_RH_ADDR" \ + "wc -c < $(printf '%q' "$remote") 2>/dev/null" 2>/dev/null | tr -d ' ') + if [ "$got" != "$local_size" ]; then + rm -f "$scp_err" + die "partial transfer: local=$local_size bytes, remote=$got bytes ($alias:$remote)" + fi + rm -f "$scp_err" + ok "pushed $local_path → $alias:$remote ($got bytes)" + return 0 + fi + local rc=$? + printf 'ssh-helper: scp push failed (rc=%d):\n' "$rc" >&2 + cat "$scp_err" >&2 2>/dev/null + rm -f "$scp_err" + return 1 +} + +# pull-smat: smart pull for a Cloverleaf thread's .smatdb file. +# Two modes: +# Full pull: pull-smat +# Locates $HCISITEDIR/exec/processes/*/.smatdb on the +# remote via find, then scp's the entire .smatdb file. +# Sampled: pull-smat +# Runs sqlite3 server-side, extracts up to 1000 most-recent +# messages from the last days, encodes each +# MessageContent BLOB as base64, returns TSV: +# unix_tsdirectiontypesourcedestmessage_blob_b64 +# The schema (table=smat_msgs, columns Time/Type/SourceConn/ +# DestConn/MessageContent) is the same one nc-msgs.sh uses. +cmd_pull_smat() { + local alias="${1:-}" site="${2:-}" thread="${3:-}" days_back="${4:-}" + [ -n "$alias" ] && [ -n "$site" ] && [ -n "$thread" ] \ + || die "usage: pull-smat [days_back]" + _resolve_open_master "$alias" + + # Discover the remote .smatdb path. We rely on HCIROOT being exported in + # the remote shell rc (typical Cloverleaf user profile), else SITEDIR is + # taken as / via ssh-resolved $HCIROOT. We do the find + # remotely to avoid hard-coding process directory names. + local find_cmd + find_cmd='set -e; SDIR="${HCISITEDIR:-${HCIROOT:-}/'"$site"'}"; ' + find_cmd+='[ -d "$SDIR" ] || { echo "ERROR: sitedir not found on remote: $SDIR" >&2; exit 2; }; ' + find_cmd+='F=$(find "$SDIR/exec/processes" -maxdepth 2 -type f -name "'"$thread"'.smatdb" 2>/dev/null | head -1); ' + find_cmd+='[ -n "$F" ] || F=$(find "$SDIR" -type f -name "'"$thread"'.smatdb" 2>/dev/null | head -1); ' + find_cmd+='[ -n "$F" ] || { echo "ERROR: no smatdb found for thread '"$thread"' under $SDIR" >&2; exit 3; }; ' + find_cmd+='printf "%s\n" "$F"' + + local remote_smatdb + remote_smatdb=$(ssh -S "$_RH_SOCK" -p "$_RH_PORT" -o BatchMode=yes "$_RH_ADDR" "$find_cmd" 2>&1 | tail -1) + case "$remote_smatdb" in + ERROR:*|'') die "remote smatdb lookup failed: $remote_smatdb" ;; + esac + + if [ -z "$days_back" ]; then + # Full mode: scp the whole .smatdb file. + local local_path + local_path=$(_pull_cache_path "$alias" "$remote_smatdb") + cmd_pull "$alias" "$remote_smatdb" "$local_path" + return $? + fi + + # Sampled mode: run sqlite3 on the remote, return TSV with b64-encoded blobs. + # base64 -w0 is GNU coreutils; on BSD use plain base64 (no -w). We accept + # whichever is present; the awk in the SQL pipeline strips internal newlines + # for sturdy TSV. + # + # Output line shape (each message): + # \t\t\t\t\t + # `direction` is "in" when DestConn=thread, else "out" (best-effort heuristic). + local sample_cmd + sample_cmd='set -e; ' + sample_cmd+='which sqlite3 >/dev/null 2>&1 || { echo "ERROR: sqlite3 not on remote PATH" >&2; exit 4; }; ' + sample_cmd+='B64() { if base64 --help 2>&1 | grep -q -- " -w"; then base64 -w0; else base64 | tr -d "\n"; fi; }; ' + # Note: sqlite3 ".mode tabs" prints rows tab-separated; we redirect blob via + # writefile() into temp files, then base64 each. That avoids any binary + # mangling in the sqlite3 -ascii path. Approach: select rowids, then for each + # rowid pull MessageContent into a per-row temp file, b64 it inline. + sample_cmd+='TMP=$(mktemp -d); trap "rm -rf $TMP" EXIT; ' + sample_cmd+='CUTOFF_MS=$(( ( $(date +%s) - '"$days_back"' * 86400 ) * 1000 )); ' + sample_cmd+='sqlite3 "'"$remote_smatdb"'" "SELECT rowid, Time, IFNULL(Type,\"\"), IFNULL(SourceConn,\"\"), IFNULL(DestConn,\"\") FROM smat_msgs WHERE Time >= $CUTOFF_MS ORDER BY Time DESC LIMIT 1000" ' + sample_cmd+='| while IFS="|" read -r rid tm typ src dst; do ' + sample_cmd+=' blobfile="$TMP/$rid.bin"; ' + sample_cmd+=' sqlite3 "'"$remote_smatdb"'" "SELECT writefile(\"$blobfile\", MessageContent) FROM smat_msgs WHERE rowid=$rid" >/dev/null 2>&1; ' + sample_cmd+=' if [ "$dst" = "'"$thread"'" ]; then dir="in"; else dir="out"; fi; ' + sample_cmd+=' printf "%s\t%s\t%s\t%s\t%s\t" "$(( tm / 1000 ))" "$dir" "$typ" "$src" "$dst"; ' + sample_cmd+=' B64 < "$blobfile"; ' + sample_cmd+=' printf "\n"; ' + sample_cmd+='done; ' + sample_cmd+='TOTAL=$(sqlite3 "'"$remote_smatdb"'" "SELECT COUNT(*) FROM smat_msgs WHERE Time >= $CUTOFF_MS"); ' + sample_cmd+='RETURNED=$(sqlite3 "'"$remote_smatdb"'" "SELECT MIN(1000, COUNT(*)) FROM smat_msgs WHERE Time >= $CUTOFF_MS"); ' + sample_cmd+='echo "# smatdb=$(basename '"$remote_smatdb"') days_back='"$days_back"' total_in_window=$TOTAL returned=$RETURNED truncated=$([ "$TOTAL" -gt 1000 ] && echo yes || echo no)" >&2' + + ssh -S "$_RH_SOCK" -p "$_RH_PORT" -o BatchMode=yes "$_RH_ADDR" "$sample_cmd" +} + case "${1:-help}" in hosts|list) shift; cmd_hosts ;; add) shift; cmd_add "$@" ;; @@ -248,6 +448,9 @@ case "${1:-help}" in close|exit) shift; cmd_close "$@" ;; status) shift; cmd_status "$@" ;; exec|run) shift; cmd_exec "$@" ;; + pull) shift; cmd_pull "$@" ;; + push) shift; cmd_push "$@" ;; + pull-smat) shift; cmd_pull_smat "$@" ;; -h|--help|help) cmd_help ;; *) die "unknown subcommand: ${1:-} (run with --help)" ;; esac