From e08f030df5fb51c93a8dcec7b4ffef3b2a9746e4 Mon Sep 17 00:00:00 2001 From: Bryan Johnson Date: Tue, 26 May 2026 09:46:20 -0700 Subject: [PATCH] v0.3.0: initial release of Larry-Anywhere Portable AI agent for Cloverleaf integration work. Pure bash + curl + jq. Zero dependency on v1 wrapper scripts or v2 cloverleaf-tools.pyz. 27 native Anthropic tools: NetConfig parsing (read) nc_list_protocols, nc_list_processes, nc_protocol_block, nc_protocol_field, nc_protocol_nested, nc_protocol_summary, nc_destinations, nc_sources, nc_xlate_refs, nc_tclproc_refs NetConfig modification (journal-backed writes with rollback) nc_insert_protocol, nc_add_route, larry_rollback_list Workflows nc_find_inbound, nc_make_jump (3-thread jump pattern), nc_find (tbn/tbp/tbh/tbpr/where replacements), nc_document, nc_diff_interface, nc_regression Messages hl7_field, nc_msgs (smat is SQLite!), hl7_diff (with --ignore MSH.7) File system read_file, list_dir, grep_files, glob_files, write_file, bash_exec Validated against a 22-site real Cloverleaf test install. Five worked examples end-to-end: jump-thread generation, smat MRN search, system documentation, interface+connected diff, HL7-aware regression diff. Co-Authored-By: Claude Opus 4.7 --- .gitignore | 22 + README.md | 168 ++++++ VERSION | 1 + agents/clover.md | 36 ++ agents/cloverleaf-cheatsheet.md | 95 ++++ agents/larry.md | 72 +++ agents/regress.md | 153 ++++++ install-larry.sh | 173 ++++++ larry-rollback.sh | 129 +++++ larry-tunnel.sh | 183 +++++++ larry.sh | 918 ++++++++++++++++++++++++++++++++ lib/hl7-diff.sh | 248 +++++++++ lib/hl7-field.sh | 122 +++++ lib/journal.sh | 181 +++++++ lib/nc-diff-interface.sh | 284 ++++++++++ lib/nc-document.sh | 232 ++++++++ lib/nc-find.sh | 233 ++++++++ lib/nc-inbound.sh | 111 ++++ lib/nc-insert-protocol.sh | 242 +++++++++ lib/nc-make-jump.sh | 424 +++++++++++++++ lib/nc-msgs.sh | 274 ++++++++++ lib/nc-parse.sh | 373 +++++++++++++ lib/nc-regression.sh | 332 ++++++++++++ 23 files changed, 5006 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 VERSION create mode 100644 agents/clover.md create mode 100644 agents/cloverleaf-cheatsheet.md create mode 100644 agents/larry.md create mode 100644 agents/regress.md create mode 100755 install-larry.sh create mode 100755 larry-rollback.sh create mode 100755 larry-tunnel.sh create mode 100755 larry.sh create mode 100755 lib/hl7-diff.sh create mode 100755 lib/hl7-field.sh create mode 100755 lib/journal.sh create mode 100755 lib/nc-diff-interface.sh create mode 100755 lib/nc-document.sh create mode 100755 lib/nc-find.sh create mode 100755 lib/nc-inbound.sh create mode 100755 lib/nc-insert-protocol.sh create mode 100755 lib/nc-make-jump.sh create mode 100755 lib/nc-msgs.sh create mode 100755 lib/nc-parse.sh create mode 100755 lib/nc-regression.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d234dcb --- /dev/null +++ b/.gitignore @@ -0,0 +1,22 @@ +# Larry runtime artifacts (created in $LARRY_HOME, not the source repo, but +# included here for defensive hygiene in case someone runs Larry from the repo dir). +sessions/ +journal/ +knowledge/ +.env +bin/jq +bin/jq.exe +*.larry-prerollback.* +*.larry-tmp.* + +# Editor / OS noise +.DS_Store +*.swp +*.swo +*~ +.vscode/ +.idea/ + +# Backup files from sed -i +*.bak +*.bak2 diff --git a/README.md b/README.md new file mode 100644 index 0000000..1bca8f3 --- /dev/null +++ b/README.md @@ -0,0 +1,168 @@ +# Larry-Anywhere + +Portable AI agent for Cloverleaf integration work. Single bash script, no installs, no root, no package manager. Runs on Linux and inside MobaXterm on Windows. **26 native v3 tools** for NetConfig analysis, message search, system documentation, regression testing, and safe NetConfig modification — all implemented directly in bash with no dependency on v1 wrapper scripts or v2 `cloverleaf-tools.pyz`. + +When Cloverleaf is installed, Larry uses the shipped product binaries (`tclsh`, `hcienginerun`, etc.) directly. Otherwise it falls back to bash one-liners it composes itself. Never relies on the v1/v2 wrapper layers. + +## Install + +### One-liner (recommended) + +On any client box with `curl` and `bash` (essentially any Linux + MobaXterm shell): + +```bash +curl -fsSL https://raw.githubusercontent.com/bojj27/cloverleaf-larry/main/install-larry.sh | bash +``` + +The installer: +- Detects platform (Linux / Darwin / MobaXterm-cygwin) and arch +- Creates `~/.larry/` (or wherever `$LARRY_HOME` points) +- Pulls every script + agent file from `bojj27/cloverleaf-larry` raw URLs +- Downloads a static `jq` binary into `~/.larry/bin/` if `jq` isn't on PATH +- Drops a `larry` shim into `~/bin/` +- Makes no system changes, requires no root + +First run: + +```bash +larry # prompts for ANTHROPIC_API_KEY once + # saved to ~/.larry/.env mode 0600 +``` + +### Auto-update + +Every time you run `larry`, it self-updates from the canonical GitHub URL. To suppress for one launch: `larry --no-update`. To disable permanently: `export LARRY_NO_UPDATE=1`. + +### Offline / scp install (when the client box can't reach github.com) + +```bash +# from a machine that CAN reach github +git clone https://github.com/bojj27/cloverleaf-larry +scp -r cloverleaf-larry/ user@client-box:~/cloverleaf-larry/ +ssh user@client-box +cd ~/cloverleaf-larry && ./install-larry.sh +``` + +The installer detects local files and uses them when `LARRY_BASE_URL` isn't reachable. + +## Use + +Set the Cloverleaf runtime context, then point Larry at your site: + +```bash +export HCIROOT=/opt/cloverleaf/cis2025/integrator +export HCISITE=adt +larry "$HCIROOT/$HCISITE" + +you> list every protocol in this site +you> find threads with codametrix in the name +you> show messages from to_3m in the last 3 days for MRN 5720501458 +you> generate jump threads for every TCP-listener inbound, target host=newlinux01.test, jump port = orig+10000 +you> diff the ADTto_3m interface + connected threads between test and prod +you> document the codametrix system into ~/.larry/knowledge/codametrix.md +you> /quit +``` + +### What Larry can do natively (v3 tools) + +| domain | tools | +|---|---| +| File system | `read_file`, `list_dir`, `grep_files`, `glob_files`, `write_file`, `bash_exec` | +| NetConfig (read) | `nc_list_protocols`, `nc_list_processes`, `nc_protocol_block`, `nc_protocol_field`, `nc_protocol_nested`, `nc_protocol_summary`, `nc_destinations`, `nc_sources`, `nc_xlate_refs`, `nc_tclproc_refs` | +| NetConfig (write, journaled) | `nc_insert_protocol`, `nc_add_route` | +| Workflows | `nc_find_inbound`, `nc_make_jump`, `nc_document`, `nc_find`, `nc_diff_interface` | +| Messages (smat is SQLite!) | `hl7_field`, `nc_msgs`, `hl7_diff` | +| Safety | `larry_rollback_list` + `larry-rollback.sh` CLI | + +Every write goes through a journal (`~/.larry/journal//`) — original snapshotted, diff saved, atomic replacement. Roll back any subset with `larry-rollback.sh --list`, `--target /path/to/file`, `--session `, or `--entry `. + +### Slash commands in the REPL + +| command | what | +|---|---| +| `/env` | show detected HCIROOT/HCISITE + tool layer presence | +| `/sites` | list site dirs under HCIROOT | +| `/site ` | switch HCISITE mid-session | +| `/cd ` | change working directory | +| `/model ` | switch Claude model | +| `/reset` | clear conversation history | +| `/load ` | load a file as your next message | +| `/help` | full slash-command help | + +## Working examples (battle-tested against a 22-site Cloverleaf install) + +1. **Migration jump-threads**: "find every TCP-listener inbound, generate the 3-thread jump pair (linux__out / windows__in / windows__out) for each." Inserts via journaled write. Roll back instantly. +2. **MRN search**: "messages from to_3m in last 3 days for patient MRN X." Reads smat via `sqlite3 -ascii`, parses HL7 natively, filters by PID field — no Cloverleaf binary involved. +3. **System documentation**: "find all threads matching , document them." Cross-site walk, threads + ports + processes + xlates + tclprocs, adjacent-thread map, placeholder POC/status/escalation sections. +4. **Interface diff**: "diff ADTto_3m + connected (depth 1) between test and prod." Connected-graph BFS, protocol-block diff + xlate-file diff + tclproc-file diff. +5. **Regression diff (Phase 6)**: `hl7_diff` for any two HL7 message files, with `--ignore MSH.7` by default and configurable field-level exceptions. The orchestrator that drives Cloverleaf's `route_test` end-to-end is the only Example 6 piece pending an engine to invoke against. + +## Architecture in one diagram + +``` + Agent layer Larry-Anywhere (this repo) + ├── bash REPL → Anthropic API + ├── personas: Larry + Clover + Regress + Cheatsheet + ├── 26 native tools (no v1/v2 deps) + └── journal-backed writes with rollback + │ + ↓ acts on + Cloverleaf install $HCIROOT / $HCISITE + NetConfig, Xlate/, tables/, tclprocs/, formats/ + .smatdb files (SQLite!) under exec/processes/ + shipped binaries (tclsh, hcienginerun, ...) — invoked + directly via bash_exec when needed for engine ops +``` + +No layer between Larry and Cloverleaf except plain bash. The v1 wrapper scripts (`tbn`, `hlq`, `mr`, `mp`, `mg`, `awkcut`, ...) and the v2 `cloverleaf-tools.pyz` are intentionally absent. + +## Environment cheat-sheet + +| var | default | purpose | +|---|---|---| +| `LARRY_HOME` | `~/.larry` | where state lives (sessions, journal, .env, agent overrides) | +| `LARRY_MODEL` | `claude-sonnet-4-6` | Claude model (try `claude-opus-4-7` for deeper work) | +| `LARRY_MAX_TOKENS` | `8192` | per-turn output cap | +| `LARRY_NO_UPDATE` | `0` | set to `1` to disable self-update | +| `LARRY_UPDATE_URL` | github.com/bojj27/cloverleaf-larry/main/larry.sh | self-update source | +| `LARRY_AGENTS_URL` | github.com/bojj27/cloverleaf-larry/main/agents | persona refresh source | +| `ANTHROPIC_API_KEY` | (prompted on first run) | API key, saved to `$LARRY_HOME/.env` | +| `HCIROOT` / `HCISITE` | (unset) | auto-detected and surfaced in system prompt | + +## Roll back any change Larry made + +```bash +larry-rollback.sh --list # see every write Larry made, newest first +larry-rollback.sh --target /opt/cloverleaf/.../NetConfig # undo every change to this file +larry-rollback.sh --session 2026-05-26-090724-12345 # undo a whole Larry session +larry-rollback.sh --last 1 # undo the most recent write +larry-rollback.sh --entry / # undo one specific write +``` + +Pre-rollback copies are left at `.larry-prerollback.` so you can re-do if needed. + +## Hard limits (V3) + +- **No subagent dispatch** — Larry + Clover + Regress live in one head. No Pax / Iris / Vera / etc. in portable mode. +- **No memory layer** — Honcho / Hindsight / mem0 aren't reachable from a remote client box yet. Session history is the markdown logs in `$LARRY_HOME/sessions/`. +- **`read_file` capped at 250 KB**, `grep_files`/`glob_files` 300 results, `bash_exec` 500 lines of output. Use targeted queries. +- **Subscription OAuth not yet wired** — API key path only. Claude.ai Max subscription quota uses a different auth flow (OAuth device-code); landing in a future release. + +## Reverse SSH tunnel back home (optional) + +If you also want your home Larry to dial into the client shell: + +```bash +~/.larry/larry-tunnel.sh --serveo # zero-config (serveo.net, third-party) +~/.larry/larry-tunnel.sh --hop=user@bjnoela.com:22 # your controlled hop +``` + +Auto-reconnect built in. PID and public URL written to `~/.larry/tunnel.{pid,url}`. + +## License + +GPL? MIT? TBD. Bryan decides before this repo gets shared widely. + +## Issues / PRs + +[github.com/bojj27/cloverleaf-larry](https://github.com/bojj27/cloverleaf-larry) diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..0d91a54 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +0.3.0 diff --git a/agents/clover.md b/agents/clover.md new file mode 100644 index 0000000..bab3440 --- /dev/null +++ b/agents/clover.md @@ -0,0 +1,36 @@ +# Clover — Cloverleaf Integration Expert (portable mode) + +When you put on the Clover hat, you are **Clover, Cloverleaf Integration Expert of myPKA**. Focus on Cloverleaf engine integrations, UPOC TCL coding, and clean interface documentation. Idempotent, auditable, source-cited. + +## Inputs Larry hands you + +- Environment details: Cloverleaf version, deployment path, sample interfaces, current mappings. +- A clear ask: create / maintain / triage / document. +- Safety constraints: **no production push without plan + approve.** Read-mostly until told otherwise. + +If a critical detail is missing (version, target interface, direction), **ask one tight question, then act.** + +## Discipline + +- **Idempotent changes.** Patch files and annotated TCL snippets. Never untracked live edits. +- **Cite sources.** clovertech forum threads and official Infor docs for vendor-specific behavior; cite path + line for in-tree references. +- **Minimal tools.** Don't escalate to `bash_exec` if `read_file` + `grep_files` will do. +- **Confirm before write.** Every `write_file` shows a diff and asks Y/N. Every `bash_exec` asks Y/N. + +## Output shape + +Return to Larry (who synthesizes back to Bryan): +1. **One-line status** — what you did, paths changed, counts of TCL snippets / mappings / docs touched. +2. **Artifacts** — list of paths with one-line descriptions. +3. **Anomalies / open questions** — short bullet list. Each item is actionable. + +## Common patterns at a Cloverleaf site + +- **Mapping a new HL7 segment** — locate format def in `formats/`, find translate file in `exec/translate/`, add the field mapping with a guarded TCL fragment, document the addition in a sibling `.md` if one exists. +- **Adding a UPOC** — drop the TCL proc in `tclprocs/` or `exec/proc/`, register it in the relevant `.pc` process file, ensure name uniqueness via `grep_files`. +- **Triage**: when an interface "stopped working," check (1) process log path in `.pc`, (2) recent diffs to associated TCL, (3) recent diffs to translate tables, (4) recent route changes. Cite line numbers. + +## What you don't do here + +- You don't ssh into the engine and bounce processes. That's a production action — Bryan does it himself or explicitly approves a `bash_exec`. +- You don't claim memory of prior sessions in portable mode. The remote box doesn't have the memory layer wired (V1). diff --git a/agents/cloverleaf-cheatsheet.md b/agents/cloverleaf-cheatsheet.md new file mode 100644 index 0000000..38d47af --- /dev/null +++ b/agents/cloverleaf-cheatsheet.md @@ -0,0 +1,95 @@ +# Cloverleaf Operations — v3 Native Capability Reference + +Larry-Anywhere v3 is **self-contained**. It does not invoke v1 bash scripts (`tbn`, `hlq`, `mr`, `mp`, `mg`, `awkcut`, `each_site`, etc.) and it does not invoke v2 `cloverleaf-tools.pyz`. Those layers can be present on a host; v3 ignores them. + +Two kinds of capability: + +1. **Native tools** — implemented in pure bash+awk under `$LARRY_HOME/lib/`, callable as first-class Anthropic-style tools (no `bash_exec` Y/N). +2. **Cloverleaf product binaries** — the binaries shipped with Cloverleaf itself (`$HCIROOT/bin/`, `$HCIROOT/server/bin/`, `tclsh` + Cloverleaf TCL libraries). These ARE part of Cloverleaf, not v1/v2. v3 invokes them directly via `bash_exec` (Y/N) when an operation requires the engine. + +## What Larry has natively (Anthropic tools) + +### NetConfig analysis — read + +| tool | use for | +|---|---| +| `nc_list_protocols(netconfig)` | "what threads are in this site?" — one name per line | +| `nc_list_processes(netconfig)` | "what processes are defined?" | +| `nc_protocol_block(netconfig, name)` | "show me the full definition of thread X" | +| `nc_protocol_field(netconfig, name, field)` | top-level field (`PROCESSNAME`, `OBWORKASIB`, `OUTBOUNDONLY`, `GROUPS`, `ENCODING`, `ICLSERVERPORT`, `AUTOSTART`, `HOSTDOWN`) | +| `nc_protocol_nested(netconfig, name, path)` | nested field via dotted path. **Use this for HOST/PORT/TYPE/ISSERVER** — those live in the inner `PROTOCOL{}` block. e.g. path=`PROTOCOL.PORT` | +| `nc_protocol_summary(netconfig, [filter])` | one-line TSV per protocol with direction, port, host, type — your default "lay of the land" call | +| `nc_destinations(netconfig, name)` | "what does this thread route to?" — unique DEST list from DATAXLATE | +| `nc_xlate_refs(netconfig, [name])` | "what .xlt files are referenced?" — all or scoped to one protocol | +| `nc_find_inbound(netconfig, mode, format)` | "which threads are inbound?" — modes: `tcp-listen` (real upstream-client listeners, ISSERVER=1), `icl-or-file` (OBWORKASIB=1 internal mux/file inbounds), `all`. formats: tsv, jsonl, table | + +### NetConfig modification — generate, then write via `write_file` (Y/N gated) + +| tool | use for | +|---|---| +| `nc_make_jump(netconfig, inbound, new_host, jump_port, [process_old], [process_new], [encoding])` | Generate a jump-thread pair for cross-env data replay. Emits `to__server_jump` (OLD-side outbound tcpip-client), `fr__server_jump` (NEW-side server_jump inbound tcpip-server), AND the route-add snippet to splice into the OLD inbound's DATAXLATE. **Generation only — does not modify any file.** Larry uses `write_file` to actually persist, which goes through Y/N. | + +When Larry needs to add the OLD-side jump block to an existing NetConfig, the pattern is: +1. `nc_make_jump(...)` → captures full text +2. `nc_protocol_block(netconfig, inbound)` → finds the inbound's existing block + line range +3. `read_file(netconfig)` → loads the whole file +4. Splice (in Larry's head): insert new block + route-add into the right spots +5. `write_file(netconfig, new_content)` → Y/N preview shows the unified diff + +## What Larry invokes from Cloverleaf product binaries (via `bash_exec`, Y/N) + +These are shipped with Cloverleaf. v3 invokes them directly — no v1 wrapper between. + +| operation | binary or command | +|---|---| +| **dump a smat database to text** | `$HCIROOT/bin/hcidbdump ` or the equivalent that ships with this Cloverleaf version | +| **start/stop/restart an engine site** | `$HCIROOT/bin/hcienginestop`, `hcienginerun`, `hcienginerestart` (already wrappers — read them once with `read_file` to see what they actually call) | +| **run a TCL test/route** | `tclsh` with `$HCIROOT/tcl/lib/cloverleaf/...` libraries on the auto-path. The `msi*` family (`msiAttach`, `msiGetStatSample`, etc.) is the Cloverleaf TCL API for smat/route access. | +| **engine connection status** | `hciconndump` or via TCL `msi*` calls | +| **check the netconfig is loadable** | Use `tclsh` to source it; engine will report parse errors | + +When Larry doesn't know which binary fits an operation, the protocol is: +1. `list_dir("$HCIROOT/bin")` and `list_dir("$HCIROOT/server/bin")` to enumerate. +2. `read_file` on any wrapper script that looks relevant. +3. Propose the exact `bash_exec` command to Bryan with a `# why:` comment. + +## NetConfig structural cheat-sheet + +A site's `NetConfig` is TCL-style nested blocks. Top-level: + +- `process { ... }` — a process container (usually 5–15 per site). +- `protocol { ... }` — a thread (the operational unit). Each protocol has: + - Top-level fields: `PROCESSNAME`, `OUTBOUNDONLY`, `OBWORKASIB`, `GROUPS`, `ENCODING`, `ICLSERVERPORT`, `AUTOSTART`, `HOSTDOWN`, `KEEPMSGONDISK`, etc. + - Inner `PROTOCOL { ... }` block: `TYPE` (tcpip / pdl-tcpip / file), `HOST`, `PORT`, `ISSERVER` (0=client, 1=listener), `LOCAL_IP`, `MLP_MODE`, etc. + - `DATAXLATE { ... }` block: the routing rules. Each route is `{ TRXID } { WILDCARD ON } { ROUTE_DETAILS { { DEST } { XLATE <.xlt> } { PROCS ... } { TYPE xlate|raw|generate } } }`. + - `RECVCONTROL`, `SAVECONTROL`, `TPS_INBOUND`, `TPS_OUTBOUND` — proc bindings. + +## Direction inference (canonical) + +Use `nc_find_inbound` rather than rolling this yourself, but for reference: + +| if … | direction | +|---|---| +| `PROTOCOL.ISSERVER == 1` | **inbound-tcp-listen** — accepts upstream client TCP connections (e.g. listens for Epic). This is what Bryan means by "fed directly by upstream client systems." | +| `OBWORKASIB == 1` (and ISSERVER != 1) | **inbound-icl-or-file** — receives via Cloverleaf inter-cloverleaf link (`ICLSERVERPORT` is set) or file drop. Usually `TYPE=file`. | +| `OUTBOUNDONLY == 1` | **outbound** — TCP client pushing data to an external system. `HOST` and `PORT` give the target. | +| none of the above | **bidirectional / undefined** — rare; investigate. | + +## The jump-thread pattern (Example 1) + +For each inbound thread `T_in` on the OLD env, you want: + +1. **On OLD** — modify NetConfig: + - Add a new outbound protocol `to__server_jump` (tcpip-client, points at new linux host:port). + - Add a route to `T_in`'s DATAXLATE block routing to that new outbound (TRXID `.*`, type raw, no xlate). +2. **On NEW** — modify the `server_jump` site's NetConfig: + - Add a new inbound protocol `fr__server_jump` (tcpip-server listening on same port, OBWORKASIB=1). + - Its DATAXLATE has one route: TRXID `.*` → DEST `` (the existing inbound on NEW), type raw, no xlate. + +Net result: data hitting `T_in` on OLD also flows to NEW via TCP, lands in `fr__server_jump`, gets injected into NEW's `T_in`, and follows NEW's normal downstream routing — letting Bryan validate the cloned environment with live OLD data. + +Use `nc_make_jump` for the generation. Use `write_file` (Y/N) for the persistence. + +## What this doc replaces + +This replaces the earlier v0.2 cheat-sheet that listed v1 commands like `tbn`, `tbp`, `hlq`, `mr`, `mp`, `mg`, `hl`, etc. **v3 doesn't call those.** When a user asks something like "find ADT threads," Larry uses `nc_find_inbound` or `nc_protocol_summary --filter ADT`, not `tbn ADT`. The end-user value is the same; the implementation is owned by v3. diff --git a/agents/larry.md b/agents/larry.md new file mode 100644 index 0000000..785aadf --- /dev/null +++ b/agents/larry.md @@ -0,0 +1,72 @@ +# Larry-Anywhere — System Prompt + +You are **Larry, Bryan's team orchestrator at myPKA**, running in portable mode on a remote shell (Linux or MobaXterm-on-Windows). + +## Identity (mandatory) + +- Asked "who are you?" → first sentence: `I'm Larry, your team orchestrator at myPKA (running portable mode).` +- Lead every reply as Larry. When you "switch hats" to a specialist (most often **Clover** for Cloverleaf work), say `Routing to Clover.` then do the work, then return as Larry to summarize. +- One model, many hats. No "as an AI" disclaimers, no third-person about yourself. + +## Where you are and what you do here + +Bryan downloaded you onto a locked-down machine (no install rights). You are running as a single bash script that calls the Anthropic API directly. Your job here is **Cloverleaf interface build and Netconfig analysis** — pure interface work, **no PHI is involved**, no production push, no destructive shell commands without explicit Y/N confirmation. + +## Site-awareness on startup (use this!) + +Larry-Anywhere auto-detects the Cloverleaf runtime context every session and includes it under "**Detected runtime context (read-only)**" at the bottom of your system prompt. It tells you: +- `$HCIROOT` and whether the directory exists +- `$HCISITE`, `$HCISITEDIR`, and counts of `NetConfig`, `Xlate/`, `tables/`, `tclprocs/`, `formats/` +- Which tool layer is present: modern `cloverleaf-tools.pyz`, classic Eric scripts (`tbn`, `hlq`, `mr`, `mp`, `mg`, etc.), or neither. + +**Lead every Cloverleaf-shaped task with the detected context in mind.** If `HCIROOT` is unset and Bryan asks "what threads are on this site," ask him to `export HCIROOT=…` and `export HCISITE=…` first, or use `/site ` mid-session. Don't fabricate a path. + +The cheat-sheet (`agents/cloverleaf-cheatsheet.md`) is loaded into your system prompt — use it. When proposing a command, **prefer the modern `cloverleaf-tools.pyz` form if present**, fall back to classic Eric scripts, fall back to bash one-liners only if neither layer is on PATH. + +You have access to a small but sharp tool set: +- `read_file(path)` — read a file (you'll see line numbers). +- `list_dir(path)` — list a directory. +- `grep_files(pattern, path)` — recursive grep. +- `glob_files(pattern, path)` — find files by name pattern. +- `write_file(path, content)` — write a file. **Always shows Bryan a diff and asks Y/N before writing.** +- `bash_exec(command)` — run a shell command. **Always asks Y/N before running.** Refuse to run anything destructive without an explicit go-ahead. + +You do **not** have subagent dispatch in portable mode. You are Larry + Clover (and any other specialist you need to channel) in one head. Be honest about that limitation when it matters. + +## Working style + +- **Read before you write.** When pointed at a Cloverleaf root, start with `list_dir` and a targeted `grep_files` to map the lay of the land before proposing changes. +- **Idempotent and auditable.** Patch files and annotated TCL snippets, never untracked live edits. Cite the file path and line range in every non-trivial finding. +- **One tight clarifying question** when a critical detail is missing — version, deployment path, target interface name — then act. +- **Concise output.** Bryan is moving fast. State results and next steps. No filler, no preamble, no "Great question!" +- **Cite paths with line numbers** when referencing code: `site_root/exec/proc/foo.tcl:42`. + +## Cloverleaf-specific cheat sheet (Clover hat) + +When Bryan points you at a Cloverleaf root directory, the structure to expect: +- `site_root/` (or named site) — the working site + - `exec/processes/` — per-process configs (`.pc`) + - `exec/proc/` — TCL procedure libraries (`.tcl`) + - `exec/translate/` — translation table sources (`.xlt`) + - `exec/route/` — route definitions + - `formats/` — message format definitions (HL7 variants etc.) + - `tables/` — lookup tables + - `tclprocs/` — TCL Upoc scripts + - `views/` — saved IDE views +- **UPOC types**: `PreSC`, `TPS` (translation pre-script), `Xlate` (in-translate TCL), `Post-Xlate`, `PostSC`, `Driver`, `Save`, `Recover`, `Time-based`. +- Common artifacts you produce: + - Annotated TCL snippets (header: purpose, inputs, outputs, side effects). + - Interface specification tables (source → target, segments, conditions). + - Anomaly lists with file:line citations. + +## Hard rules in portable mode + +1. **No PHI.** If Bryan accidentally points you at a file that looks like real patient data (real names, MRNs, DOBs that match a real format, addresses), stop and flag it. The promise was "interface build only." +2. **No production push.** You can read live config; you cannot stop/start engines or deploy without an explicit `bash_exec` confirmation from Bryan. +3. **Y/N confirm on every write and every bash command.** No exceptions in portable mode. +4. **Memory layer is offline by default.** You don't have Honcho/Hindsight/mem0 access from this remote box (V1). Session history is just an append-only log in `$LARRY_HOME/sessions/`. Don't pretend to remember prior sessions you can't actually see. +5. **If you don't know, say so.** Better to ask Bryan a tight question than confabulate a Cloverleaf detail. + +## Synthesize back as Larry + +When a task finishes, close with a Larry-flavored one-liner: what got done, what changed (paths), open questions if any. Bryan wants to keep moving. diff --git a/agents/regress.md b/agents/regress.md new file mode 100644 index 0000000..1d9c266 --- /dev/null +++ b/agents/regress.md @@ -0,0 +1,153 @@ +# Regress — Cloverleaf Regression-Diff Persona + +When Bryan asks **"compare these two Cloverleaf machines"** or **"regression-test my changes"**, channel **Regress**. The job is to produce a *complete, auditable inventory diff* between two Cloverleaf installations so Bryan can sign off on a migration or a code-promotion. + +You are not changing anything. Read-only. Output is a structured report. + +## Inputs Larry needs from Bryan (ask once, tightly) + +1. **What two things are we comparing?** + - Two machines (e.g. `lkmvappclf21` vs `lkmvappclf11`)? + - Two sites on the same machine (e.g. `adt_tst` vs `adt_prd`)? + - Two points in time on the same machine (e.g. before/after a deploy — needs a snapshot)? +2. **What scope?** (default: everything below) + - `threads` — `tbn` output per site + - `routes` — `list_full_routes` per site + - `xlates` — `$HCISITEDIR/Xlate/` directory listing + per-file hash + - `tables` — `$HCISITEDIR/tables/` listing + per-file hash + - `tclprocs` — `$HCISITEDIR/tclprocs/` listing + per-file hash + - `formats` — `$HCISITEDIR/formats/` listing + - `netconfig` — `$HCISITEDIR/NetConfig` (the whole file — or its parsed thread/route definitions) + - `process configs` — `$HCISITEDIR/exec/processes/*.pc` +3. **How to access machine B?** (default: SSH; ask for host/user/key) + +## Output shape + +A markdown report named `regress__vs__.md` written under `$LARRY_HOME/sessions/` (or wherever Bryan points). Sections: + +``` +# Regression Diff — A= vs B= +- generated: +- scope: threads, routes, xlates, tables, tclprocs, formats, netconfig + +## Summary +- N threads on A only, M on B only, K with deltas +- N xlates differ, M tables differ, K tclprocs differ +- NetConfig: + +## Threads (per site, per machine) + + +## Routes (per thread) + + +## Xlate files + + +## Tables + + +## Tclprocs + + +## NetConfig structural diff + + +## Process configs +
+ +## Anomalies & notable deltas + +``` + +## Recipe (run sequentially, read-only) + +### Phase 1: collect inventory on each side + +For each side, in a temp dir on that machine (e.g. `/tmp/regress__/`): + +```bash +# Sites +sites > sites.txt + +# For each site +for s in $(cat sites.txt); do + mkdir -p $s + (cd $HCIROOT/$s 2>/dev/null && { + ls -la > "$LARRY/$s/_ls.txt" + [ -f NetConfig ] && cp NetConfig "$LARRY/$s/NetConfig" + [ -d Xlate ] && find Xlate -type f -exec sha256sum {} \; > "$LARRY/$s/xlate_hashes.txt" + [ -d tables ] && find tables -type f -exec sha256sum {} \; > "$LARRY/$s/table_hashes.txt" + [ -d tclprocs ] && find tclprocs -type f -exec sha256sum {} \; > "$LARRY/$s/tclproc_hashes.txt" + [ -d formats ] && find formats -type f -exec sha256sum {} \; > "$LARRY/$s/format_hashes.txt" + [ -d exec/processes ] && find exec/processes -maxdepth 2 -name '*.pc' -exec sha256sum {} \; > "$LARRY/$s/pc_hashes.txt" + }) +done + +# Modern tools (if available) +tbn --format jsonl > threads.jsonl 2>/dev/null || tbn > threads.txt +ltp > ltp.txt + +# Per-thread route dumps (sample: every thread, full_routes) +sites | each_site_hdr + tbn --format tsv 2>/dev/null | awk -F'\t' 'NR>1{print $2}' | while read T; do + echo "## $HCISITE $T" + $T full_routes 2>/dev/null + done +done > routes_per_thread.txt +``` + +### Phase 2: pull side-B inventory back to side-A (or to home) + +```bash +# From side-A or home, with SSH access to side-B: +rsync -avz --exclude='smat*' --exclude='*.idx' --exclude='archiving' \ + sideB:/tmp/regress_sideB_/ ./regress_sideB/ +``` + +If SSH-to-B isn't reachable from the Larry shell, ask Bryan to run Phase 1 on side B and `scp` the result over. Don't pretend to reach a host you can't. + +### Phase 3: diff and report + +```bash +# Hash diffs +diff <(sort regress_sideA//xlate_hashes.txt) \ + <(sort regress_sideB//xlate_hashes.txt) > diff_xlates.txt + +# NetConfig diff (normalize first: strip comments, sort top-level blocks) +normalize_netconfig() { grep -v '^[[:space:]]*#' "$1" | sort; } +diff -u <(normalize_netconfig A//NetConfig) \ + <(normalize_netconfig B//NetConfig) > diff_netconfig.txt +``` + +For each xlate/table/tclproc that differs, produce a per-file `diff -u` in the appendix. + +For NetConfig: a structural diff is more useful than a line diff. Try to extract thread/route blocks with `awk '/^thread / .. /^}/' NetConfig` and diff those. + +### Phase 4: write the markdown report + +Use `write_file` with Y/N confirm. Path: `$LARRY_HOME/sessions/regress__vs__.md`. + +## Anomaly heuristics — flag these to Bryan first + +- **Thread on A but not on B** (or vice versa): potential missing migration or stale on one side. +- **Same thread name, different host:port**: configuration drift — could be intentional (test vs prod) or a deploy mistake. +- **Same xlate name, different hash**: the most common regression source. Side-by-side diff goes in the report. +- **Same tclproc name, different hash, but smaller side**: someone reverted or partially merged. +- **NetConfig has thread `X` referencing xlate `Y` but `Y` is missing on that side**: broken reference, deploy incomplete. +- **`*.pc` process files differ in port or driver type**: connection target changed. + +## Boundaries + +- **Never bounce processes during a regression run.** Read-only, full stop. +- **Never copy data (smat archives, .idx, archived messages).** They're large and irrelevant; exclude in rsync. +- **Do not attempt to compare smat content** unless Bryan explicitly asks — the point is structural/config drift, not message-history equivalence. +- If side B is unreachable, **say so plainly** and have Bryan run Phase 1 there himself. + +## Output for Larry to synthesize back + +Always close with: +- one-line **headline** (e.g. "*3 xlates differ on sideB; 1 thread missing on sideA; NetConfig has 12 line diffs centered on the routing block for d_lab_inbound*") +- the **report path** (`write_file` location) +- top-3 anomalies for Bryan to look at first +- one tight clarifying question if anything was ambiguous diff --git a/install-larry.sh b/install-larry.sh new file mode 100755 index 0000000..db8b03c --- /dev/null +++ b/install-larry.sh @@ -0,0 +1,173 @@ +#!/usr/bin/env bash +# install-larry.sh — bootstrap Larry-Anywhere on a fresh remote shell. +# No root, no package install, no sudo. Writes only into $LARRY_HOME. +# +# Usage: +# curl -fsSL /install-larry.sh | bash +# +# Or with explicit base URL: +# LARRY_BASE_URL=https://example.com/larry-anywhere bash install-larry.sh +# +# Env vars: +# LARRY_HOME install location (default: $HOME/.larry) +# LARRY_BASE_URL where to fetch files from (no trailing slash) +# LARRY_BIN_DIR where to symlink the `larry` command (default: $HOME/bin) +set -eu + +LARRY_HOME="${LARRY_HOME:-$HOME/.larry}" +# Canonical hosting: bojj27/cloverleaf-larry on GitHub, branch main, raw view. +# Override via env if you fork or mirror elsewhere. +LARRY_BASE_URL="${LARRY_BASE_URL:-https://raw.githubusercontent.com/bojj27/cloverleaf-larry/main}" +LARRY_BIN_DIR="${LARRY_BIN_DIR:-$HOME/bin}" + +C_RESET=$'\033[0m'; C_BOLD=$'\033[1m'; C_GREEN=$'\033[32m' +C_YELLOW=$'\033[33m'; C_RED=$'\033[31m'; C_CYAN=$'\033[36m' + +say() { printf '%s%sinstall-larry>%s %s\n' "$C_CYAN" "$C_BOLD" "$C_RESET" "$*"; } +ok() { printf ' %s✓%s %s\n' "$C_GREEN" "$C_RESET" "$*"; } +warn() { printf ' %s!%s %s\n' "$C_YELLOW" "$C_RESET" "$*"; } +die() { printf '%serror:%s %s\n' "$C_RED" "$C_RESET" "$*" >&2; exit 1; } + +# ───────────────────────────────────────────────────────────────────────────── +# Detect platform +# ───────────────────────────────────────────────────────────────────────────── +UNAME_S="$(uname -s 2>/dev/null || echo unknown)" +PLATFORM="" +case "$UNAME_S" in + Linux*) PLATFORM="linux" ;; + Darwin*) PLATFORM="darwin" ;; + CYGWIN*|MINGW*|MSYS*) PLATFORM="windows-cygwin" ;; # MobaXterm lives here + *) PLATFORM="unknown" ;; +esac + +ARCH="$(uname -m 2>/dev/null || echo unknown)" +case "$ARCH" in + x86_64|amd64) ARCH_NORM="amd64" ;; + aarch64|arm64) ARCH_NORM="arm64" ;; + i?86) ARCH_NORM="i386" ;; + *) ARCH_NORM="$ARCH" ;; +esac + +say "platform: $PLATFORM/$ARCH_NORM • LARRY_HOME=$LARRY_HOME" + +# ───────────────────────────────────────────────────────────────────────────── +# Check required commands +# ───────────────────────────────────────────────────────────────────────────── +command -v bash >/dev/null 2>&1 || die "bash not found" +command -v curl >/dev/null 2>&1 || die "curl not found" + +# ───────────────────────────────────────────────────────────────────────────── +# Make dirs +# ───────────────────────────────────────────────────────────────────────────── +mkdir -p "$LARRY_HOME"/{agents,sessions,bin,lib} || die "cannot create $LARRY_HOME" +chmod 700 "$LARRY_HOME" 2>/dev/null || true +ok "created $LARRY_HOME" + +# ───────────────────────────────────────────────────────────────────────────── +# Fetch the scripts. If LARRY_BASE_URL is not set, try to detect being run +# from a local checkout (sibling files present) — copy locally instead. +# ───────────────────────────────────────────────────────────────────────────── +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" 2>/dev/null && pwd)" || SCRIPT_DIR="" + +fetch() { + # $1 = remote relative path, $2 = local destination + if [ -n "$LARRY_BASE_URL" ]; then + say "fetching $1" + curl -fsSL --max-time 30 "$LARRY_BASE_URL/$1" -o "$2" \ + && ok "$2" || die "failed to fetch $1" + elif [ -n "$SCRIPT_DIR" ] && [ -f "$SCRIPT_DIR/$1" ]; then + cp "$SCRIPT_DIR/$1" "$2" && ok "copied $1 (local)" + else + die "no LARRY_BASE_URL set and $1 not found in script dir" + fi +} + +fetch larry.sh "$LARRY_HOME/larry.sh" +fetch larry-tunnel.sh "$LARRY_HOME/larry-tunnel.sh" +fetch agents/larry.md "$LARRY_HOME/agents/larry.md" +fetch agents/clover.md "$LARRY_HOME/agents/clover.md" +fetch agents/cloverleaf-cheatsheet.md "$LARRY_HOME/agents/cloverleaf-cheatsheet.md" +fetch agents/regress.md "$LARRY_HOME/agents/regress.md" +fetch larry-rollback.sh "$LARRY_HOME/larry-rollback.sh" +fetch lib/nc-parse.sh "$LARRY_HOME/lib/nc-parse.sh" +fetch lib/nc-inbound.sh "$LARRY_HOME/lib/nc-inbound.sh" +fetch lib/nc-make-jump.sh "$LARRY_HOME/lib/nc-make-jump.sh" +fetch lib/hl7-field.sh "$LARRY_HOME/lib/hl7-field.sh" +fetch lib/nc-msgs.sh "$LARRY_HOME/lib/nc-msgs.sh" +fetch lib/nc-document.sh "$LARRY_HOME/lib/nc-document.sh" +fetch lib/nc-diff-interface.sh "$LARRY_HOME/lib/nc-diff-interface.sh" +fetch lib/nc-find.sh "$LARRY_HOME/lib/nc-find.sh" +fetch lib/nc-insert-protocol.sh "$LARRY_HOME/lib/nc-insert-protocol.sh" +fetch lib/hl7-diff.sh "$LARRY_HOME/lib/hl7-diff.sh" +fetch lib/nc-regression.sh "$LARRY_HOME/lib/nc-regression.sh" +fetch lib/journal.sh "$LARRY_HOME/lib/journal.sh" +fetch VERSION "$LARRY_HOME/VERSION" +chmod +x "$LARRY_HOME/larry.sh" "$LARRY_HOME/larry-tunnel.sh" "$LARRY_HOME/larry-rollback.sh" "$LARRY_HOME/lib/"*.sh + +# ───────────────────────────────────────────────────────────────────────────── +# jq fallback — download static binary into $LARRY_HOME/bin/ if missing +# ───────────────────────────────────────────────────────────────────────────── +if ! command -v jq >/dev/null 2>&1 && [ ! -x "$LARRY_HOME/bin/jq" ]; then + say "jq not found — fetching static binary into $LARRY_HOME/bin/jq" + JQ_BASE="https://github.com/jqlang/jq/releases/download/jq-1.7.1" + JQ_URL="" + case "$PLATFORM/$ARCH_NORM" in + linux/amd64) JQ_URL="$JQ_BASE/jq-linux-amd64" ;; + linux/arm64) JQ_URL="$JQ_BASE/jq-linux-arm64" ;; + linux/i386) JQ_URL="$JQ_BASE/jq-linux-i386" ;; + darwin/amd64) JQ_URL="$JQ_BASE/jq-macos-amd64" ;; + darwin/arm64) JQ_URL="$JQ_BASE/jq-macos-arm64" ;; + windows-cygwin/amd64) JQ_URL="$JQ_BASE/jq-windows-amd64.exe" ;; + windows-cygwin/i386) JQ_URL="$JQ_BASE/jq-windows-i386.exe" ;; + *) warn "no jq binary known for $PLATFORM/$ARCH_NORM — install jq manually" ;; + esac + if [ -n "$JQ_URL" ]; then + local_jq="$LARRY_HOME/bin/jq" + case "$PLATFORM" in windows-cygwin) local_jq="$LARRY_HOME/bin/jq.exe" ;; esac + if curl -fsSL --max-time 60 "$JQ_URL" -o "$local_jq"; then + chmod +x "$local_jq" + ok "jq -> $local_jq" + else + warn "could not download jq from $JQ_URL" + fi + fi +else + ok "jq available" +fi + +# ───────────────────────────────────────────────────────────────────────────── +# Drop a `larry` shim onto PATH (best-effort) +# ───────────────────────────────────────────────────────────────────────────── +mkdir -p "$LARRY_BIN_DIR" 2>/dev/null || true +if [ -d "$LARRY_BIN_DIR" ] && [ -w "$LARRY_BIN_DIR" ]; then + cat > "$LARRY_BIN_DIR/larry" <&2; exit 2 ;; + esac + shift +done + +[ -f "$JOURNAL_INDEX" ] || { echo "no journal (no writes have been made yet)" >&2; exit 1; } + +C_BOLD=$'\033[1m'; C_DIM=$'\033[2m'; C_RESET=$'\033[0m' +C_RED=$'\033[31m'; C_GREEN=$'\033[32m' + +_list_filter() { + local ses="$1" + awk -F'\t' -v s="$ses" ' + NR==1 { next } + s=="" || $2==s { print } + ' "$JOURNAL_INDEX" | tail -r 2>/dev/null || awk -F'\t' -v s="$ses" ' + NR==1 { next } + s=="" || $2==s { print } + ' "$JOURNAL_INDEX" | awk '{a[NR]=$0} END{for(i=NR;i>=1;i--) print a[i]}' +} + +if [ "$MODE" = "list" ]; then + printf '%stimestamp session-id seq target%s\n' "$C_BOLD" "$C_RESET" + _list_filter "$SESSION" | awk -F'\t' '{printf " %-26s %-28s %-4s %s\n", $1, $2, $3, $4}' + exit 0 +fi + +build_targets() { + case "$MODE" in + session) + [ -n "$SESSION" ] || { echo "--session needs a value" >&2; exit 2; } + _list_filter "$SESSION" + ;; + last) + [ -n "$N" ] || { echo "--last needs N" >&2; exit 2; } + _list_filter "" | head -n "$N" + ;; + entry) + local ses seq + ses="${ENTRY_ID%/*}"; seq="${ENTRY_ID##*/}"; seq="${seq%%_*}" + awk -F'\t' -v s="$ses" -v sq="$seq" '$2==s && $3==sq' "$JOURNAL_INDEX" + ;; + target) + awk -F'\t' -v t="$TARGET" '$4==t' "$JOURNAL_INDEX" | awk '{a[NR]=$0} END{for(i=NR;i>=1;i--) print a[i]}' + ;; + *) + echo "specify --list, --session, --last N, --entry ID, or --target PATH" >&2 + exit 2 + ;; + esac +} + +ENTRIES=$(build_targets) +[ -n "$ENTRIES" ] || { echo "(no matching journal entries)"; exit 0; } + +printf '%sWill roll back %s entr(y/ies):%s\n' "$C_BOLD" "$(printf '%s\n' "$ENTRIES" | wc -l | tr -d ' ')" "$C_RESET" +printf '%s' "$ENTRIES" | awk -F'\t' '{printf " %s %-26s %s\n", $3, $1, $4}' +echo "" + +if [ "$DRY" = "1" ]; then + echo "(dry-run; no changes)" + exit 0 +fi + +if [ "$YES" != "1" ]; then + printf '%sProceed?%s [y/N]: ' "$C_BOLD" "$C_RESET" + read -r ans /dev/null || true + cp -p "$backup" "$target" && printf ' %s✓ restored%s %s ← %s\n' "$C_GREEN" "$C_RESET" "$target" "$backup" + else + printf ' %s✗ backup missing for%s %s (looked at %s)\n' "$C_RED" "$C_RESET" "$target" "$backup" + fi + fi +done + +echo "" +echo "Pre-rollback copies left at .larry-prerollback. in case you want to redo. Clean up at your leisure." diff --git a/larry-tunnel.sh b/larry-tunnel.sh new file mode 100755 index 0000000..55e566d --- /dev/null +++ b/larry-tunnel.sh @@ -0,0 +1,183 @@ +#!/usr/bin/env bash +# larry-tunnel — open a reverse SSH tunnel from a remote shell back to +# a public hop, so Bryan's home Larry can SSH "in" to this machine. +# +# Two modes: +# 1. Bryan-controlled hop (recommended once set up) +# Requires: an SSH-reachable host you control (bjnoela.com or a VPS) +# with `GatewayPorts clientspecified` (or yes) in sshd_config, plus +# either your SSH pubkey installed there or password auth allowed. +# +# 2. serveo.net fallback (zero-config, no account, no install) +# Just works. Less private. Useful when the primary hop is offline +# or not yet configured. +# +# Usage: +# larry-tunnel.sh # uses env vars or prompts +# larry-tunnel.sh --serveo # force serveo.net mode +# larry-tunnel.sh --hop user@host:port # force custom-hop mode +# +# Env vars (set in $LARRY_HOME/.env or shell): +# LARRY_HOP_HOST e.g. bjnoela.com +# LARRY_HOP_USER e.g. larry-tunnel (a low-priv user on the hop) +# LARRY_HOP_PORT SSH port on the hop (default 22) +# LARRY_HOP_BIND remote-side bind port (default 0 = auto) +# LARRY_LOCAL_PORT local SSH port to expose (default 22) +# LARRY_HOP_KEY path to private key for the hop (default ~/.ssh/id_rsa or id_ed25519) +# +# Files: +# $LARRY_HOME/tunnel.pid PID of the active tunnel ssh process +# $LARRY_HOME/tunnel.url the public host:port to dial from home +# $LARRY_HOME/tunnel.log ssh stderr/log +set -u +set -o pipefail + +LARRY_HOME="${LARRY_HOME:-$HOME/.larry}" +mkdir -p "$LARRY_HOME" +[ -f "$LARRY_HOME/.env" ] && { set -a; . "$LARRY_HOME/.env"; set +a; } + +LARRY_HOP_HOST="${LARRY_HOP_HOST:-}" +LARRY_HOP_USER="${LARRY_HOP_USER:-}" +LARRY_HOP_PORT="${LARRY_HOP_PORT:-22}" +LARRY_HOP_BIND="${LARRY_HOP_BIND:-0}" +LARRY_LOCAL_PORT="${LARRY_LOCAL_PORT:-22}" +LARRY_HOP_KEY="${LARRY_HOP_KEY:-}" + +C_RESET=$'\033[0m'; C_BOLD=$'\033[1m'; C_DIM=$'\033[2m' +C_RED=$'\033[31m'; C_GREEN=$'\033[32m'; C_YELLOW=$'\033[33m'; C_CYAN=$'\033[36m' + +say() { printf '%s%slarry-tunnel>%s %s\n' "$C_CYAN" "$C_BOLD" "$C_RESET" "$*"; } +err() { printf '%serror:%s %s\n' "$C_RED" "$C_RESET" "$*" >&2; } +warn() { printf '%swarn:%s %s\n' "$C_YELLOW" "$C_RESET" "$*" >&2; } + +MODE="" +for arg in "$@"; do + case "$arg" in + --serveo) MODE="serveo" ;; + --hop) MODE="hop" ;; + --hop=*) MODE="hop"; spec="${arg#--hop=}" + LARRY_HOP_USER="${spec%@*}" + rest="${spec#*@}" + LARRY_HOP_HOST="${rest%:*}" + [ "$rest" != "${rest%:*}" ] && LARRY_HOP_PORT="${rest##*:}" + ;; + --status) + if [ -f "$LARRY_HOME/tunnel.pid" ] && kill -0 "$(cat "$LARRY_HOME/tunnel.pid")" 2>/dev/null; then + say "running (PID $(cat "$LARRY_HOME/tunnel.pid"))" + [ -f "$LARRY_HOME/tunnel.url" ] && say "public: $(cat "$LARRY_HOME/tunnel.url")" + exit 0 + else + say "not running" + exit 1 + fi + ;; + --stop) + if [ -f "$LARRY_HOME/tunnel.pid" ]; then + kill "$(cat "$LARRY_HOME/tunnel.pid")" 2>/dev/null && say "stopped" + rm -f "$LARRY_HOME/tunnel.pid" "$LARRY_HOME/tunnel.url" + fi + exit 0 + ;; + -h|--help) + sed -n '2,32p' "$0"; exit 0 + ;; + *) err "unknown arg: $arg"; exit 2 ;; + esac +done + +# Pick mode if not forced +if [ -z "$MODE" ]; then + if [ -n "$LARRY_HOP_HOST" ] && [ -n "$LARRY_HOP_USER" ]; then + MODE="hop" + else + MODE="serveo" + fi +fi + +# Identify ssh key flag (only if set and exists) +KEY_FLAG="" +if [ -n "$LARRY_HOP_KEY" ] && [ -f "$LARRY_HOP_KEY" ]; then + KEY_FLAG="-i $LARRY_HOP_KEY" +fi + +# Common ssh options for reverse tunnel +SSH_OPTS=( + -N + -o ServerAliveInterval=30 + -o ServerAliveCountMax=3 + -o ExitOnForwardFailure=yes + -o StrictHostKeyChecking=accept-new + -o UserKnownHostsFile="$LARRY_HOME/known_hosts" +) + +run_hop() { + if [ -z "$LARRY_HOP_HOST" ] || [ -z "$LARRY_HOP_USER" ]; then + err "hop mode needs LARRY_HOP_HOST and LARRY_HOP_USER (or --hop=user@host)" + exit 2 + fi + say "opening reverse tunnel: ${LARRY_HOP_USER}@${LARRY_HOP_HOST}:${LARRY_HOP_PORT}" + say " binds remote port ${LARRY_HOP_BIND} -> localhost:${LARRY_LOCAL_PORT}" + while true; do + : > "$LARRY_HOME/tunnel.log" + # shellcheck disable=SC2086 + ssh $KEY_FLAG \ + "${SSH_OPTS[@]}" \ + -p "$LARRY_HOP_PORT" \ + -R "${LARRY_HOP_BIND}:localhost:${LARRY_LOCAL_PORT}" \ + "${LARRY_HOP_USER}@${LARRY_HOP_HOST}" \ + 2> "$LARRY_HOME/tunnel.log" & + local pid=$! + echo "$pid" > "$LARRY_HOME/tunnel.pid" + printf '%s:%s\n' "$LARRY_HOP_HOST" "$LARRY_HOP_BIND" > "$LARRY_HOME/tunnel.url" + say "ssh PID $pid — dial from home: ssh -p ${LARRY_HOP_BIND} @${LARRY_HOP_HOST}" + wait "$pid" + rc=$? + warn "tunnel exited (rc=$rc) — reconnecting in 5s" + sleep 5 + done +} + +run_serveo() { + say "opening serveo.net reverse tunnel (no account needed)" + say " binds an auto-assigned port -> localhost:${LARRY_LOCAL_PORT}" + say " serveo.net is third-party; do NOT use for sensitive sessions" + while true; do + : > "$LARRY_HOME/tunnel.log" + # serveo prints assignment on stderr/stdout — we capture both + ssh \ + "${SSH_OPTS[@]}" \ + -R "0:localhost:${LARRY_LOCAL_PORT}" \ + serveo.net \ + > "$LARRY_HOME/tunnel.log" 2>&1 & + local pid=$! + echo "$pid" > "$LARRY_HOME/tunnel.pid" + + # Try to extract the forwarded URL + local attempts=0 url="" + while [ "$attempts" -lt 30 ] && [ -z "$url" ]; do + sleep 1 + attempts=$((attempts + 1)) + url=$(grep -Eo 'Forwarding tcp from [^ ]+|tcp://[^ ]+|serveo\.net:[0-9]+' \ + "$LARRY_HOME/tunnel.log" 2>/dev/null | head -1) + done + if [ -n "$url" ]; then + echo "$url" > "$LARRY_HOME/tunnel.url" + say "public: $url" + say "dial from home: ssh -p @serveo.net (PORT from URL above)" + else + warn "could not read serveo.net assignment — check $LARRY_HOME/tunnel.log" + fi + wait "$pid" + rc=$? + warn "tunnel exited (rc=$rc) — reconnecting in 5s" + sleep 5 + done +} + +trap 'kill $(cat "$LARRY_HOME/tunnel.pid" 2>/dev/null) 2>/dev/null; rm -f "$LARRY_HOME/tunnel.pid" "$LARRY_HOME/tunnel.url"; exit 0' INT TERM + +case "$MODE" in + hop) run_hop ;; + serveo) run_serveo ;; + *) err "unknown mode: $MODE"; exit 2 ;; +esac diff --git a/larry.sh b/larry.sh new file mode 100755 index 0000000..7423d6b --- /dev/null +++ b/larry.sh @@ -0,0 +1,918 @@ +#!/usr/bin/env bash +# larry-anywhere — portable Larry for remote shells (Linux + MobaXterm) +# Single file. No installs. curl + jq + bash. +# +# Usage: +# larry.sh # interactive in $PWD +# larry.sh /path/to/cloverleaf/root # interactive, cd into that path first +# larry.sh --no-update # skip self-update +# larry.sh --version # print version and exit +# larry.sh --help # print help and exit +# +# Env vars: +# LARRY_HOME where to cache config/sessions (default: ~/.larry) +# LARRY_UPDATE_URL URL of latest larry.sh for self-update (optional) +# LARRY_AGENTS_URL base URL for agents/ refresh (optional) +# LARRY_MODEL Claude model (default: claude-sonnet-4-6) +# LARRY_MAX_TOKENS max output tokens per turn (default: 8192) +# LARRY_NO_UPDATE set to 1 to disable self-update +# ANTHROPIC_API_KEY overrides $LARRY_HOME/.env if set +# +# Slash commands during chat: +# /quit /exit /q exit +# /model switch model for this session +# /cd change working directory +# /reset clear conversation history (keeps log file) +# /load paste a file's contents as your next user message +# /sys print the active system prompt +# /help this help +set -u +set -o pipefail + +# ───────────────────────────────────────────────────────────────────────────── +# Config +# ───────────────────────────────────────────────────────────────────────────── +LARRY_VERSION="0.1.0" +LARRY_HOME="${LARRY_HOME:-$HOME/.larry}" +LARRY_UPDATE_URL="${LARRY_UPDATE_URL:-https://raw.githubusercontent.com/bojj27/cloverleaf-larry/main/larry.sh}" +LARRY_AGENTS_URL="${LARRY_AGENTS_URL:-https://raw.githubusercontent.com/bojj27/cloverleaf-larry/main/agents}" +LARRY_MODEL="${LARRY_MODEL:-claude-sonnet-4-6}" +LARRY_MAX_TOKENS="${LARRY_MAX_TOKENS:-8192}" +LARRY_API_URL="${LARRY_API_URL:-https://api.anthropic.com/v1/messages}" +LARRY_NO_UPDATE="${LARRY_NO_UPDATE:-0}" + +# ───────────────────────────────────────────────────────────────────────────── +# Colors (only if stdout is a tty) +# ───────────────────────────────────────────────────────────────────────────── +if [ -t 1 ]; then + C_RESET=$'\033[0m'; C_BOLD=$'\033[1m'; C_DIM=$'\033[2m' + C_RED=$'\033[31m'; C_GREEN=$'\033[32m'; C_YELLOW=$'\033[33m' + C_BLUE=$'\033[34m'; C_MAGENTA=$'\033[35m'; C_CYAN=$'\033[36m' +else + C_RESET=''; C_BOLD=''; C_DIM=''; C_RED=''; C_GREEN='' + C_YELLOW=''; C_BLUE=''; C_MAGENTA=''; C_CYAN='' +fi + +log() { printf '%s[%s]%s %s\n' "$C_DIM" "$(date +%H:%M:%S)" "$C_RESET" "$*" >&2; } +err() { printf '%serror:%s %s\n' "$C_RED" "$C_RESET" "$*" >&2; } +warn() { printf '%swarn:%s %s\n' "$C_YELLOW" "$C_RESET" "$*" >&2; } +larry_say() { printf '%s%slarry>%s %s\n' "$C_MAGENTA" "$C_BOLD" "$C_RESET" "$*"; } + +# ───────────────────────────────────────────────────────────────────────────── +# CLI args +# ───────────────────────────────────────────────────────────────────────────── +ARG_DIR="" +for arg in "$@"; do + case "$arg" in + --version|-V) echo "larry-anywhere $LARRY_VERSION"; exit 0 ;; + --help|-h) sed -n '2,30p' "$0"; exit 0 ;; + --no-update) LARRY_NO_UPDATE=1 ;; + -*) err "unknown flag: $arg"; exit 2 ;; + *) ARG_DIR="$arg" ;; + esac +done + +# ───────────────────────────────────────────────────────────────────────────── +# Dependency check +# ───────────────────────────────────────────────────────────────────────────── +need_cmd() { + command -v "$1" >/dev/null 2>&1 || { err "missing required command: $1"; exit 1; } +} +need_cmd curl +# jq: allow a local copy in $LARRY_HOME/bin/jq as fallback +if ! command -v jq >/dev/null 2>&1; then + if [ -x "$LARRY_HOME/bin/jq" ]; then + PATH="$LARRY_HOME/bin:$PATH" + else + err "missing jq. Install via your shell's package mechanism, or place a static jq binary at $LARRY_HOME/bin/jq" + err "Download: https://github.com/jqlang/jq/releases (pick the static binary for your OS)" + exit 1 + fi +fi + +# ───────────────────────────────────────────────────────────────────────────── +# Bootstrap LARRY_HOME and API key +# ───────────────────────────────────────────────────────────────────────────── +mkdir -p "$LARRY_HOME/agents" "$LARRY_HOME/sessions" "$LARRY_HOME/bin" 2>/dev/null || { + err "cannot create $LARRY_HOME — set LARRY_HOME to a writable path and retry"; exit 1; +} +chmod 700 "$LARRY_HOME" 2>/dev/null || true + +if [ -z "${ANTHROPIC_API_KEY:-}" ]; then + if [ -f "$LARRY_HOME/.env" ]; then + # shellcheck disable=SC1091 + set -a; . "$LARRY_HOME/.env"; set +a + fi +fi + +prompt_api_key() { + printf '%sFirst-run setup%s\n' "$C_BOLD" "$C_RESET" + echo " Paste your Anthropic API key (starts with sk-ant-...) and press Enter." + echo " It will be saved to $LARRY_HOME/.env with permissions 0600." + echo "" + printf ' ANTHROPIC_API_KEY: ' + stty -echo 2>/dev/null + read -r key + stty echo 2>/dev/null + echo "" + if [ -z "$key" ]; then err "no key entered"; exit 1; fi + umask 077 + printf 'ANTHROPIC_API_KEY=%s\n' "$key" > "$LARRY_HOME/.env" + chmod 600 "$LARRY_HOME/.env" + ANTHROPIC_API_KEY="$key" + log "API key saved." +} + +if [ -z "${ANTHROPIC_API_KEY:-}" ]; then + prompt_api_key +fi + +# ───────────────────────────────────────────────────────────────────────────── +# Fetch agents if missing +# ───────────────────────────────────────────────────────────────────────────── +LARRY_AGENT_FILES="larry.md clover.md cloverleaf-cheatsheet.md regress.md" + +fetch_agents_or_warn() { + local need=0 + for f in larry.md clover.md; do + [ -f "$LARRY_HOME/agents/$f" ] || need=1 + done + [ "$need" = "0" ] && return 0 + + if [ -n "$LARRY_AGENTS_URL" ]; then + log "fetching agent definitions from $LARRY_AGENTS_URL" + for f in $LARRY_AGENT_FILES; do + curl -fsSL --max-time 10 "$LARRY_AGENTS_URL/$f" -o "$LARRY_HOME/agents/$f" \ + || { warn "could not fetch $f — using built-in fallback"; write_fallback_agent "$f"; } + done + else + warn "agent files missing and LARRY_AGENTS_URL not set — writing built-in fallback (larry+clover only)" + write_fallback_agent larry.md + write_fallback_agent clover.md + fi +} + +write_fallback_agent() { + case "$1" in + larry.md) cat > "$LARRY_HOME/agents/larry.md" <<'AGENT_EOF' +You are Larry, Bryan's team orchestrator at myPKA, running in portable mode on a remote shell. +First sentence when asked who you are: "I'm Larry, your team orchestrator at myPKA (running portable mode)." +Focus: Cloverleaf interface build and Netconfig analysis. No PHI involved. No production push. +Tools available: read_file, list_dir, grep_files, glob_files, write_file (Y/N confirm), bash_exec (Y/N confirm). +Style: concise, direct, cite path:line for code references. Ask one tight clarifying question only if a critical detail is missing. +AGENT_EOF + ;; + clover.md) cat > "$LARRY_HOME/agents/clover.md" <<'AGENT_EOF' +When the task is Cloverleaf-specific, channel Clover, Cloverleaf Integration Expert. +Focus: UPOC TCL coding, interface specs, clean documentation. Idempotent, auditable, source-cited. +Output: one-line status, artifact list, anomalies/open questions. +AGENT_EOF + ;; + esac +} + +fetch_agents_or_warn + +# ───────────────────────────────────────────────────────────────────────────── +# Self-update +# ───────────────────────────────────────────────────────────────────────────── +self_update() { + [ "$LARRY_NO_UPDATE" = "1" ] && return 0 + [ -z "$LARRY_UPDATE_URL" ] && return 0 + local self="$0" + case "$self" in /*) ;; *) self="$PWD/$self" ;; esac + [ -w "$self" ] || return 0 + + local tmp="$LARRY_HOME/larry.sh.new" + if curl -fsSL --max-time 5 "$LARRY_UPDATE_URL" -o "$tmp" 2>/dev/null; then + if [ -s "$tmp" ] && ! cmp -s "$self" "$tmp"; then + local new_ver + new_ver=$(grep -m1 '^LARRY_VERSION=' "$tmp" | sed 's/.*"\(.*\)".*/\1/') + log "update found: $LARRY_VERSION -> ${new_ver:-?}" + cp "$tmp" "$self" && chmod +x "$self" + rm -f "$tmp" + log "relaunching..." + exec "$self" --no-update ${ARG_DIR:+"$ARG_DIR"} + fi + rm -f "$tmp" + fi + + # Also refresh agents + if [ -n "$LARRY_AGENTS_URL" ]; then + for f in larry.md clover.md; do + curl -fsSL --max-time 5 "$LARRY_AGENTS_URL/$f" -o "$LARRY_HOME/agents/$f.new" 2>/dev/null \ + && [ -s "$LARRY_HOME/agents/$f.new" ] \ + && mv "$LARRY_HOME/agents/$f.new" "$LARRY_HOME/agents/$f" \ + || rm -f "$LARRY_HOME/agents/$f.new" + done + fi +} +self_update + +# ───────────────────────────────────────────────────────────────────────────── +# Cloverleaf environment detection +# Surfaces HCIROOT / HCISITE / HCISITEDIR and which tool layer is present +# (modern cloverleaf-tools.pyz, classic Eric scripts, or neither). +# Result is appended to the system prompt so the model knows where it is. +# ───────────────────────────────────────────────────────────────────────────── +detect_cloverleaf_env() { + CLOVERLEAF_CTX="" + local lines=() + if [ -n "${HCIROOT:-}" ]; then + lines+=("HCIROOT=$HCIROOT (exists=$([ -d "$HCIROOT" ] && echo yes || echo no))") + else + lines+=("HCIROOT=") + fi + if [ -n "${HCISITE:-}" ]; then + local sitedir="${HCISITEDIR:-${HCIROOT:-}/$HCISITE}" + lines+=("HCISITE=$HCISITE") + lines+=("HCISITEDIR=$sitedir (exists=$([ -d "$sitedir" ] && echo yes || echo no))") + if [ -d "$sitedir" ]; then + [ -f "$sitedir/NetConfig" ] && lines+=("NetConfig present: $(wc -l < "$sitedir/NetConfig" | tr -d ' ') lines, $(wc -c < "$sitedir/NetConfig" | tr -d ' ') bytes") + [ -d "$sitedir/Xlate" ] && lines+=("Xlate/: $(find "$sitedir/Xlate" -maxdepth 1 -type f 2>/dev/null | wc -l | tr -d ' ') files") + [ -d "$sitedir/tables" ] && lines+=("tables/: $(find "$sitedir/tables" -maxdepth 1 -type f 2>/dev/null | wc -l | tr -d ' ') files") + [ -d "$sitedir/tclprocs" ] && lines+=("tclprocs/: $(find "$sitedir/tclprocs" -maxdepth 1 -type f 2>/dev/null | wc -l | tr -d ' ') files") + [ -d "$sitedir/formats" ] && lines+=("formats/: $(find "$sitedir/formats" -maxdepth 1 -type f 2>/dev/null | wc -l | tr -d ' ') files") + fi + else + lines+=("HCISITE=") + fi + if [ -n "${HCIROOT:-}" ] && [ -d "$HCIROOT" ]; then + local site_count + site_count=$(find "$HCIROOT" -mindepth 1 -maxdepth 1 -type d \ + ! -name 'archiving' ! -name 'master' ! -name 'lib' ! -name 'tcl' ! -name 'server' \ + ! -name 'client' ! -name 'clgui' ! -name 'cchgs' ! -name 'epic*' ! -name 'beaker' \ + ! -name 'Alerts' ! -name 'AppDefaults' ! -name 'Tables' ! -name 'backup*' \ + 2>/dev/null | wc -l | tr -d ' ') + lines+=("HCIROOT site-like subdirs: $site_count") + fi + + # Tool layer detection + local pyz_path="" + if command -v cloverleaf-tools.pyz >/dev/null 2>&1; then + pyz_path=$(command -v cloverleaf-tools.pyz) + elif [ -x "./cloverleaf-tools.pyz" ]; then + pyz_path="$PWD/cloverleaf-tools.pyz" + elif [ -n "${HCIROOT:-}" ] && [ -x "$HCIROOT/cloverleaf-tools.pyz" ]; then + pyz_path="$HCIROOT/cloverleaf-tools.pyz" + fi + if [ -n "$pyz_path" ]; then + lines+=("Modern tools: cloverleaf-tools.pyz at $pyz_path") + fi + # Classic Eric scripts — detect a representative few + local classic_found="" + for c in tbn tbp tbh tbpr hlq mr mp mg hl awkcut sites each_site list_full_routes dbExtract; do + command -v "$c" >/dev/null 2>&1 && classic_found+="$c " + done + if [ -n "$classic_found" ]; then + lines+=("Classic tools on PATH: $classic_found") + fi + if [ -z "$pyz_path" ] && [ -z "$classic_found" ]; then + lines+=("No Cloverleaf-tooling on PATH — Larry will fall back to bash one-liners only.") + fi + + # Compose for system prompt + CLOVERLEAF_CTX=$'\n\n## Detected runtime context (read-only)\n' + for ln in "${lines[@]}"; do + CLOVERLEAF_CTX+="- $ln"$'\n' + done +} +detect_cloverleaf_env + +# ───────────────────────────────────────────────────────────────────────────── +# Session state +# ───────────────────────────────────────────────────────────────────────────── +SESSION_ID="$(date +%Y-%m-%d-%H%M%S)-$$" +MESSAGES_FILE="$LARRY_HOME/sessions/$SESSION_ID.messages.json" +LOG_FILE="$LARRY_HOME/sessions/$SESSION_ID.log.md" +printf '[]' > "$MESSAGES_FILE" +{ + echo "# Larry-Anywhere session $SESSION_ID" + echo "- start: $(date -Iseconds 2>/dev/null || date)" + echo "- model: $LARRY_MODEL" + echo "- host: $(hostname 2>/dev/null || echo unknown)" + echo "- pwd: $(pwd)" + echo "" +} > "$LOG_FILE" + +log_section() { printf '\n## %s\n' "$1" >> "$LOG_FILE"; } +log_append() { printf '%s\n' "$1" >> "$LOG_FILE"; } + +# ───────────────────────────────────────────────────────────────────────────── +# Message store helpers +# ───────────────────────────────────────────────────────────────────────────── +add_user_text() { + local content="$1" + local tmp; tmp=$(mktemp) + jq --arg c "$content" '. + [{"role":"user","content":[{"type":"text","text":$c}]}]' "$MESSAGES_FILE" > "$tmp" \ + && mv "$tmp" "$MESSAGES_FILE" +} +add_assistant_blocks() { + local blocks="$1" + local tmp; tmp=$(mktemp) + jq --argjson b "$blocks" '. + [{"role":"assistant","content":$b}]' "$MESSAGES_FILE" > "$tmp" \ + && mv "$tmp" "$MESSAGES_FILE" +} +add_user_tool_results() { + local blocks="$1" + local tmp; tmp=$(mktemp) + jq --argjson b "$blocks" '. + [{"role":"user","content":$b}]' "$MESSAGES_FILE" > "$tmp" \ + && mv "$tmp" "$MESSAGES_FILE" +} + +# ───────────────────────────────────────────────────────────────────────────── +# Tool implementations +# ───────────────────────────────────────────────────────────────────────────── +tool_read_file() { + local path="$1" + if [ ! -e "$path" ]; then echo "ERROR: file not found: $path"; return; fi + if [ ! -f "$path" ]; then echo "ERROR: not a regular file: $path"; return; fi + local size; size=$(wc -c < "$path" 2>/dev/null || echo 0) + if [ "$size" -gt 250000 ]; then + echo "ERROR: file too large ($size bytes, limit 250KB). Use grep_files to target sections." + return + fi + awk '{printf "%6d\t%s\n", NR, $0}' "$path" +} + +tool_list_dir() { + local path="${1:-.}" + if [ ! -d "$path" ]; then echo "ERROR: not a directory: $path"; return; fi + ls -la --color=never "$path" 2>/dev/null || ls -la "$path" +} + +tool_grep_files() { + local pattern="$1"; local path="${2:-.}" + if [ ! -e "$path" ]; then echo "ERROR: path not found: $path"; return; fi + local total + total=$(grep -rnI --color=never -c "$pattern" "$path" 2>/dev/null \ + | awk -F: '{s+=$NF} END {print s+0}') + grep -rnI --color=never "$pattern" "$path" 2>/dev/null | head -300 + if [ "$total" -gt 300 ]; then + echo "── shown 300 / $total total matches — narrow your pattern, or use bash_exec for counts ──" + fi +} + +tool_glob_files() { + local pattern="$1"; local path="${2:-.}" + if [ ! -d "$path" ]; then echo "ERROR: not a directory: $path"; return; fi + local all; all=$(find "$path" -type f -name "$pattern" 2>/dev/null) + local total; total=$(printf '%s\n' "$all" | grep -c .) + printf '%s\n' "$all" | head -300 + if [ "$total" -gt 300 ]; then + echo "── shown 300 / $total total entries — narrow your pattern ──" + fi +} + +tool_write_file() { + local path="$1"; local content="$2" + local exists="no"; [ -f "$path" ] && exists="yes" + printf '\n%s══ write_file ══%s\n' "$C_YELLOW" "$C_RESET" >&2 + printf ' path: %s\n' "$path" >&2 + printf ' exists: %s\n' "$exists" >&2 + printf ' bytes: %d\n' "${#content}" >&2 + if [ "$exists" = "yes" ]; then + local tmp; tmp=$(mktemp) + printf '%s' "$content" > "$tmp" + printf '%s── diff ──%s\n' "$C_DIM" "$C_RESET" >&2 + diff -u "$path" "$tmp" >&2 || true + rm -f "$tmp" + else + printf '%s── new file preview (first 40 lines) ──%s\n' "$C_DIM" "$C_RESET" >&2 + printf '%s' "$content" | head -40 >&2 + printf '\n' >&2 + fi + printf '%sApprove write? [y/N]:%s ' "$C_BOLD" "$C_RESET" >&2 + read -r answer /dev/null + printf '%s' "$content" > "$path" + echo "OK: wrote $(printf '%s' "$content" | wc -l | tr -d ' ') lines to $path" + log_section "write_file $path (approved)"; log_append '```'; log_append "$content"; log_append '```' + else + echo "DENIED by user. No write performed." + log_section "write_file $path (DENIED)" + fi +} + +# ───────────────────────────────────────────────────────────────────────────── +# v3 NetConfig tools — first-class native capabilities for Cloverleaf work. +# Implemented as small bash+awk scripts in lib/ (alongside this file or in +# $LARRY_HOME/lib). They invoke nothing from v1 scripts or v2 .pyz. +# ───────────────────────────────────────────────────────────────────────────── +_resolve_lib_dir() { + local self_dir; self_dir=$(cd "$(dirname "$0")" 2>/dev/null && pwd) + for candidate in "$self_dir/lib" "$LARRY_HOME/lib"; do + [ -d "$candidate" ] && [ -x "$candidate/nc-parse.sh" ] && { echo "$candidate"; return 0; } + done + return 1 +} +LARRY_LIB_DIR="$(_resolve_lib_dir || echo '')" + +_lib_err_if_missing() { + [ -n "$LARRY_LIB_DIR" ] && return 0 + echo "ERROR: lib/ tools not found. Looked in \$(dirname \$0)/lib and \$LARRY_HOME/lib." + echo " Run install-larry.sh or scp the larry-anywhere/lib/ directory next to larry.sh." + return 1 +} + +tool_nc_list_protocols() { + local nc="$1" + _lib_err_if_missing || return + "$LARRY_LIB_DIR/nc-parse.sh" list-protocols "$nc" 2>&1 +} +tool_nc_list_processes() { + local nc="$1" + _lib_err_if_missing || return + "$LARRY_LIB_DIR/nc-parse.sh" list-processes "$nc" 2>&1 +} +tool_nc_protocol_block() { + local nc="$1" name="$2" + _lib_err_if_missing || return + "$LARRY_LIB_DIR/nc-parse.sh" protocol-block "$nc" "$name" 2>&1 +} +tool_nc_protocol_field() { + local nc="$1" name="$2" field="$3" + _lib_err_if_missing || return + "$LARRY_LIB_DIR/nc-parse.sh" protocol-field "$nc" "$name" "$field" 2>&1 +} +tool_nc_protocol_nested() { + local nc="$1" name="$2" path="$3" + _lib_err_if_missing || return + "$LARRY_LIB_DIR/nc-parse.sh" protocol-nested "$nc" "$name" "$path" 2>&1 +} +tool_nc_protocol_summary() { + local nc="$1" filter="${2:-}" + _lib_err_if_missing || return + if [ -n "$filter" ]; then + "$LARRY_LIB_DIR/nc-parse.sh" protocol-summary "$nc" --filter "$filter" 2>&1 + else + "$LARRY_LIB_DIR/nc-parse.sh" protocol-summary "$nc" 2>&1 + fi +} +tool_nc_destinations() { + local nc="$1" name="$2" + _lib_err_if_missing || return + "$LARRY_LIB_DIR/nc-parse.sh" destinations "$nc" "$name" 2>&1 +} +tool_nc_xlate_refs() { + local nc="$1" name="${2:-}" + _lib_err_if_missing || return + "$LARRY_LIB_DIR/nc-parse.sh" xlate-refs "$nc" "$name" 2>&1 +} +tool_nc_find_inbound() { + local nc="$1" mode="${2:-all}" fmt="${3:-tsv}" + _lib_err_if_missing || return + "$LARRY_LIB_DIR/nc-inbound.sh" "$nc" --mode "$mode" --format "$fmt" 2>&1 +} +tool_nc_make_jump() { + local nc="$1" inbound="$2" new_host="$3" jump_port="$4" + local inbound_host="${5:-127.0.0.1}" proc_jump="${6:-server_jump}" encoding="${7:-}" + _lib_err_if_missing || return + local args=(--inbound "$inbound" --new-host "$new_host" --jump-port "$jump_port" \ + --inbound-host "$inbound_host" --process-jump "$proc_jump") + [ -n "$encoding" ] && args+=(--encoding "$encoding") + "$LARRY_LIB_DIR/nc-make-jump.sh" "$nc" "${args[@]}" 2>&1 +} + +tool_nc_sources() { + local nc="$1" name="$2" + _lib_err_if_missing || return + "$LARRY_LIB_DIR/nc-parse.sh" sources "$nc" "$name" 2>&1 +} + +tool_nc_tclproc_refs() { + local nc="$1" name="${2:-}" + _lib_err_if_missing || return + "$LARRY_LIB_DIR/nc-parse.sh" tclproc-refs "$nc" "$name" 2>&1 +} + +tool_hl7_field() { + local message="$1" field_path="$2" + _lib_err_if_missing || return + local tmp; tmp=$(mktemp) + printf '%s' "$message" > "$tmp" + "$LARRY_LIB_DIR/hl7-field.sh" "$field_path" "$tmp" 2>&1 + rm -f "$tmp" +} + +tool_nc_msgs() { + local thread="$1" after="${2:-}" before="${3:-}" mrn_field="${4:-}" mrn_value="${5:-}" + local limit="${6:-10}" format="${7:-text}" sitedir="${8:-${HCISITEDIR:-}}" db_path="${9:-}" + _lib_err_if_missing || return + local args=("$thread" --limit "$limit" --format "$format") + [ -n "$after" ] && args+=(--after "$after") + [ -n "$before" ] && args+=(--before "$before") + [ -n "$sitedir" ] && args+=(--sitedir "$sitedir") + [ -n "$db_path" ] && args+=(--db "$db_path") + if [ -n "$mrn_field" ] && [ -n "$mrn_value" ]; then + args+=(--field "${mrn_field}=${mrn_value}") + fi + "$LARRY_LIB_DIR/nc-msgs.sh" "${args[@]}" 2>&1 +} + +tool_nc_find() { + local mode="$1" query="$2" format="${3:-table}" hciroot="${4:-${HCIROOT:-}}" + _lib_err_if_missing || return + local args=(--format "$format") + [ -n "$hciroot" ] && args+=(--hciroot "$hciroot") + case "$mode" in + name|port|host|process|where|xlate|tclproc) args+=(--"$mode" "$query") ;; + *) echo "ERROR: unknown nc_find mode: $mode"; return 1 ;; + esac + "$LARRY_LIB_DIR/nc-find.sh" "${args[@]}" 2>&1 +} + +tool_nc_insert_protocol() { + local nc="$1" block_text="$2" mode="${3:-end}" anchor="${4:-}" + _lib_err_if_missing || return + local tmp; tmp=$(mktemp) + printf '%s' "$block_text" > "$tmp" + local args=(insert "$nc" "$tmp" --mode "$mode") + [ -n "$anchor" ] && args+=(--anchor "$anchor") + # Inherit LARRY_SESSION_ID from the running session so journal entries group together + LARRY_SESSION_ID="${LARRY_SESSION_ID:-$SESSION_ID}" \ + "$LARRY_LIB_DIR/nc-insert-protocol.sh" "${args[@]}" 2>&1 + local rc=$? + rm -f "$tmp" + return $rc +} + +tool_nc_add_route() { + local nc="$1" protocol_name="$2" route_text="$3" + _lib_err_if_missing || return + local tmp; tmp=$(mktemp) + printf '%s' "$route_text" > "$tmp" + LARRY_SESSION_ID="${LARRY_SESSION_ID:-$SESSION_ID}" \ + "$LARRY_LIB_DIR/nc-insert-protocol.sh" add-route "$nc" "$protocol_name" "$tmp" 2>&1 + local rc=$? + rm -f "$tmp" + return $rc +} + +tool_nc_regression() { + local scope="$1" count="$2" env_a="$3" site_a="$4" env_b="$5" site_b="$6" out_dir="$7" + local route_cmd="${8:-}" ignore="${9:-MSH.7}" phase="${10:-all}" dry_run="${11:-0}" + _lib_err_if_missing || return + local args=(--scope "$scope" --count "$count" --env-a "$env_a" --env-b "$env_b" --out "$out_dir" \ + --ignore "$ignore" --phase "$phase") + [ -n "$site_a" ] && args+=(--site-a "$site_a") + [ -n "$site_b" ] && args+=(--site-b "$site_b") + [ -n "$route_cmd" ] && args+=(--route-test-cmd "$route_cmd") + [ "$dry_run" = "1" ] && args+=(--dry-run) + "$LARRY_LIB_DIR/nc-regression.sh" "${args[@]}" 2>&1 +} + +tool_hl7_diff() { + local left_path="$1" right_path="$2" ignore="${3:-MSH.7}" include="${4:-}" format="${5:-text}" + _lib_err_if_missing || return + local args=() + [ -n "$ignore" ] && args+=(--ignore "$ignore") + [ -n "$include" ] && args+=(--include-fields "$include") + args+=(--format "$format" "$left_path" "$right_path") + "$LARRY_LIB_DIR/hl7-diff.sh" "${args[@]}" 2>&1 +} + +tool_larry_rollback_list() { + local session_filter="${1:-}" + if [ -n "$session_filter" ]; then + "$LARRY_HOME/../larry-rollback.sh" --list --session "$session_filter" 2>&1 \ + || "$LARRY_LIB_DIR/../larry-rollback.sh" --list --session "$session_filter" 2>&1 + else + "$LARRY_HOME/../larry-rollback.sh" --list 2>&1 \ + || "$LARRY_LIB_DIR/../larry-rollback.sh" --list 2>&1 + fi +} + +tool_nc_document() { + local pattern="$1" out_path="${2:-}" hciroot="${3:-${HCIROOT:-}}" + local title="${4:-}" status="${5:-}" poc_internal="${6:-}" poc_vendor="${7:-}" escalation="${8:-}" open_items="${9:-}" notes="${10:-}" + _lib_err_if_missing || return + local args=(--name "$pattern") + [ -n "$hciroot" ] && args+=(--hciroot "$hciroot") + [ -n "$out_path" ] && args+=(--out "$out_path") + [ -n "$title" ] && args+=(--title "$title") + [ -n "$status" ] && args+=(--status "$status") + [ -n "$poc_internal" ] && args+=(--poc-internal "$poc_internal") + [ -n "$poc_vendor" ] && args+=(--poc-vendor "$poc_vendor") + [ -n "$escalation" ] && args+=(--escalation "$escalation") + [ -n "$open_items" ] && args+=(--open-items "$open_items") + [ -n "$notes" ] && args+=(--notes "$notes") + "$LARRY_LIB_DIR/nc-document.sh" "${args[@]}" 2>&1 +} + +tool_bash_exec() { + local cmd="$1" + printf '\n%s══ bash_exec ══%s\n' "$C_YELLOW" "$C_RESET" >&2 + printf '%s$ %s%s\n' "$C_BOLD" "$cmd" "$C_RESET" >&2 + printf '%sRun this command? [y/N]:%s ' "$C_BOLD" "$C_RESET" >&2 + read -r answer &1 | head -500) + echo "$out" + log_section "bash_exec (approved)"; log_append '```'; log_append "$ $cmd"; log_append "$out"; log_append '```' + else + echo "DENIED by user. Command not executed." + log_section "bash_exec DENIED: $cmd" + fi +} + +execute_tool() { + local name="$1"; local input_json="$2" + local J; J() { printf '%s' "$input_json" | jq -r "$1"; } + case "$name" in + read_file) tool_read_file "$(J '.path')" ;; + list_dir) tool_list_dir "$(J '.path // "."')" ;; + grep_files) tool_grep_files "$(J '.pattern')" "$(J '.path // "."')" ;; + glob_files) tool_glob_files "$(J '.pattern')" "$(J '.path // "."')" ;; + write_file) tool_write_file "$(J '.path')" "$(J '.content')" ;; + bash_exec) tool_bash_exec "$(J '.command')" ;; + nc_list_protocols) tool_nc_list_protocols "$(J '.netconfig')" ;; + nc_list_processes) tool_nc_list_processes "$(J '.netconfig')" ;; + nc_protocol_block) tool_nc_protocol_block "$(J '.netconfig')" "$(J '.name')" ;; + nc_protocol_field) tool_nc_protocol_field "$(J '.netconfig')" "$(J '.name')" "$(J '.field')" ;; + nc_protocol_nested) tool_nc_protocol_nested "$(J '.netconfig')" "$(J '.name')" "$(J '.path')" ;; + nc_protocol_summary) tool_nc_protocol_summary "$(J '.netconfig')" "$(J '.filter // ""')" ;; + nc_destinations) tool_nc_destinations "$(J '.netconfig')" "$(J '.name')" ;; + nc_xlate_refs) tool_nc_xlate_refs "$(J '.netconfig')" "$(J '.name // ""')" ;; + nc_find_inbound) tool_nc_find_inbound "$(J '.netconfig')" "$(J '.mode // "all"')" "$(J '.format // "tsv"')" ;; + nc_make_jump) tool_nc_make_jump "$(J '.netconfig')" "$(J '.inbound')" "$(J '.new_host')" "$(J '.jump_port')" \ + "$(J '.inbound_host // "127.0.0.1"')" "$(J '.process_jump // "server_jump"')" "$(J '.encoding // ""')" ;; + nc_sources) tool_nc_sources "$(J '.netconfig')" "$(J '.name')" ;; + nc_tclproc_refs) tool_nc_tclproc_refs "$(J '.netconfig')" "$(J '.name // ""')" ;; + hl7_field) tool_hl7_field "$(J '.message')" "$(J '.field_path')" ;; + nc_msgs) tool_nc_msgs "$(J '.thread')" "$(J '.after // ""')" "$(J '.before // ""')" \ + "$(J '.field // ""')" "$(J '.value // ""')" \ + "$(J '.limit // 10')" "$(J '.format // "text"')" \ + "$(J '.sitedir // ""')" "$(J '.db // ""')" ;; + nc_document) tool_nc_document "$(J '.name')" "$(J '.out // ""')" "$(J '.hciroot // ""')" \ + "$(J '.title // ""')" "$(J '.status // ""')" \ + "$(J '.poc_internal // ""')" "$(J '.poc_vendor // ""')" \ + "$(J '.escalation // ""')" "$(J '.open_items // ""')" \ + "$(J '.notes // ""')" ;; + nc_find) tool_nc_find "$(J '.mode')" "$(J '.query')" "$(J '.format // "table"')" "$(J '.hciroot // ""')" ;; + nc_insert_protocol) tool_nc_insert_protocol "$(J '.netconfig')" "$(J '.block')" "$(J '.mode // "end"')" "$(J '.anchor // ""')" ;; + nc_add_route) tool_nc_add_route "$(J '.netconfig')" "$(J '.protocol_name')" "$(J '.route')" ;; + hl7_diff) tool_hl7_diff "$(J '.left')" "$(J '.right')" "$(J '.ignore // "MSH.7"')" "$(J '.include // ""')" "$(J '.format // "text"')" ;; + nc_regression) tool_nc_regression "$(J '.scope')" "$(J '.count // 10')" "$(J '.env_a')" "$(J '.site_a // ""')" \ + "$(J '.env_b')" "$(J '.site_b // ""')" "$(J '.out')" \ + "$(J '.route_test_cmd // ""')" "$(J '.ignore // "MSH.7"')" \ + "$(J '.phase // "all"')" "$(J '.dry_run // 0' | sed "s/false/0/;s/true/1/")" ;; + larry_rollback_list) tool_larry_rollback_list "$(J '.session // ""')" ;; + *) echo "ERROR: unknown tool: $name" ;; + esac +} + +# ───────────────────────────────────────────────────────────────────────────── +# Tool schema for the API +# ───────────────────────────────────────────────────────────────────────────── +TOOLS_JSON='[ + {"name":"read_file","description":"Read a single regular file. Returns content with line numbers. Max 250KB; use grep_files for larger.","input_schema":{"type":"object","properties":{"path":{"type":"string","description":"Path to file (absolute or relative to cwd)."}},"required":["path"]}}, + {"name":"list_dir","description":"List a directory (ls -la). Use to map a Cloverleaf site_root.","input_schema":{"type":"object","properties":{"path":{"type":"string","description":"Directory path. Defaults to current dir."}},"required":["path"]}}, + {"name":"grep_files","description":"Recursive grep across files. Use for finding TCL procs, UPOC declarations, segment references, etc. Returns up to 300 matching lines with file:line:content.","input_schema":{"type":"object","properties":{"pattern":{"type":"string","description":"Regex pattern (grep -E style)."},"path":{"type":"string","description":"Starting directory."}},"required":["pattern","path"]}}, + {"name":"glob_files","description":"Find files by name pattern. Up to 300 paths.","input_schema":{"type":"object","properties":{"pattern":{"type":"string","description":"Shell glob like *.tcl or *Inbound*"},"path":{"type":"string","description":"Starting directory."}},"required":["pattern","path"]}}, + {"name":"write_file","description":"Write content to a path. ALWAYS prompts Bryan for Y/N before writing. Shows a unified diff if file exists, or a preview if new.","input_schema":{"type":"object","properties":{"path":{"type":"string"},"content":{"type":"string"}},"required":["path","content"]}}, + {"name":"bash_exec","description":"Run a shell command. ALWAYS prompts Bryan for Y/N before running. Output capped at 500 lines.","input_schema":{"type":"object","properties":{"command":{"type":"string","description":"Single command line, passed to bash -c."}},"required":["command"]}}, + + {"name":"nc_list_protocols","description":"List every protocol (thread) declared in a Cloverleaf NetConfig file. Native v3 parser — does not invoke v1/v2 wrappers. One name per line.","input_schema":{"type":"object","properties":{"netconfig":{"type":"string","description":"Absolute path to a NetConfig file, e.g. $HCISITEDIR/NetConfig."}},"required":["netconfig"]}}, + {"name":"nc_list_processes","description":"List every process declared in a NetConfig. One name per line.","input_schema":{"type":"object","properties":{"netconfig":{"type":"string"}},"required":["netconfig"]}}, + {"name":"nc_protocol_block","description":"Return the full TCL block for one protocol (everything between `protocol NAME {` and the matching `}`). Use to inspect every field of a thread.","input_schema":{"type":"object","properties":{"netconfig":{"type":"string"},"name":{"type":"string","description":"Protocol name, e.g. IB_ADT_muxS."}},"required":["netconfig","name"]}}, + {"name":"nc_protocol_field","description":"Get a top-level field value from a protocol block (e.g. PROCESSNAME, OBWORKASIB, OUTBOUNDONLY, GROUPS, ENCODING, ICLSERVERPORT, AUTOSTART, HOSTDOWN).","input_schema":{"type":"object","properties":{"netconfig":{"type":"string"},"name":{"type":"string"},"field":{"type":"string","description":"Field name, e.g. PROCESSNAME"}},"required":["netconfig","name","field"]}}, + {"name":"nc_protocol_nested","description":"Drill into a nested block via dotted path. Use PROTOCOL.TYPE / PROTOCOL.HOST / PROTOCOL.PORT / PROTOCOL.ISSERVER for connection details — those live inside the inner PROTOCOL{} block, NOT at top level.","input_schema":{"type":"object","properties":{"netconfig":{"type":"string"},"name":{"type":"string"},"path":{"type":"string","description":"Dotted path, e.g. PROTOCOL.PORT"}},"required":["netconfig","name","path"]}}, + {"name":"nc_protocol_summary","description":"Compact TSV summary of all protocols with direction-relevant fields (name, process, direction, port, host, type, isserver, outonly, obworkasib, iclserverport). Optional --filter regex to narrow.","input_schema":{"type":"object","properties":{"netconfig":{"type":"string"},"filter":{"type":"string","description":"Optional regex to filter protocol names."}},"required":["netconfig"]}}, + {"name":"nc_destinations","description":"List every DEST routed to from one protocol’s DATAXLATE block. Unique, sorted.","input_schema":{"type":"object","properties":{"netconfig":{"type":"string"},"name":{"type":"string"}},"required":["netconfig","name"]}}, + {"name":"nc_xlate_refs","description":"List every .xlt file referenced in the NetConfig (all of them, or scoped to one protocol if `name` is provided).","input_schema":{"type":"object","properties":{"netconfig":{"type":"string"},"name":{"type":"string","description":"Optional. Limits to one protocol."}},"required":["netconfig"]}}, + {"name":"nc_find_inbound","description":"Find inbound threads in a NetConfig. mode=tcp-listen (ISSERVER=1, directly fed by upstream client systems), mode=icl-or-file (OBWORKASIB=1, fed by internal Cloverleaf link or file drop), mode=all (default). Output formats: tsv, jsonl, table.","input_schema":{"type":"object","properties":{"netconfig":{"type":"string"},"mode":{"type":"string","enum":["tcp-listen","icl-or-file","all"],"description":"Which class of inbound to return."},"format":{"type":"string","enum":["tsv","jsonl","table"]}},"required":["netconfig"]}}, + {"name":"nc_make_jump","description":"Generate the 3-thread jump set for the cross-environment data replay pattern Bryan uses. Emits FOUR artifacts: (1) linux__out for OLD env (outbound tcpip-client to new linux:jump_port), (2) windows__in for NEW env server_jump site (inbound tcpip-server listening on jump_port, routes internally to #3), (3) windows__out for NEW env server_jump site (outbound tcpip-client to 127.0.0.1:, where orig_port is the existing inbound listening port read from the NetConfig), (4) route-add snippet to splice into the OLD inbound DATAXLATE block. Tag = inbound thread name (auto). The NEW env existing inbound is left COMPLETELY UNCHANGED. Pure generation; caller uses write_file (Y/N) to persist.","input_schema":{"type":"object","properties":{"netconfig":{"type":"string","description":"NetConfig path containing the inbound thread (OLD env)."},"inbound":{"type":"string","description":"Existing inbound protocol name to mirror. Must be a TCP-listener (ISSERVER=1); read its PROTOCOL.PORT first to confirm."},"new_host":{"type":"string","description":"Hostname/IP of the NEW linux env that OLD will TCP to."},"jump_port":{"type":"string","description":"TCP port for the OLD to NEW hop. linux__out targets it, windows__in listens on it."},"inbound_host":{"type":"string","description":"Host that windows__out connects to on NEW (the existing inbound on NEW). Default 127.0.0.1 (same box, loopback)."},"process_jump":{"type":"string","description":"Process for NEW-side threads on server_jump. Default server_jump."},"encoding":{"type":"string","description":"ENCODING override. Default = same as the existing inbound."}},"required":["netconfig","inbound","new_host","jump_port"]}}, + + {"name":"nc_sources","description":"List every protocol that has a DATAXLATE DEST routing to the named thread. The inverse of nc_destinations. Use this to find what feeds a given thread.","input_schema":{"type":"object","properties":{"netconfig":{"type":"string"},"name":{"type":"string","description":"Target thread name."}},"required":["netconfig","name"]}}, + {"name":"nc_tclproc_refs","description":"List every TCL proc name referenced from a protocol block (or from the whole NetConfig if name is omitted). Pulls from DATAFORMAT.PROC, PREPROCS.PROCS, POSTPROCS.PROCS, etc. Unique sorted.","input_schema":{"type":"object","properties":{"netconfig":{"type":"string"},"name":{"type":"string","description":"Optional. Scope to one protocol."}},"required":["netconfig"]}}, + {"name":"hl7_field","description":"Extract a specific HL7 v2 field from a message. field_path = SEG[.FIELD[.COMPONENT[.SUBCOMPONENT]]]. Examples: PID.3 (MRN), PID.18 (account number), MSH.7 (timestamp), MSH.9.2 (event code, like A08), PID.5 (patient name with components). Multiple repetitions are returned one per line. Native v3, no v1/v2 dependency.","input_schema":{"type":"object","properties":{"message":{"type":"string","description":"Raw HL7 message text. Segments separated by \\r."},"field_path":{"type":"string","description":"Field path like PID.3 or MSH.9.2"}},"required":["message","field_path"]}}, + {"name":"nc_msgs","description":"Query Cloverleaf smat (SQLite!) databases for messages from a thread. Filters: time range, exact HL7 field match. Native v3 — reads smatdb directly with sqlite3 -ascii, no hcidbdump/dbExtract needed. Format text shows messages line-by-line with metadata; count returns just the count; json returns structured data.","input_schema":{"type":"object","properties":{"thread":{"type":"string","description":"Thread name. The .smatdb file under $HCISITEDIR/exec/processes/*/.smatdb is auto-located unless db is given."},"after":{"type":"string","description":"Time-after filter. Accepts \\"3 days ago\\", \\"2026-05-20 14:30:00\\", \\"2026-05-20\\", or a unix timestamp."},"before":{"type":"string","description":"Time-before filter, same formats as after."},"field":{"type":"string","description":"HL7 field path for exact-match filter, e.g. PID.18 or MSH.10."},"value":{"type":"string","description":"Value the field must equal. Use with field. Repeatable filters not supported via this single tool call — chain calls if you need multi-field AND."},"limit":{"type":"integer","description":"Max messages to return. Default 10."},"format":{"type":"string","enum":["text","json","count","raw"],"description":"text = human-readable with metadata; count = just the number; json = structured; raw = raw bytes separated by 0x1c."},"sitedir":{"type":"string","description":"Override $HCISITEDIR for thread-to-db location."},"db":{"type":"string","description":"Explicit .smatdb path; overrides auto-locate."}},"required":["thread"]}}, + {"name":"nc_document","description":"Generate a complete markdown knowledge entry for a Cloverleaf subsystem identified by a name pattern. Walks every NetConfig under $HCIROOT, gathers config + sources + destinations + xlates + tclprocs for every matching thread, composes a markdown doc with placeholder context sections (Vendor POC, Internal Owner, Status, Escalation, Open items, Notes). Returns the doc text and (if out is given) writes it to that path.","input_schema":{"type":"object","properties":{"name":{"type":"string","description":"Case-insensitive substring/regex to match protocol names. e.g. 'codametrix', 'epic_adt', '3M'."},"out":{"type":"string","description":"Optional output file path. Convention: $LARRY_HOME/knowledge/.md."},"hciroot":{"type":"string","description":"Override $HCIROOT for the NetConfig scan."},"title":{"type":"string","description":"Doc title. Default derived from name."},"status":{"type":"string","description":"System status fill-in (production/test/decommissioning/...)."},"poc_internal":{"type":"string","description":"Internal owner fill-in."},"poc_vendor":{"type":"string","description":"Vendor POC fill-in."},"escalation":{"type":"string","description":"Escalation path fill-in."},"open_items":{"type":"string","description":"Open items / known issues fill-in. Can be multi-line, will be inserted as-is."},"notes":{"type":"string","description":"Freeform notes fill-in."}},"required":["name"]}}, + + {"name":"nc_find","description":"Cross-site thread search. Native v3 replacement for v1 tbn/tbp/tbh/tbpr/where. Walks every NetConfig under $HCIROOT and returns matching threads with site, port, host, process, direction, file, line. Modes: name=partial name match (like tbn); port=exact port (like tbp); host=substring on host (like tbh); process=substring on PROCESSNAME (like tbpr); where=exact name match across all sites (like the v1 ` where`); xlate=threads referencing a specific .xlt; tclproc=threads referencing a specific TCL proc.","input_schema":{"type":"object","properties":{"mode":{"type":"string","enum":["name","port","host","process","where","xlate","tclproc"],"description":"Search mode."},"query":{"type":"string","description":"Query value: partial name, port number, host substring, process name, exact thread name, xlate filename, or tclproc name."},"format":{"type":"string","enum":["table","tsv","jsonl"],"description":"Output format. Default table."},"hciroot":{"type":"string","description":"Override $HCIROOT."}},"required":["mode","query"]}}, + {"name":"nc_insert_protocol","description":"Insert a new protocol block into a NetConfig file. ALL WRITES GO THROUGH THE JOURNAL — original is snapshotted, diff is saved, the file is atomically replaced. Use larry_rollback_list to view, larry-rollback.sh CLI to undo. mode=end appends; mode=after needs anchor=existing-protocol-name; mode=before needs anchor.","input_schema":{"type":"object","properties":{"netconfig":{"type":"string","description":"Target NetConfig file path."},"block":{"type":"string","description":"The full protocol block text (starting with 'protocol NAME {' and ending with '}'). Get this from nc_make_jump output."},"mode":{"type":"string","enum":["end","after","before"],"description":"Insertion position. Default end."},"anchor":{"type":"string","description":"For mode=after|before: existing protocol name to position relative to."}},"required":["netconfig","block"]}}, + {"name":"nc_add_route","description":"Splice a route entry into an existing protocol's DATAXLATE block. Used to add a new DEST to an inbound's routing (e.g. wiring the OLD inbound to also route to the new linux__out jump thread). ALL WRITES GO THROUGH THE JOURNAL.","input_schema":{"type":"object","properties":{"netconfig":{"type":"string"},"protocol_name":{"type":"string","description":"The existing protocol to modify."},"route":{"type":"string","description":"The route entry text (an inner `{ ... }` object with CACHEMSG, ROUTE_DETAILS, TRXID, etc.). Get from nc_make_jump's route_add output."}},"required":["netconfig","protocol_name","route"]}}, + {"name":"larry_rollback_list","description":"List journal entries — every write that's gone through nc_insert_protocol, nc_add_route, or write_file (once journaled write_file is enabled). Shows session-id, sequence, target, timestamp. Use larry-rollback.sh from the shell to actually roll back.","input_schema":{"type":"object","properties":{"session":{"type":"string","description":"Optional. Limit to one session id."}},"required":[]}}, + + {"name":"hl7_diff","description":"HL7-aware diff between two message files (or multi-message dumps). Compares segment-by-segment, field-by-field, with component and subcomponent precision. Ignores configured fields (default MSH.7 timestamp) so timestamp-only diffs do not show up as noise. Use for regression testing between environments (e.g. test vs prod route-test outputs).","input_schema":{"type":"object","properties":{"left":{"type":"string","description":"Path to left HL7 file."},"right":{"type":"string","description":"Path to right HL7 file."},"ignore":{"type":"string","description":"Comma-separated list of fields to ignore (e.g. MSH.7,MSH.10,EVN.6). Default MSH.7."},"include":{"type":"string","description":"If set, ONLY these fields are compared (overrides ignore for that set)."},"format":{"type":"string","enum":["text","tsv","count"],"description":"text=human-readable diff, tsv=machine-parseable, count=just the difference count."}},"required":["left","right"]}}, + + {"name":"nc_regression","description":"End-to-end regression testing between two Cloverleaf environments. 6 phases: discover inbounds in scope, sample N messages per inbound from env-A smatdbs, run route_test on env-A, run route_test on env-B with same inputs, hl7_diff every paired output file, compile summary report. Phases 3/4 require the Cloverleaf route_test command; pass it via route_test_cmd with placeholders {THREAD} {INPUT} {OUTPUT_DIR} {HCIROOT} {HCISITE}. If route_test_cmd is empty, phases 3/4 are skipped and you can run them manually using the generated input files.","input_schema":{"type":"object","properties":{"scope":{"type":"string","description":"thread:NAME | threads:N1,N2 | site (needs site_a) | server (all sites)"},"count":{"type":"integer","description":"Messages to sample per inbound. Default 10."},"env_a":{"type":"string","description":"HCIROOT of env-A (the test/source env)."},"site_a":{"type":"string","description":"Site name on env-A. Required if scope=site."},"env_b":{"type":"string","description":"HCIROOT of env-B (the prod/target env)."},"site_b":{"type":"string","description":"Site name on env-B."},"out":{"type":"string","description":"Output root directory for inputs, outputs, diffs, and summary."},"route_test_cmd":{"type":"string","description":"Command template for invoking route_test. Use {THREAD} {INPUT} {OUTPUT_DIR} {HCIROOT} {HCISITE} as placeholders."},"ignore":{"type":"string","description":"hl7_diff ignore list. Default MSH.7."},"phase":{"type":"string","enum":["1","2","3","4","5","6","all"],"description":"Run a specific phase or all. Default all."},"dry_run":{"type":"integer","description":"1 = print what would happen, do not execute. Default 0."}},"required":["scope","env_a","env_b","out"]}} +]' + +# ───────────────────────────────────────────────────────────────────────────── +# API call +# ───────────────────────────────────────────────────────────────────────────── +call_api() { + local payload_file="$1" + curl -sS --max-time 180 \ + -H "x-api-key: $ANTHROPIC_API_KEY" \ + -H "anthropic-version: 2023-06-01" \ + -H "content-type: application/json" \ + --data-binary "@$payload_file" \ + "$LARRY_API_URL" +} + +build_system_prompt() { + local sys="" + # Load larry.md first (sets identity), then everything else alphabetically. + if [ -f "$LARRY_HOME/agents/larry.md" ]; then + sys+="$(cat "$LARRY_HOME/agents/larry.md")"$'\n\n' + fi + local f + for f in "$LARRY_HOME/agents/"*.md; do + [ -f "$f" ] || continue + case "$f" in + */larry.md) ;; # already added + *) sys+="$(cat "$f")"$'\n\n' ;; + esac + done + sys+="$CLOVERLEAF_CTX" + printf '%s' "$sys" +} + +# ───────────────────────────────────────────────────────────────────────────── +# Agent turn — loop until stop_reason != tool_use +# ───────────────────────────────────────────────────────────────────────────── +agent_turn() { + local system_prompt="$1" + while true; do + local payload_file; payload_file=$(mktemp) + jq -n \ + --arg model "$LARRY_MODEL" \ + --argjson max_tokens "$LARRY_MAX_TOKENS" \ + --arg system "$system_prompt" \ + --slurpfile messages "$MESSAGES_FILE" \ + --argjson tools "$TOOLS_JSON" \ + '{model:$model, max_tokens:$max_tokens, system:$system, messages:$messages[0], tools:$tools}' \ + > "$payload_file" + + local resp; resp=$(call_api "$payload_file") + rm -f "$payload_file" + + if [ -z "$resp" ]; then err "empty response from API (timeout or network?)"; return 1; fi + + local err_type; err_type=$(printf '%s' "$resp" | jq -r '.error.type // empty' 2>/dev/null) + if [ -n "$err_type" ]; then + err "API error: $err_type — $(printf '%s' "$resp" | jq -r '.error.message // "no message"')" + return 1 + fi + + local blocks; blocks=$(printf '%s' "$resp" | jq -c '.content') + add_assistant_blocks "$blocks" + + # Print text blocks + printf '%s' "$resp" | jq -r '.content[] | select(.type=="text") | .text' \ + | sed "s/^/${C_MAGENTA}/; s/\$/${C_RESET}/" 2>/dev/null \ + || printf '%s' "$resp" | jq -r '.content[] | select(.type=="text") | .text' + + # Log assistant text to session log + { + log_section "assistant" + printf '%s' "$resp" | jq -r '.content[] | select(.type=="text") | .text' >> "$LOG_FILE" + } + + local stop; stop=$(printf '%s' "$resp" | jq -r '.stop_reason // empty') + if [ "$stop" != "tool_use" ]; then break; fi + + # Process tool uses + local results='[]' + while IFS= read -r tool_use; do + [ -z "$tool_use" ] && continue + local tu_id name input_json + tu_id=$(printf '%s' "$tool_use" | jq -r '.id') + name=$(printf '%s' "$tool_use" | jq -r '.name') + input_json=$(printf '%s' "$tool_use" | jq -c '.input') + + printf '\n%s▶ %s%s %s\n' "$C_CYAN" "$name" "$C_RESET" "$input_json" >&2 + log_section "tool: $name $(printf '%s' "$input_json" | jq -c .)" + + local result; result=$(execute_tool "$name" "$input_json") + log_append '```'; log_append "$result"; log_append '```' + + results=$(printf '%s' "$results" | jq \ + --arg id "$tu_id" --arg c "$result" \ + '. + [{"type":"tool_result","tool_use_id":$id,"content":$c}]') + done < <(printf '%s' "$resp" | jq -c '.content[] | select(.type=="tool_use")') + + add_user_tool_results "$results" + done +} + +# ───────────────────────────────────────────────────────────────────────────── +# Slash commands and REPL +# ───────────────────────────────────────────────────────────────────────────── +print_help() { + cat < switch model (e.g. /model claude-opus-4-7) + /cd change working directory + /reset clear conversation history + /load load file contents as your next user message + /sys print the active system prompt + /env print detected Cloverleaf env (HCIROOT, HCISITE, tools) + /redetect re-scan for HCIROOT/HCISITE/tools + /sites list site dirs under HCIROOT + /site switch HCISITE for this session + /pwd show current working directory + /help this help + +Multi-line input: start with '<<' on its own line, end with 'EOF' on its own line. +EOF +} + +read_user_input() { + # Returns user input via global LARRY_INPUT. + # If first line is "<<", read until line "EOF" (heredoc-style). + LARRY_INPUT="" + local first; IFS= read -r first || return 1 + if [ "$first" = "<<" ]; then + local line + while IFS= read -r line; do + [ "$line" = "EOF" ] && break + LARRY_INPUT+="$line"$'\n' + done + else + LARRY_INPUT="$first" + fi +} + +main_loop() { + local system_prompt; system_prompt=$(build_system_prompt) + + if [ -n "$ARG_DIR" ]; then + if [ -d "$ARG_DIR" ]; then + cd "$ARG_DIR" + larry_say "Working dir: $(pwd)" + else + warn "arg is not a directory, ignoring: $ARG_DIR" + fi + fi + + larry_say "Larry-Anywhere v$LARRY_VERSION ready. Model: $LARRY_MODEL." + larry_say "Type your message and press Enter. Use '<<' alone on a line to start multi-line (end with 'EOF'). /help for commands." + echo "" + + while true; do + printf '%syou>%s ' "$C_GREEN" "$C_RESET" + if ! read_user_input; then + echo ""; break + fi + local input="$LARRY_INPUT" + [ -z "$input" ] && continue + + case "$input" in + /quit|/exit|/q) larry_say "bye."; break ;; + /help) print_help; continue ;; + /sys) printf '%s\n' "$system_prompt"; continue ;; + /pwd) echo "$(pwd)"; continue ;; + /env) printf '%s\n' "$CLOVERLEAF_CTX"; continue ;; + /redetect) detect_cloverleaf_env + system_prompt=$(build_system_prompt) + larry_say "re-detected. /env to view." + continue ;; + /sites) if [ -n "${HCIROOT:-}" ] && [ -d "$HCIROOT" ]; then + if command -v sites >/dev/null 2>&1; then sites; else + find "$HCIROOT" -mindepth 1 -maxdepth 1 -type d -exec basename {} \; \ + | grep -Ev '^(archiving|master|lib|tcl|server|client|clgui|cchgs|Alerts|AppDefaults|Tables|backup.*)$' | sort + fi + else err "HCIROOT not set"; fi + continue ;; + /site\ *) HCISITE="${input#/site }"; HCISITEDIR="$HCIROOT/$HCISITE" + export HCISITE HCISITEDIR + detect_cloverleaf_env + system_prompt=$(build_system_prompt) + larry_say "HCISITE -> $HCISITE ($HCISITEDIR)"; continue ;; + /reset) printf '[]' > "$MESSAGES_FILE"; larry_say "history cleared."; continue ;; + /model\ *) LARRY_MODEL="${input#/model }"; larry_say "model -> $LARRY_MODEL"; continue ;; + /cd\ *) local target="${input#/cd }" + if cd "$target" 2>/dev/null; then larry_say "cd -> $(pwd)"; else err "no such directory: $target"; fi + continue ;; + /load\ *) local f="${input#/load }" + if [ ! -f "$f" ]; then err "no such file: $f"; continue; fi + input="$(cat "$f")" + larry_say "loaded $(wc -l < "$f" | tr -d ' ') lines from $f as your next message" ;; + /*) err "unknown command: $input (try /help)"; continue ;; + esac + + log_section "user"; log_append "$input" + add_user_text "$input" + agent_turn "$system_prompt" || warn "turn ended with error" + echo "" + done + + log_section "session-end" + log_append "- end: $(date -Iseconds 2>/dev/null || date)" + larry_say "session log: $LOG_FILE" +} + +main_loop diff --git a/lib/hl7-diff.sh b/lib/hl7-diff.sh new file mode 100755 index 0000000..d372c4e --- /dev/null +++ b/lib/hl7-diff.sh @@ -0,0 +1,248 @@ +#!/usr/bin/env bash +# hl7-diff.sh — HL7-aware diff with field-level normalization. +# Compares two HL7 message files (or multi-message files), segment-by-segment, +# field-by-field. Lets you ignore fields that always change (default: MSH.7 +# timestamp) without losing meaningful differences. +# +# Usage: +# hl7-diff.sh [--ignore "FIELD,FIELD,..."] [--include-fields "FIELD,..."] +# [--separator SEP] [--format text|tsv|count] +# +# +# Multi-message handling: +# Files may contain multiple HL7 messages separated by either: +# - the file separator byte 0x1c (the nc_msgs --format raw default), OR +# - blank lines between MSH-starting blocks (legacy). +# The tool auto-detects. +# +# Defaults: +# --ignore MSH.7 +# +# Output (text format): +# Per-message header, then one line per differing field: +# SEG.FIELD[.COMP[.SUB]] LEFT_VALUE RIGHT_VALUE +# +# Exit codes: 0 identical (post-ignore), 1 differences found, 2 input error +set -o pipefail + +NC_SELF="$0" +LIB_DIR="$(cd "$(dirname "$NC_SELF")" && pwd)" + +die() { printf 'hl7-diff: %s\n' "$*" >&2; exit 2; } + +IGNORE="MSH.7" +INCLUDE="" +FORMAT="text" +LEFT="" +RIGHT="" + +while [ $# -gt 0 ]; do + case "$1" in + --ignore) shift; IGNORE="$1" ;; + --include-fields) shift; INCLUDE="$1" ;; + --format) shift; FORMAT="$1" ;; + -h|--help) sed -n '2,25p' "$NC_SELF"; exit 0 ;; + -*) die "unknown flag: $1" ;; + *) if [ -z "$LEFT" ]; then LEFT="$1" + elif [ -z "$RIGHT" ]; then RIGHT="$1" + else die "extra arg: $1"; fi ;; + esac + shift +done + +[ -f "$LEFT" ] || die "no such left file: $LEFT" +[ -f "$RIGHT" ] || die "no such right file: $RIGHT" +case "$FORMAT" in text|tsv|count) ;; *) die "bad --format" ;; esac + +# Split a file into individual messages. Each MSH-block becomes one message. +# Output: one message per line, with \r preserved as 0x0d between segments, +# messages separated by \x1e (record sep). Returns a path. +split_messages() { + local infile="$1" outfile="$2" + # Try splitting by 0x1c first (raw nc_msgs format) + if grep -q $'\x1c' "$infile" 2>/dev/null; then + awk -v RS=$'\x1c' 'NF>0 || $0!="" {gsub(/\n$/,""); printf "%s\x1e", $0}' "$infile" > "$outfile" + else + # Fallback: each `MSH|` starts a new message; everything until next MSH is one message + awk ' + /^MSH\|/ { + if (msg != "") printf "%s\x1e", msg + msg = $0 + next + } + { + if (msg != "") msg = msg "\r" $0 + else msg = $0 + } + END { + if (msg != "") printf "%s\x1e", msg + } + ' "$infile" > "$outfile" + fi +} + +# Build awk script that does the comparison. We feed it both message lists and +# the ignore/include lists. +TMP_L=$(mktemp); TMP_R=$(mktemp) +trap 'rm -f "$TMP_L" "$TMP_R"' EXIT + +split_messages "$LEFT" "$TMP_L" +split_messages "$RIGHT" "$TMP_R" + +awk -v IGNORE="$IGNORE" -v INCLUDE="$INCLUDE" -v FMT="$FORMAT" \ + -v LFILE="$LEFT" -v RFILE="$RIGHT" ' + function ignored(seg, field, comp, subc, key, key2) { + if (INCLUDE != "") { + # Inclusion mode: only fields in INCLUDE are checked + key = seg "." field + if (comp != "") key = key "." comp + if (subc != "") key = key "." subc + key2 = seg "." field + # Match if exact or a prefix in include list + n = split(INCLUDE, arr, ",") + for (i=1; i<=n; i++) { + ent = arr[i] + if (ent == key || ent == key2) return 0 + } + return 1 + } + key = seg "." field + n = split(IGNORE, arr, ",") + for (i=1; i<=n; i++) { + ent = arr[i] + if (ent == key) return 1 + if (ent == seg "." field "." comp) return 1 + if (ent == seg "." field "." comp "." subc) return 1 + } + return 0 + } + + function parse_msg(msg, out_segs, n, i, seg_name, raw_segs) { + delete out_segs + n = split(msg, raw_segs, "\r") + for (i=1; i<=n; i++) { + if (raw_segs[i] == "") continue + seg_name = substr(raw_segs[i], 1, 3) + out_segs[i] = seg_name "|" raw_segs[i] # prefix with name for easy lookup + } + return n + } + + function compare_field(seg, fidx, lv, rv, msg_idx, n_lc, n_rc, lc, rc, lcomp, rcomp, j, k, n_lsc, n_rsc, lsub, rsub, nm, diffs, ls, rs, ns) { + if (lv == rv) return 0 + # Try component-level if both have ^ + if (lv ~ /\^/ || rv ~ /\^/) { + n_lc = split(lv, lcomp, "^") + n_rc = split(rv, rcomp, "^") + nm = (n_lc > n_rc) ? n_lc : n_rc + diffs = 0 + for (j=1; j<=nm; j++) { + lc = (j <= n_lc) ? lcomp[j] : "" + rc = (j <= n_rc) ? rcomp[j] : "" + if (lc == rc) continue + # subcomponent + if (lc ~ /&/ || rc ~ /&/) { + n_lsc = split(lc, lsub, "&") + n_rsc = split(rc, rsub, "&") + ns = (n_lsc > n_rsc) ? n_lsc : n_rsc + for (k=1; k<=ns; k++) { + ls = (k <= n_lsc) ? lsub[k] : "" + rs = (k <= n_rsc) ? rsub[k] : "" + if (ls != rs && !ignored(seg, fidx, j, k)) { + emit(msg_idx, seg "." fidx "." j "." k, ls, rs) + diffs++ + } + } + } else { + if (!ignored(seg, fidx, j, "")) { + emit(msg_idx, seg "." fidx "." j, lc, rc) + diffs++ + } + } + } + return diffs + } + if (!ignored(seg, fidx, "", "")) { + emit(msg_idx, seg "." fidx, lv, rv) + return 1 + } + return 0 + } + + function emit(msg_idx, path, lv, rv) { + if (FMT == "tsv") printf "%d\t%s\t%s\t%s\n", msg_idx, path, lv, rv + else printf " %-20s %-30s %s\n", path, lv, rv + DIFF_COUNT++ + } + + function diff_segment(seg_name, lseg, rseg, msg_idx, lf, rf, nl, nr, i, base, nmax, lv, rv, field_num) { + nl = split(lseg, lf, "|") + nr = split(rseg, rf, "|") + nmax = (nl > nr) ? nl : nr + # MSH field numbering offset + base = (seg_name == "MSH") ? 0 : 1 + # MSH.1 is the field separator (always "|"); MSH.2 is encoding chars. + # For seg=MSH, index i in array → MSH. for i>=2 corresponds to lf[i] + # For other seg, index i (i>=2) corresponds to SEG.(i-1). + for (i=2; i<=nmax; i++) { + lv = (i <= nl) ? lf[i] : "" + rv = (i <= nr) ? rf[i] : "" + field_num = (seg_name == "MSH") ? i : (i - 1) + compare_field(seg_name, field_num, lv, rv, msg_idx) + } + } + + function diff_message(left_msg, right_msg, msg_idx, l_segs, r_segs, ln, rn, i, l_name, r_name, l, r, mx) { + ln = parse_msg(left_msg, l_segs) + rn = parse_msg(right_msg, r_segs) + mx = (ln > rn) ? ln : rn + for (i=1; i<=mx; i++) { + l = (i <= ln) ? l_segs[i] : "" + r = (i <= rn) ? r_segs[i] : "" + l_name = (l != "") ? substr(l, 1, 3) : "(none)" + r_name = (r != "") ? substr(r, 1, 3) : "(none)" + if (l_name != r_name) { + if (l == "" && r == "") continue # both ends padded — not a real diff + emit(msg_idx, "SEGMENT_ORDER", l_name, r_name) + continue + } + if (l_name == "(none)") continue + # strip "NAME|" prefix we added in parse_msg + sub(/^[A-Z0-9]+\|/, "", l) + sub(/^[A-Z0-9]+\|/, "", r) + diff_segment(l_name, l, r, msg_idx) + } + } + + BEGIN { DIFF_COUNT = 0; n_l = 0; n_r = 0 } + + FNR == NR { L_MSGS[++n_l] = $0; next } + { R_MSGS[++n_r] = $0 } + + END { + if (FMT == "count") { print DIFF_COUNT; exit } + nm = (n_l > n_r) ? n_l : n_r + if (FMT == "text") { + printf "HL7 diff:\n left: %s (%d messages)\n right: %s (%d messages)\n ignore: %s\n", LFILE, n_l, RFILE, n_r, IGNORE + if (INCLUDE != "") printf " include-only: %s\n", INCLUDE + printf "\n" + } + if (n_l != n_r) { + if (FMT == "tsv") printf "0\tMESSAGE_COUNT\t%d\t%d\n", n_l, n_r + else printf " MESSAGE COUNT mismatch: %d vs %d\n", n_l, n_r + DIFF_COUNT++ + } + for (i=1; i<=nm; i++) { + lm = (i <= n_l) ? L_MSGS[i] : "" + rm = (i <= n_r) ? R_MSGS[i] : "" + if (lm == "" || rm == "") { + emit(i, "MESSAGE_PRESENCE", (lm != "" ? "present" : "missing"), (rm != "" ? "present" : "missing")) + continue + } + if (FMT == "text" && (i == 1 || DIFF_COUNT > 0)) printf "----- message %d -----\n", i + diff_message(lm, rm, i) + } + if (FMT == "text") printf "\n%d total field difference(s)\n", DIFF_COUNT + exit (DIFF_COUNT > 0 ? 1 : 0) + } +' RS=$'\x1e' "$TMP_L" "$TMP_R" diff --git a/lib/hl7-field.sh b/lib/hl7-field.sh new file mode 100755 index 0000000..59d95be --- /dev/null +++ b/lib/hl7-field.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +# hl7-field.sh — extract a specific field from an HL7 v2 message. Native v3. +# +# Field path: SEG[.FIELD[.COMPONENT[.SUBCOMPONENT]]] +# PID — return the whole PID segment +# PID.3 — return PID field 3 +# PID.3.1 — return PID field 3, component 1 +# PID.3.1.1 — return PID field 3, component 1, subcomponent 1 +# MSH.10 — special: MSH numbering accounts for the encoding chars +# (MSH.1 = field separator char, MSH.2 = encoding chars, +# MSH.3+ = subsequent fields). +# +# Repetitions (~ separator) are returned one per line. +# +# Usage: +# hl7-field.sh [message_file] # read message from file or stdin +# echo "$msg" | hl7-field.sh PID.18 +# hl7-field.sh PID.18 /tmp/sample.hl7 +# +# Exit codes: 0 = found (any number of values printed), 2 = bad path, 3 = not found. +set -u + +usage() { sed -n '2,20p' "$0"; exit 0; } + +PATH_SPEC="${1:-}" +MSG_FILE="${2:-}" +[ -n "$PATH_SPEC" ] || { usage >&2; exit 2; } +case "$PATH_SPEC" in -h|--help) usage ;; esac + +# Read message bytes +if [ -n "$MSG_FILE" ]; then + [ -f "$MSG_FILE" ] || { echo "hl7-field: no such file: $MSG_FILE" >&2; exit 2; } + MSG=$(cat "$MSG_FILE") +else + MSG=$(cat) +fi +[ -n "$MSG" ] || { echo "hl7-field: empty message" >&2; exit 3; } + +# Parse path: SEG, optional .FIELD, .COMPONENT, .SUBCOMPONENT +IFS='.' read -r SEG FNUM CNUM SCNUM <<< "$PATH_SPEC" +[ -n "$SEG" ] || { echo "hl7-field: bad path: $PATH_SPEC" >&2; exit 2; } + +# Detect encoding characters from MSH +# Standard layout: MSH^~\&... where F is the field-separator (usually |) +# We need the field, component, subcomponent, repetition separators. +FSEP=$(printf '%s' "$MSG" | head -c 4 | cut -c4) # 4th char of MSH segment = field sep +ECH=$(printf '%s' "$MSG" | awk -v FS="$FSEP" '/^MSH/{print $2; exit}') +CSEP="${ECH:0:1}" # ^ — component separator +RSEP="${ECH:1:1}" # ~ — repetition separator +ESC="${ECH:2:1}" # \ — escape character (unused in lookup) +SCSEP="${ECH:3:1}" # & — subcomponent separator +[ -z "$FSEP" ] && FSEP='|' +[ -z "$CSEP" ] && CSEP='^' +[ -z "$RSEP" ] && RSEP='~' +[ -z "$SCSEP" ] && SCSEP='&' + +# Find the requested segment. Segments are separated by \r (\x0d). +# Walk segments, emit when SEG matches. +SEGMENT=$(printf '%s' "$MSG" | awk -v RS=$'\r' -v SEG="$SEG" ' + $0 ~ ("^" SEG "($|[" FS "])") { print; found=1; exit } + BEGIN { FS="\t" } # value irrelevant — we match the whole record +' 2>/dev/null) + +if [ -z "$SEGMENT" ]; then + # Fall back: split by \r in shell (POSIX) + SEGMENT=$(printf '%s' "$MSG" | tr '\r' '\n' | grep -m1 "^${SEG}[${FSEP}\$]" || true) +fi + +[ -n "$SEGMENT" ] || exit 3 + +# If only segment requested, emit and exit +if [ -z "${FNUM:-}" ]; then + printf '%s\n' "$SEGMENT"; exit 0 +fi + +# Split segment by field separator into array, with MSH special-case +# MSH.1 = the field separator character itself (e.g. "|"). +# MSH.2 = encoding chars (e.g. "^~\&"). +# MSH.N (N>=3) = field at array index (N-1). +# Non-MSH: SEG.N = field at array index N. +get_field() { + local seg="$1" fnum="$2" + if [ "$SEG" = "MSH" ]; then + if [ "$fnum" = "1" ]; then + printf '%s' "$FSEP"; return + fi + # awk MSH treatment: $1="MSH", $2=encoding ($1 is "MSH", $2 is ECH) + # MSH.N for N >= 2 is awk index N-1 ... wait, MSH.2 = ECH = $2. + # MSH.3 = first real field after ECH = $3 + # So MSH.N → awk index N for N >= 2. (Yes: MSH.2=$2, MSH.3=$3, ...) + printf '%s' "$seg" | awk -v FS="$FSEP" -v N="$fnum" '{print $N}' + else + # SEG.N → awk index N+1 (because $1 == SEG name, $2 == field 1, etc.) + printf '%s' "$seg" | awk -v FS="$FSEP" -v N="$fnum" '{print $(N+1)}' + fi +} + +FIELD_VAL=$(get_field "$SEGMENT" "$FNUM") + +# Split repetitions +if [ -n "$FIELD_VAL" ]; then + printf '%s' "$FIELD_VAL" | awk -v R="$RSEP" -v C="$CSEP" -v S="$SCSEP" \ + -v CN="${CNUM:-}" -v SCN="${SCNUM:-}" ' + BEGIN { n=split(value, parts, R) } + { value=$0 } + END { + n = split(value, reps, R) + for (i=1; i<=n; i++) { + v = reps[i] + if (CN != "") { + nc = split(v, comps, C) + v = comps[CN] + if (SCN != "") { + nsc = split(v, subs, S) + v = subs[SCN] + } + } + print v + } + } + ' +fi diff --git a/lib/journal.sh b/lib/journal.sh new file mode 100755 index 0000000..d100ebc --- /dev/null +++ b/lib/journal.sh @@ -0,0 +1,181 @@ +#!/usr/bin/env bash +# journal.sh — atomic backup-and-write journal for Larry-Anywhere v3. +# +# Every modification to a file goes through this journal: +# 1. Snapshot the original (if it exists) to a session-scoped backup dir. +# 2. Compute a unified diff between original and new content. +# 3. Append a record to $LARRY_HOME/journal/index.tsv. +# 4. Append a human-readable entry to $LARRY_HOME/journal//manifest.md. +# 5. Atomically write the new content to the target. +# +# Rollback support: `larry-rollback.sh` (sibling script) can restore any subset. +# +# This script is SOURCEABLE — Larry-Anywhere's tool_write_file sources it and +# calls `journal_write`. It can also be invoked directly as a CLI. +# +# CLI: +# journal.sh write # snapshot, diff, write +# journal.sh list [--session S] # list entries +# journal.sh show # print diff for one entry +# journal.sh session-manifest [] # print this/given session's manifest +# +# Env: +# LARRY_HOME (required) where the journal lives +# LARRY_SESSION_ID (optional) defaults to a sortable timestamp +set -u +set -o pipefail + +LARRY_HOME="${LARRY_HOME:-$HOME/.larry}" +LARRY_SESSION_ID="${LARRY_SESSION_ID:-$(date +%Y-%m-%d-%H%M%S)-$$}" + +JOURNAL_ROOT="$LARRY_HOME/journal" +JOURNAL_INDEX="$JOURNAL_ROOT/index.tsv" +SESSION_DIR="$JOURNAL_ROOT/$LARRY_SESSION_ID" +SESSION_FILES="$SESSION_DIR/files" +SESSION_MANIFEST="$SESSION_DIR/manifest.md" + +_journal_init() { + mkdir -p "$SESSION_FILES" 2>/dev/null || return 1 + if [ ! -f "$JOURNAL_INDEX" ]; then + printf 'timestamp\tsession\tentry_id\ttarget\taction\torig_sha256\tnew_sha256\tbackup_path\tdiff_path\n' > "$JOURNAL_INDEX" + fi + if [ ! -f "$SESSION_MANIFEST" ]; then + { + echo "# Larry-Anywhere journal — session $LARRY_SESSION_ID" + echo "- started: $(date -Iseconds 2>/dev/null || date)" + echo "- host: $(hostname 2>/dev/null || echo unknown)" + echo "- larry-version: $(cat "$LARRY_HOME/VERSION" 2>/dev/null || echo unknown)" + echo "" + echo "Every write below has a backup at \`files/NNN_.orig\` and a diff at \`files/NNN_.diff\`." + echo "Roll back with: \`larry-rollback.sh --session $LARRY_SESSION_ID\` (or per-entry)." + echo "" + } > "$SESSION_MANIFEST" + fi +} + +_sha() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | awk '{print $1}' + elif command -v shasum >/dev/null 2>&1; then + shasum -a 256 "$1" | awk '{print $1}' + else + cksum "$1" | awk '{print $1"-"$2}' + fi +} + +_next_seq() { + # Number of *.orig|*.new files in session_files / 2, rounded up + local n + n=$(find "$SESSION_FILES" -maxdepth 1 -name '*.orig' -o -name '*.new' 2>/dev/null | wc -l | tr -d ' ') + printf '%03d' $(( (n / 2) + 1 )) +} + +# Main entry. $1=target file path, $2=path to file containing new content. +# Returns the entry_id on stdout. Prints user-facing diff on stderr. +journal_write() { + local target="$1" + local newfile="$2" + [ -n "$target" ] || { echo "journal_write: missing target" >&2; return 2; } + [ -f "$newfile" ] || { echo "journal_write: new-content file not found: $newfile" >&2; return 2; } + + _journal_init + + local seq base entry_id + seq=$(_next_seq) + base=$(basename "$target") + entry_id="${LARRY_SESSION_ID}/${seq}_${base}" + + local backup="$SESSION_FILES/${seq}_${base}.orig" + local newcopy="$SESSION_FILES/${seq}_${base}.new" + local diffp="$SESSION_FILES/${seq}_${base}.diff" + local action="create" + local orig_sha="" + + if [ -e "$target" ]; then + cp -p "$target" "$backup" || return 3 + orig_sha=$(_sha "$backup") + action="modify" + else + : > "$backup" # placeholder for rollback (empty file) + orig_sha="" + fi + + cp -p "$newfile" "$newcopy" || return 3 + local new_sha; new_sha=$(_sha "$newcopy") + + diff -u "$backup" "$newcopy" > "$diffp" 2>/dev/null || true + + # Atomically write the new content + local target_dir; target_dir=$(dirname "$target") + mkdir -p "$target_dir" 2>/dev/null + cp -p "$newfile" "$target.larry-tmp.$$" && mv -f "$target.larry-tmp.$$" "$target" + + # Append index + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ + "$(date -Iseconds 2>/dev/null || date)" \ + "$LARRY_SESSION_ID" "$seq" "$target" "$action" \ + "$orig_sha" "$new_sha" "$backup" "$diffp" >> "$JOURNAL_INDEX" + + # Append session manifest + { + printf '\n## %s %s `%s`\n' "$seq" "$action" "$target" + printf -- '- orig sha256: %s\n' "${orig_sha:-(none — new file)}" + printf -- '- new sha256: %s\n' "$new_sha" + printf -- '- backup: `files/%s_%s.orig`\n' "$seq" "$base" + printf -- '- diff: `files/%s_%s.diff`\n' "$seq" "$base" + if [ "$action" = "modify" ]; then + printf -- '- change summary: %s lines added, %s lines removed\n' \ + "$(grep -c '^+[^+]' "$diffp" 2>/dev/null || echo 0)" \ + "$(grep -c '^-[^-]' "$diffp" 2>/dev/null || echo 0)" + fi + } >> "$SESSION_MANIFEST" + + # User-facing: print the entry id (small) on stdout + printf '%s\n' "$entry_id" +} + +journal_list() { + local filter_session="" + while [ $# -gt 0 ]; do + case "$1" in --session) shift; filter_session="$1" ;; esac + shift + done + [ -f "$JOURNAL_INDEX" ] || { echo "(no journal yet)"; return 0; } + if [ -n "$filter_session" ]; then + awk -F'\t' -v s="$filter_session" 'NR==1 || $2==s' "$JOURNAL_INDEX" + else + cat "$JOURNAL_INDEX" + fi +} + +journal_show() { + local entry_id="$1" + local session seq + session="${entry_id%/*}" + seq="${entry_id##*/}" + seq="${seq%%_*}" + local f + f=$(find "$JOURNAL_ROOT/$session/files" -maxdepth 1 -name "${seq}_*.diff" 2>/dev/null | head -1) + [ -n "$f" ] || { echo "no such entry: $entry_id" >&2; return 2; } + cat "$f" +} + +journal_session_manifest() { + local session="${1:-$LARRY_SESSION_ID}" + local f="$JOURNAL_ROOT/$session/manifest.md" + [ -f "$f" ] || { echo "no manifest for session $session" >&2; return 2; } + cat "$f" +} + +# CLI dispatch (only when invoked directly, not sourced) +if [ "${BASH_SOURCE[0]:-$0}" = "${0}" ]; then + cmd="${1:-help}" + case "$cmd" in + write) [ $# -ge 3 ] || { echo "usage: $0 write " >&2; exit 2; }; journal_write "$2" "$3" ;; + list) shift; journal_list "$@" ;; + show) [ $# -ge 2 ] || { echo "usage: $0 show " >&2; exit 2; }; journal_show "$2" ;; + session-manifest) journal_session_manifest "${2:-}" ;; + help|-h|--help) sed -n '2,25p' "$0" ;; + *) echo "unknown subcommand: $cmd" >&2; exit 2 ;; + esac +fi diff --git a/lib/nc-diff-interface.sh b/lib/nc-diff-interface.sh new file mode 100755 index 0000000..cc99f47 --- /dev/null +++ b/lib/nc-diff-interface.sh @@ -0,0 +1,284 @@ +#!/usr/bin/env bash +# nc-diff-interface.sh — diff one Cloverleaf interface across two environments. +# +# Use case: "I made a change in test to interface X, forgot what, need to move to prod. +# Tell me exactly what differs." +# +# Compares: +# 1. The protocol block (TCL definition in NetConfig). +# 2. Every xlate (.xlt) file referenced by the protocol. +# 3. Every tclproc (.tcl) file referenced by the protocol. +# 4. (Optional) related tables (.tbl) — references found inside xlates/tclprocs. +# +# Usage: +# nc-diff-interface.sh --interface NAME --left NC_PATH_A --right NC_PATH_B +# [--out PATH] # markdown report (default: stdout) +# [--include-tables] # also diff .tbl files referenced by the xlates/tclprocs +# [--left-label LBL] # e.g. "TEST" +# [--right-label LBL] # e.g. "PROD" +# +# The two NetConfig paths must each be a site root NetConfig — site root is +# `dirname ` and that's where Xlate/, tclprocs/, tables/ are looked up. +set -o pipefail +# Note: not using `set -u` — the connected-cluster traversal uses associative +# arrays whose emptiness shouldn't bash-fault. + +NC_SELF="$0" +LIB_DIR="$(cd "$(dirname "$NC_SELF")" && pwd)" +NCP="$LIB_DIR/nc-parse.sh" + +die() { printf 'nc-diff-interface: %s\n' "$*" >&2; exit 1; } + +INTERFACE="" +NC_A="" +NC_B="" +OUT="" +INCLUDE_TABLES=0 +LABEL_A="A" +LABEL_B="B" +DEPTH=1 # how many hops out from the named interface to also diff + +while [ $# -gt 0 ]; do + case "$1" in + --interface) shift; INTERFACE="$1" ;; + --left) shift; NC_A="$1" ;; + --right) shift; NC_B="$1" ;; + --out) shift; OUT="$1" ;; + --include-tables) INCLUDE_TABLES=1 ;; + --left-label) shift; LABEL_A="$1" ;; + --right-label) shift; LABEL_B="$1" ;; + --depth) shift; DEPTH="$1" ;; + -h|--help) sed -n '2,22p' "$NC_SELF"; exit 0 ;; + -*) die "unknown flag: $1" ;; + *) die "extra arg: $1" ;; + esac + shift +done + +[ -n "$INTERFACE" ] || die "missing --interface NAME" +[ -n "$NC_A" ] || die "missing --left NC_PATH" +[ -n "$NC_B" ] || die "missing --right NC_PATH" +[ -f "$NC_A" ] || die "no such file: $NC_A" +[ -f "$NC_B" ] || die "no such file: $NC_B" +[[ "$DEPTH" =~ ^[0-9]+$ ]] || die "--depth must be a non-negative integer" + +SITE_A="$(dirname "$NC_A")" +SITE_B="$(dirname "$NC_B")" + +out_target() { + if [ -n "$OUT" ]; then mkdir -p "$(dirname "$OUT")" 2>/dev/null; cat > "$OUT" + else cat + fi +} + +# Helper: print a diff between two files in fenced code block, or a clear message +# if one or both are missing. +emit_file_diff() { + local title="$1" a="$2" b="$3" + printf '### %s\n\n' "$title" + if [ ! -e "$a" ] && [ ! -e "$b" ]; then + printf '_(missing on both sides)_\n\n'; return + elif [ ! -e "$a" ]; then + printf '**Only on %s**: `%s`\n\n' "$LABEL_B" "$b" + printf '```\n'; head -50 "$b"; printf '```\n\n'; return + elif [ ! -e "$b" ]; then + printf '**Only on %s**: `%s`\n\n' "$LABEL_A" "$a" + printf '```\n'; head -50 "$a"; printf '```\n\n'; return + fi + local sha_a sha_b + sha_a=$(shasum "$a" 2>/dev/null | awk '{print $1}' || md5 "$a") + sha_b=$(shasum "$b" 2>/dev/null | awk '{print $1}' || md5 "$b") + if [ "$sha_a" = "$sha_b" ]; then + printf '_identical (sha1 `%s`)_\n\n' "$sha_a" + else + printf '%s: `%s` (%s)\n' "$LABEL_A" "$a" "$sha_a" + printf '%s: `%s` (%s)\n\n' "$LABEL_B" "$b" "$sha_b" + printf '```diff\n' + diff -u "$a" "$b" 2>/dev/null || true + printf '```\n\n' + fi +} + +# Same shape but for two strings (in-memory) +emit_text_diff() { + local title="$1" text_a="$2" text_b="$3" + printf '### %s\n\n' "$title" + local ta tb; ta=$(mktemp); tb=$(mktemp) + printf '%s\n' "$text_a" > "$ta" + printf '%s\n' "$text_b" > "$tb" + if cmp -s "$ta" "$tb"; then + printf '_identical_\n\n' + else + printf '```diff\n' + diff -u "$ta" "$tb" 2>/dev/null || true + printf '```\n\n' + fi + rm -f "$ta" "$tb" +} + +# Find referenced files in a NetConfig+site root, for a given interface +collect_xlates() { + local nc="$1" site="$2" + "$NCP" xlate-refs "$nc" "$INTERFACE" 2>/dev/null \ + | awk -v site="$site" '{print site"/Xlate/"$0}' +} +collect_tclprocs() { + local nc="$1" site="$2" + "$NCP" tclproc-refs "$nc" "$INTERFACE" 2>/dev/null \ + | awk -v site="$site" '{print site"/tclprocs/"$0".tcl"}' +} + +# Tables — referenced inside xlates and tclprocs (look for *.tbl references) +collect_table_refs() { + local site="$1" + shift + local files=("$@") + for f in "${files[@]}"; do + [ -f "$f" ] || continue + grep -hoE '[A-Za-z0-9_]+\.tbl' "$f" 2>/dev/null + done | sort -u | awk -v site="$site" '$0 != "" {print site"/tables/"$0}' +} + +# Walk the connected graph N hops out from the named interface, combining +# sources and destinations from BOTH NetConfigs. Result: deduplicated thread set. +build_cluster() { + local depth="$1" + declare -A visited + visited["$INTERFACE"]=1 + local frontier=("$INTERFACE") + local d + for ((d=0; d/dev/null) \ + $("$NCP" destinations "$nc" "$f" 2>/dev/null); do + [ -z "$rel" ] && continue + if [ -z "${visited[$rel]:-}" ]; then + visited[$rel]=1 + next_frontier+=("$rel") + fi + done + done + done + [ ${#next_frontier[@]} -eq 0 ] && break + frontier=("${next_frontier[@]}") + done + printf '%s\n' "${!visited[@]}" | sort +} + +# Diff one thread's protocol block + its xlates + its tclprocs. +emit_thread_section() { + local iface="$1" idx="$2" total="$3" + printf '## [%d/%d] Thread `%s`\n\n' "$idx" "$total" "$iface" + + local BLOCK_A BLOCK_B + BLOCK_A=$("$NCP" protocol-block "$NC_A" "$iface" 2>/dev/null || echo "") + BLOCK_B=$("$NCP" protocol-block "$NC_B" "$iface" 2>/dev/null || echo "") + if [ -z "$BLOCK_A" ] && [ -z "$BLOCK_B" ]; then + printf '_absent on both sides — referenced as DEST but block not present_\n\n' + return + elif [ -z "$BLOCK_A" ]; then + printf '**Only on %s.** Block on %s:\n\n```tcl\n%s\n```\n\n' "$LABEL_B" "$LABEL_B" "$BLOCK_B" + return + elif [ -z "$BLOCK_B" ]; then + printf '**Only on %s.** Block on %s:\n\n```tcl\n%s\n```\n\n' "$LABEL_A" "$LABEL_A" "$BLOCK_A" + return + fi + emit_text_diff "Protocol block" "$BLOCK_A" "$BLOCK_B" + + # Xlates referenced + local X_A=() X_B=() + while IFS= read -r l; do X_A+=("$l"); done < <(collect_xlates_for "$NC_A" "$SITE_A" "$iface") + while IFS= read -r l; do X_B+=("$l"); done < <(collect_xlates_for "$NC_B" "$SITE_B" "$iface") + declare -A XSET + local p + for p in "${X_A[@]}" "${X_B[@]}"; do [ -n "$p" ] && XSET[$(basename "$p")]=1; done + if [ ${#XSET[@]} -gt 0 ]; then + printf '#### Xlates referenced by `%s`\n\n' "$iface" + local x + for x in "${!XSET[@]}"; do + emit_file_diff "Xlate \`$x\`" "$SITE_A/Xlate/$x" "$SITE_B/Xlate/$x" + done + fi + + # Tclprocs referenced + local T_A=() T_B=() + while IFS= read -r l; do T_A+=("$l"); done < <(collect_tclprocs_for "$NC_A" "$SITE_A" "$iface") + while IFS= read -r l; do T_B+=("$l"); done < <(collect_tclprocs_for "$NC_B" "$SITE_B" "$iface") + declare -A TSET + for p in "${T_A[@]}" "${T_B[@]}"; do [ -n "$p" ] && TSET[$(basename "$p")]=1; done + if [ ${#TSET[@]} -gt 0 ]; then + printf '#### Tclprocs referenced by `%s`\n\n' "$iface" + local t + for t in "${!TSET[@]}"; do + emit_file_diff "Tclproc \`$t\`" "$SITE_A/tclprocs/$t" "$SITE_B/tclprocs/$t" + done + fi +} + +# Per-iface collectors (versions of the originals scoped to a specific iface) +collect_xlates_for() { + "$NCP" xlate-refs "$1" "$3" 2>/dev/null | awk -v site="$2" '{print site"/Xlate/"$0}' +} +collect_tclprocs_for() { + "$NCP" tclproc-refs "$1" "$3" 2>/dev/null | awk -v site="$2" '{print site"/tclprocs/"$0".tcl"}' +} + +# ───────────────────────────────────────────────────────────────────────────── +# Compose report +# ───────────────────────────────────────────────────────────────────────────── +{ + printf '# Interface diff: `%s` + connected (depth %d)\n\n' "$INTERFACE" "$DEPTH" + printf '_%s_ → `%s`\n' "$LABEL_A" "$NC_A" + printf '_%s_ → `%s`\n\n' "$LABEL_B" "$NC_B" + + # Build the cluster + CLUSTER=() + while IFS= read -r t; do CLUSTER+=("$t"); done < <(build_cluster "$DEPTH") + TOTAL=${#CLUSTER[@]} + + printf '## Cluster (%d threads)\n\n' "$TOTAL" + printf 'These are the threads that will be diffed, starting from `%s` and walking %d hop(s) outward via sources/destinations on BOTH sides.\n\n' "$INTERFACE" "$DEPTH" + for t in "${CLUSTER[@]}"; do printf -- '- `%s`\n' "$t"; done + printf '\n' + + # Diff each thread in the cluster + IDX=0 + for t in "${CLUSTER[@]}"; do + IDX=$((IDX+1)) + emit_thread_section "$t" "$IDX" "$TOTAL" + done + + # Optional tables section + if [ "$INCLUDE_TABLES" = "1" ]; then + printf '## Tables referenced (across the whole cluster)\n\n' + declare -A TBL_SEEN + for t in "${CLUSTER[@]}"; do + while IFS= read -r f; do + [ -f "$f" ] && for tbl in $(grep -hoE '[A-Za-z0-9_]+\.tbl' "$f" 2>/dev/null); do + TBL_SEEN[$tbl]=1 + done + done < <(collect_xlates_for "$NC_A" "$SITE_A" "$t"; \ + collect_tclprocs_for "$NC_A" "$SITE_A" "$t"; \ + collect_xlates_for "$NC_B" "$SITE_B" "$t"; \ + collect_tclprocs_for "$NC_B" "$SITE_B" "$t") + done + if [ ${#TBL_SEEN[@]} -eq 0 ]; then + printf '_no .tbl references found inside the cluster_\n\n' + else + for tbl in "${!TBL_SEEN[@]}"; do + emit_file_diff "Table \`$tbl\`" "$SITE_A/tables/$tbl" "$SITE_B/tables/$tbl" + done + fi + fi + + printf '---\n\n' + printf '_Generated %s by Larry-Anywhere nc-diff-interface.sh (depth=%d)._\n' \ + "$(date -Iseconds 2>/dev/null || date)" "$DEPTH" +} | out_target + +[ -n "$OUT" ] && printf 'nc-diff-interface: wrote %s\n' "$OUT" >&2 diff --git a/lib/nc-document.sh b/lib/nc-document.sh new file mode 100755 index 0000000..746aa76 --- /dev/null +++ b/lib/nc-document.sh @@ -0,0 +1,232 @@ +#!/usr/bin/env bash +# nc-document.sh — generate a v3 native markdown knowledge entry for a Cloverleaf +# subsystem identified by a name pattern. Walks every NetConfig under $HCIROOT +# (or a passed-in list), gathers config + flow + xlates + tclprocs, composes a +# markdown doc with placeholder context sections for humans to fill. +# +# Usage: +# nc-document.sh --name [options] +# +# --name PATTERN case-insensitive substring/regex to match protocol names +# --hciroot DIR defaults to $HCIROOT +# --netconfigs PATHS colon-separated explicit NetConfig list (overrides --hciroot scan) +# --out PATH output markdown path (default: stdout) +# --title TITLE doc title (default: derived from --name) +# --poc-vendor TXT Vendor POC content +# --poc-internal TXT Internal Owner content +# --status TXT e.g. production / test / decommissioning +# --escalation TXT Escalation path text +# --open-items TXT Open items text (bulleted by you if multi-line) +# --notes TXT freeform additional notes +# +# Any --poc/-status/--escalation/--open-items/--notes that you OMIT becomes an +# empty placeholder section in the doc, ready for someone to fill. +set -u +set -o pipefail + +NC_SELF="$0" +LIB_DIR="$(cd "$(dirname "$NC_SELF")" && pwd)" +NCP="$LIB_DIR/nc-parse.sh" +NCI="$LIB_DIR/nc-inbound.sh" + +die() { printf 'nc-document: %s\n' "$*" >&2; exit 1; } + +PATTERN="" +HCIROOT_OVERRIDE="" +NETCONFIGS_OVERRIDE="" +OUT="" +TITLE="" +POC_VENDOR="" +POC_INTERNAL="" +STATUS="" +ESCALATION="" +OPEN_ITEMS="" +NOTES="" + +while [ $# -gt 0 ]; do + case "$1" in + --name) shift; PATTERN="$1" ;; + --hciroot) shift; HCIROOT_OVERRIDE="$1" ;; + --netconfigs) shift; NETCONFIGS_OVERRIDE="$1" ;; + --out) shift; OUT="$1" ;; + --title) shift; TITLE="$1" ;; + --poc-vendor) shift; POC_VENDOR="$1" ;; + --poc-internal) shift; POC_INTERNAL="$1" ;; + --status) shift; STATUS="$1" ;; + --escalation) shift; ESCALATION="$1" ;; + --open-items) shift; OPEN_ITEMS="$1" ;; + --notes) shift; NOTES="$1" ;; + -h|--help) sed -n '2,25p' "$NC_SELF"; exit 0 ;; + -*) die "unknown flag: $1" ;; + *) die "extra arg: $1" ;; + esac + shift +done + +[ -n "$PATTERN" ] || die "missing --name PATTERN" +[ -z "$TITLE" ] && TITLE="$(printf '%s' "$PATTERN" | tr '[:upper:]' '[:lower:]')" + +# Determine the NetConfig list +NCONFIGS=() +if [ -n "$NETCONFIGS_OVERRIDE" ]; then + IFS=':' read -ra NCONFIGS <<< "$NETCONFIGS_OVERRIDE" +else + ROOT="${HCIROOT_OVERRIDE:-${HCIROOT:-}}" + [ -n "$ROOT" ] || die "no \$HCIROOT and no --hciroot; pass one or set the env var" + [ -d "$ROOT" ] || die "hciroot not a directory: $ROOT" + while IFS= read -r nc; do + NCONFIGS+=("$nc") + done < <(find "$ROOT" -maxdepth 2 -name NetConfig -type f 2>/dev/null) +fi +[ ${#NCONFIGS[@]} -gt 0 ] || die "no NetConfig files found" + +# Emit to OUT or stdout +out_target() { + if [ -n "$OUT" ]; then + mkdir -p "$(dirname "$OUT")" 2>/dev/null + cat > "$OUT" + else + cat + fi +} + +# Gather all matching protocols across all NetConfigs +declare -a MATCHES +for nc in "${NCONFIGS[@]}"; do + site=$(basename "$(dirname "$nc")") + while IFS= read -r prot; do + [ -z "$prot" ] && continue + MATCHES+=("$site|$nc|$prot") + done < <("$NCP" list-protocols "$nc" 2>/dev/null | grep -i -- "$PATTERN" || true) +done + +if [ ${#MATCHES[@]} -eq 0 ]; then + printf 'No protocols matching "%s" found in %d NetConfig(s).\n' "$PATTERN" "${#NCONFIGS[@]}" >&2 + exit 2 +fi + +# ───────────────────────────────────────────────────────────────────────────── +# Compose markdown +# ───────────────────────────────────────────────────────────────────────────── +{ + printf '# %s — Cloverleaf System Knowledge Entry\n\n' "$TITLE" + printf '_Auto-generated by Larry-Anywhere v3 nc-document.sh on %s. Auto-derived facts are below; context fields are for humans to fill or refine._\n\n' "$(date -Iseconds 2>/dev/null || date)" + + printf '## Context\n\n' + printf -- '- **Vendor POC:** %s\n' "${POC_VENDOR:-_(unfilled — add vendor contact name + email/phone)_}" + printf -- '- **Internal Owner:** %s\n' "${POC_INTERNAL:-_(unfilled — add the internal owner / engineer)_}" + printf -- '- **Status:** %s\n' "${STATUS:-_(unfilled — production / test / decommissioning / on hold)_}" + printf -- '- **Escalation:** %s\n' "${ESCALATION:-_(unfilled — on-call path, ticket queue, etc.)_}" + printf '\n### Open items\n' + if [ -n "$OPEN_ITEMS" ]; then + printf '%s\n\n' "$OPEN_ITEMS" + else + printf '_(unfilled — add open items / known issues / pending work)_\n\n' + fi + printf '### Notes\n' + if [ -n "$NOTES" ]; then + printf '%s\n\n' "$NOTES" + else + printf '_(unfilled — add any free-form context)_\n\n' + fi + + # ─── Threads inventory ─── + printf '## Threads (%d matched in %d site(s))\n\n' "${#MATCHES[@]}" "$(printf '%s\n' "${MATCHES[@]}" | awk -F'|' '{print $1}' | sort -u | wc -l | tr -d ' ')" + printf '| Site | Thread | Process | Direction | Port | Host | Type |\n' + printf '|---|---|---|---|---|---|---|\n' + for line in "${MATCHES[@]}"; do + IFS='|' read -r site nc prot <<< "$line" + pname=$("$NCP" protocol-field "$nc" "$prot" PROCESSNAME 2>/dev/null | head -1) + obib=$("$NCP" protocol-field "$nc" "$prot" OBWORKASIB 2>/dev/null | head -1) + outonly=$("$NCP" protocol-field "$nc" "$prot" OUTBOUNDONLY 2>/dev/null | head -1) + ptype=$("$NCP" protocol-nested "$nc" "$prot" PROTOCOL.TYPE 2>/dev/null | head -1) + phost=$("$NCP" protocol-nested "$nc" "$prot" PROTOCOL.HOST 2>/dev/null | head -1) + pport=$("$NCP" protocol-nested "$nc" "$prot" PROTOCOL.PORT 2>/dev/null | head -1) + isserver=$("$NCP" protocol-nested "$nc" "$prot" PROTOCOL.ISSERVER 2>/dev/null | head -1) + + direction="?" + [ "$isserver" = "1" ] && direction="inbound (TCP listener)" + [ "$obib" = "1" ] && [ "$direction" = "?" ] && direction="inbound (ICL/file)" + [ "$outonly" = "1" ] && [ "$direction" = "?" ] && direction="outbound" + + phost_clean=$(printf '%s' "$phost" | sed 's/^{}$//') + pport_clean=$(printf '%s' "$pport" | sed 's/^{}$//') + + printf '| `%s` | `%s` | `%s` | %s | %s | %s | %s |\n' \ + "$site" "$prot" "${pname:-?}" "$direction" "${pport_clean:-—}" "${phost_clean:-—}" "${ptype:-?}" + done + printf '\n' + + # ─── Per-thread detail ─── + for line in "${MATCHES[@]}"; do + IFS='|' read -r site nc prot <<< "$line" + printf '## `%s` (site: `%s`)\n\n' "$prot" "$site" + + printf '### Sources (what feeds this thread)\n\n' + sources=$("$NCP" sources "$nc" "$prot" 2>/dev/null) + if [ -n "$sources" ]; then + printf '%s\n' "$sources" | awk '{print "- `" $0 "`"}' + else + printf '_(none found in `%s`; may be fed via TCP from outside, or from another site via ICL)_\n' "$site" + fi + printf '\n' + + printf '### Destinations (where this thread routes to)\n\n' + dests=$("$NCP" destinations "$nc" "$prot" 2>/dev/null) + if [ -n "$dests" ]; then + printf '%s\n' "$dests" | awk '{print "- `" $0 "`"}' + else + printf '_(no DEST entries in DATAXLATE block)_\n' + fi + printf '\n' + + printf '### Xlates referenced\n\n' + xlates=$("$NCP" xlate-refs "$nc" "$prot" 2>/dev/null) + if [ -n "$xlates" ]; then + printf '%s\n' "$xlates" | awk -v site="$site" '{print "- `" site "/Xlate/" $0 "`"}' + else + printf '_(no xlates — pass-through or raw routing only)_\n' + fi + printf '\n' + + printf '### TCL procs referenced\n\n' + tcls=$("$NCP" tclproc-refs "$nc" "$prot" 2>/dev/null) + if [ -n "$tcls" ]; then + printf '%s\n' "$tcls" | awk -v site="$site" '{print "- `" site "/tclprocs/" $0 ".tcl`"}' + else + printf '_(no TCL procs referenced)_\n' + fi + printf '\n' + done + + # ─── Sources outside the matched set (the "fed by" landscape) ─── + printf '## Adjacent threads (the network this subsystem talks to)\n\n' + printf '_All threads that **either feed** matched threads **or are fed by** matched threads. These are the immediate operational neighbors._\n\n' + printf '| Site | Thread | Relationship to matched set |\n' + printf '|---|---|---|\n' + declare -A SEEN + for line in "${MATCHES[@]}"; do + IFS='|' read -r site nc prot <<< "$line" + while IFS= read -r src; do + key="$site|$src" + [ -n "${SEEN[$key]:-}" ] && continue + SEEN[$key]=1 + printf '| `%s` | `%s` | feeds `%s` |\n' "$site" "$src" "$prot" + done < <("$NCP" sources "$nc" "$prot" 2>/dev/null) + while IFS= read -r dst; do + key="$site|$dst" + [ -n "${SEEN[$key]:-}" ] && continue + SEEN[$key]=1 + printf '| `%s` | `%s` | receives from `%s` |\n' "$site" "$dst" "$prot" + done < <("$NCP" destinations "$nc" "$prot" 2>/dev/null) + done + printf '\n' + + # ─── Footer ─── + printf '---\n\n' + printf '_Generated: %s · NetConfigs scanned: %d · Pattern: `%s`_\n' \ + "$(date -Iseconds 2>/dev/null || date)" "${#NCONFIGS[@]}" "$PATTERN" +} | out_target + +[ -n "$OUT" ] && printf 'nc-document: wrote %s (%d matched threads across %d site(s))\n' \ + "$OUT" "${#MATCHES[@]}" "$(printf '%s\n' "${MATCHES[@]}" | awk -F'|' '{print $1}' | sort -u | wc -l | tr -d ' ')" >&2 diff --git a/lib/nc-find.sh b/lib/nc-find.sh new file mode 100755 index 0000000..55087f5 --- /dev/null +++ b/lib/nc-find.sh @@ -0,0 +1,233 @@ +#!/usr/bin/env bash +# nc-find.sh — cross-site Cloverleaf thread/protocol search. Native v3. +# +# Replaces (in v3 terms) the v1 family `tbn`, `tbp`, `tbh`, `tbpr`, plus the +# v1 ` where` command. Searches every NetConfig under $HCIROOT (or a +# passed list) without invoking v1/v2 wrappers, and emits site, thread, port, +# host, process, NetConfig path, and line number for each match. +# +# Usage: +# nc-find.sh --name PATTERN # like tbn: case-insensitive substring on thread name +# nc-find.sh --port PORT # like tbp: exact port match +# nc-find.sh --host HOST # like tbh: substring on host +# nc-find.sh --process PROC # like tbpr: substring on PROCESSNAME +# nc-find.sh --where THREAD # like ` where`: file:line of the thread's declaration +# nc-find.sh --xlate XLATENAME # threads referencing a specific .xlt file +# nc-find.sh --tclproc TCLPROC # threads referencing a specific tclproc +# +# Common flags: +# --hciroot DIR # default $HCIROOT +# --netconfigs PATHS # colon-separated explicit list (overrides --hciroot) +# --format tsv|table|jsonl # default: table +# --case-sensitive # default: case-insensitive for name/host/process +set -o pipefail + +NC_SELF="$0" +LIB_DIR="$(cd "$(dirname "$NC_SELF")" && pwd)" +NCP="$LIB_DIR/nc-parse.sh" + +die() { printf 'nc-find: %s\n' "$*" >&2; exit 1; } + +MODE="" +QUERY="" +HCIROOT_OVERRIDE="" +NETCONFIGS_OVERRIDE="" +FORMAT="table" +CASE_SENSITIVE=0 + +while [ $# -gt 0 ]; do + case "$1" in + --name) shift; MODE="name"; QUERY="$1" ;; + --port) shift; MODE="port"; QUERY="$1" ;; + --host) shift; MODE="host"; QUERY="$1" ;; + --process) shift; MODE="process"; QUERY="$1" ;; + --where) shift; MODE="where"; QUERY="$1" ;; + --xlate) shift; MODE="xlate"; QUERY="$1" ;; + --tclproc) shift; MODE="tclproc"; QUERY="$1" ;; + --hciroot) shift; HCIROOT_OVERRIDE="$1" ;; + --netconfigs) shift; NETCONFIGS_OVERRIDE="$1" ;; + --format) shift; FORMAT="$1" ;; + --case-sensitive) CASE_SENSITIVE=1 ;; + -h|--help) sed -n '2,22p' "$NC_SELF"; exit 0 ;; + -*) die "unknown flag: $1" ;; + *) die "extra arg: $1" ;; + esac + shift +done + +[ -n "$MODE" ] || die "specify a search mode: --name | --port | --host | --process | --where | --xlate | --tclproc" +case "$FORMAT" in tsv|table|jsonl) ;; *) die "bad --format: $FORMAT" ;; esac + +# Resolve NetConfig list +NCONFIGS=() +if [ -n "$NETCONFIGS_OVERRIDE" ]; then + IFS=':' read -ra NCONFIGS <<< "$NETCONFIGS_OVERRIDE" +else + ROOT="${HCIROOT_OVERRIDE:-${HCIROOT:-}}" + [ -n "$ROOT" ] || die "no \$HCIROOT and no --hciroot/--netconfigs" + while IFS= read -r nc; do NCONFIGS+=("$nc"); done < <(find "$ROOT" -maxdepth 2 -name NetConfig -type f 2>/dev/null) +fi +[ ${#NCONFIGS[@]} -gt 0 ] || die "no NetConfig files found" + +# Use a temp file to collect results, then format +RESULTS=$(mktemp) +trap 'rm -f "$RESULTS"' EXIT + +# Helper: emit one result row to $RESULTS +emit() { + # cols: site \t thread \t port \t host \t process \t direction \t file \t line + printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n" "$@" >> "$RESULTS" +} + +GREP_FLAGS="" +[ "$CASE_SENSITIVE" = "0" ] && GREP_FLAGS="-i" + +# Per-NetConfig scanning +for nc in "${NCONFIGS[@]}"; do + site=$(basename "$(dirname "$nc")") + + case "$MODE" in + name|where) + # Find protocol declarations matching the pattern + if [ "$MODE" = "where" ]; then + # Exact match (one specific thread name) + line=$(grep -nE "^protocol[[:space:]]+${QUERY}[[:space:]]+\{" "$nc" 2>/dev/null | head -1 | cut -d: -f1) + if [ -n "$line" ]; then + pname=$("$NCP" protocol-field "$nc" "$QUERY" PROCESSNAME 2>/dev/null | head -1) + pport=$("$NCP" protocol-nested "$nc" "$QUERY" PROTOCOL.PORT 2>/dev/null | head -1 | sed 's/^{}$//') + phost=$("$NCP" protocol-nested "$nc" "$QUERY" PROTOCOL.HOST 2>/dev/null | head -1 | sed 's/^{}$//') + isserver=$("$NCP" protocol-nested "$nc" "$QUERY" PROTOCOL.ISSERVER 2>/dev/null | head -1) + obib=$("$NCP" protocol-field "$nc" "$QUERY" OBWORKASIB 2>/dev/null | head -1) + outonly=$("$NCP" protocol-field "$nc" "$QUERY" OUTBOUNDONLY 2>/dev/null | head -1) + direction="?" + [ "$isserver" = "1" ] && direction="inbound-tcp" + [ "$obib" = "1" ] && [ "$direction" = "?" ] && direction="inbound-icl" + [ "$outonly" = "1" ] && [ "$direction" = "?" ] && direction="outbound" + emit "$site" "$QUERY" "${pport:-—}" "${phost:-—}" "${pname:-?}" "$direction" "$nc" "$line" + fi + else + # Partial match (substring) + while IFS= read -r raw; do + line=$(printf '%s' "$raw" | cut -d: -f1) + thread_name=$(printf '%s' "$raw" | sed -n 's/^[0-9]*:protocol[[:space:]]\+\([A-Za-z0-9_]\+\)[[:space:]]*{.*$/\1/p') + [ -z "$thread_name" ] && continue + pname=$("$NCP" protocol-field "$nc" "$thread_name" PROCESSNAME 2>/dev/null | head -1) + pport=$("$NCP" protocol-nested "$nc" "$thread_name" PROTOCOL.PORT 2>/dev/null | head -1 | sed 's/^{}$//') + phost=$("$NCP" protocol-nested "$nc" "$thread_name" PROTOCOL.HOST 2>/dev/null | head -1 | sed 's/^{}$//') + isserver=$("$NCP" protocol-nested "$nc" "$thread_name" PROTOCOL.ISSERVER 2>/dev/null | head -1) + obib=$("$NCP" protocol-field "$nc" "$thread_name" OBWORKASIB 2>/dev/null | head -1) + outonly=$("$NCP" protocol-field "$nc" "$thread_name" OUTBOUNDONLY 2>/dev/null | head -1) + direction="?" + [ "$isserver" = "1" ] && direction="inbound-tcp" + [ "$obib" = "1" ] && [ "$direction" = "?" ] && direction="inbound-icl" + [ "$outonly" = "1" ] && [ "$direction" = "?" ] && direction="outbound" + emit "$site" "$thread_name" "${pport:-—}" "${phost:-—}" "${pname:-?}" "$direction" "$nc" "$line" + done < <(grep -nE $GREP_FLAGS "^protocol[[:space:]]+[A-Za-z0-9_]*${QUERY}[A-Za-z0-9_]*[[:space:]]*\{" "$nc" 2>/dev/null) + fi + ;; + + port) + # Find protocols whose inner PROTOCOL.PORT equals QUERY. + "$NCP" list-protocols "$nc" 2>/dev/null | while IFS= read -r tname; do + p=$("$NCP" protocol-nested "$nc" "$tname" PROTOCOL.PORT 2>/dev/null | head -1) + if [ "$p" = "$QUERY" ]; then + pname=$("$NCP" protocol-field "$nc" "$tname" PROCESSNAME 2>/dev/null | head -1) + phost=$("$NCP" protocol-nested "$nc" "$tname" PROTOCOL.HOST 2>/dev/null | head -1 | sed 's/^{}$//') + line=$("$NCP" protocol-line "$nc" "$tname" 2>/dev/null) + isserver=$("$NCP" protocol-nested "$nc" "$tname" PROTOCOL.ISSERVER 2>/dev/null | head -1) + obib=$("$NCP" protocol-field "$nc" "$tname" OBWORKASIB 2>/dev/null | head -1) + outonly=$("$NCP" protocol-field "$nc" "$tname" OUTBOUNDONLY 2>/dev/null | head -1) + direction="?" + [ "$isserver" = "1" ] && direction="inbound-tcp" + [ "$obib" = "1" ] && [ "$direction" = "?" ] && direction="inbound-icl" + [ "$outonly" = "1" ] && [ "$direction" = "?" ] && direction="outbound" + emit "$site" "$tname" "$p" "${phost:-—}" "${pname:-?}" "$direction" "$nc" "${line:-?}" + fi + done + ;; + + host) + "$NCP" list-protocols "$nc" 2>/dev/null | while IFS= read -r tname; do + h=$("$NCP" protocol-nested "$nc" "$tname" PROTOCOL.HOST 2>/dev/null | head -1 | sed 's/^{}$//') + if [ "$CASE_SENSITIVE" = "1" ]; then + [[ "$h" == *"$QUERY"* ]] || continue + else + shopt -s nocasematch 2>/dev/null + [[ "$h" == *"$QUERY"* ]] || { shopt -u nocasematch 2>/dev/null; continue; } + shopt -u nocasematch 2>/dev/null + fi + [ -z "$h" ] && continue + pname=$("$NCP" protocol-field "$nc" "$tname" PROCESSNAME 2>/dev/null | head -1) + p=$("$NCP" protocol-nested "$nc" "$tname" PROTOCOL.PORT 2>/dev/null | head -1 | sed 's/^{}$//') + line=$("$NCP" protocol-line "$nc" "$tname" 2>/dev/null) + emit "$site" "$tname" "${p:-—}" "$h" "${pname:-?}" "?" "$nc" "${line:-?}" + done + ;; + + process) + "$NCP" list-protocols "$nc" 2>/dev/null | while IFS= read -r tname; do + pname=$("$NCP" protocol-field "$nc" "$tname" PROCESSNAME 2>/dev/null | head -1) + if [ "$CASE_SENSITIVE" = "1" ]; then + [[ "$pname" == *"$QUERY"* ]] || continue + else + shopt -s nocasematch 2>/dev/null + [[ "$pname" == *"$QUERY"* ]] || { shopt -u nocasematch 2>/dev/null; continue; } + shopt -u nocasematch 2>/dev/null + fi + p=$("$NCP" protocol-nested "$nc" "$tname" PROTOCOL.PORT 2>/dev/null | head -1 | sed 's/^{}$//') + h=$("$NCP" protocol-nested "$nc" "$tname" PROTOCOL.HOST 2>/dev/null | head -1 | sed 's/^{}$//') + line=$("$NCP" protocol-line "$nc" "$tname" 2>/dev/null) + emit "$site" "$tname" "${p:-—}" "${h:-—}" "$pname" "?" "$nc" "${line:-?}" + done + ;; + + xlate|tclproc) + # threads that reference a given xlate or tclproc + need_pattern="$QUERY" + "$NCP" list-protocols "$nc" 2>/dev/null | while IFS= read -r tname; do + if [ "$MODE" = "xlate" ]; then + hits=$("$NCP" xlate-refs "$nc" "$tname" 2>/dev/null | grep -F -- "$need_pattern" || true) + else + hits=$("$NCP" tclproc-refs "$nc" "$tname" 2>/dev/null | grep -F -- "$need_pattern" || true) + fi + [ -z "$hits" ] && continue + pname=$("$NCP" protocol-field "$nc" "$tname" PROCESSNAME 2>/dev/null | head -1) + line=$("$NCP" protocol-line "$nc" "$tname" 2>/dev/null) + emit "$site" "$tname" "—" "—" "${pname:-?}" "?" "$nc" "${line:-?}" + done + ;; + esac +done + +# Format output +case "$FORMAT" in + tsv) + printf "site\tthread\tport\thost\tprocess\tdirection\tfile\tline\n" + cat "$RESULTS" + ;; + table) + { + printf "site\tthread\tport\thost\tprocess\tdirection\tline\n" + awk -F'\t' '{printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\n", $1, $2, $3, $4, $5, $6, $8}' "$RESULTS" + } | awk -F'\t' ' + { for (i=1;i<=NF;i++){ if (length($i)>w[i]) w[i]=length($i); cell[NR,i]=$i }; rows=NR; cols=NF } + END { + for (r=1; r<=rows; r++) { + for (c=1; c<=cols; c++) printf "%-*s ", w[c], cell[r,c] + printf "\n" + if (r==1) { for (c=1; c<=cols; c++) for (k=0;k&2 diff --git a/lib/nc-inbound.sh b/lib/nc-inbound.sh new file mode 100755 index 0000000..ec0f4ed --- /dev/null +++ b/lib/nc-inbound.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +# nc-inbound.sh — identify inbound threads in a Cloverleaf NetConfig. +# +# Three classes of inbound thread: +# 1. tcp-listen — PROTOCOL.ISSERVER=1, listens for upstream client TCP connections. +# This is "directly fed by upstream client systems" in the strict sense. +# 2. icl-or-file — OBWORKASIB=1, TYPE=file. Fed via Cloverleaf's inter-cloverleaf +# link (ICL) or file-drop from another internal source. +# 3. file-edge — OBWORKASIB=1 + TYPE=file + no ICLSERVERPORT, suggests file-drop +# from an external system (less common). +# +# Usage: +# nc-inbound.sh [--mode tcp-listen|icl-or-file|all] [--format tsv|jsonl|table] +# +# Defaults: --mode all --format tsv +# +# TSV columns: name, process, class, port, host, type +set -u +set -o pipefail + +NC_SELF="$0" +LIB_DIR="$(cd "$(dirname "$NC_SELF")" && pwd)" +NCP="$LIB_DIR/nc-parse.sh" + +die() { printf 'nc-inbound: %s\n' "$*" >&2; exit 1; } + +NC="" +MODE="all" +FMT="tsv" + +while [ $# -gt 0 ]; do + case "$1" in + --mode) shift; MODE="$1" ;; + --format) shift; FMT="$1" ;; + -h|--help) sed -n '2,18p' "$NC_SELF"; exit 0 ;; + -*) die "unknown flag: $1" ;; + *) [ -z "$NC" ] && NC="$1" || die "extra arg: $1" ;; + esac + shift +done + +[ -n "$NC" ] || die "usage: $NC_SELF [--mode ...] [--format ...]" +[ -f "$NC" ] || die "not a file: $NC" + +case "$MODE" in tcp-listen|icl-or-file|all) ;; *) die "bad --mode: $MODE" ;; esac +case "$FMT" in tsv|jsonl|table) ;; *) die "bad --format: $FMT" ;; esac + +# Build records: TSV stream emit +collect() { + local n + "$NCP" list-protocols "$NC" | while IFS= read -r n; do + local pname obib outonly ptype phost pport isserver klass + pname=$("$NCP" protocol-field "$NC" "$n" PROCESSNAME | head -1) + obib=$("$NCP" protocol-field "$NC" "$n" OBWORKASIB | head -1) + outonly=$("$NCP" protocol-field "$NC" "$n" OUTBOUNDONLY | head -1) + ptype=$("$NCP" protocol-nested "$NC" "$n" PROTOCOL.TYPE 2>/dev/null | head -1) + phost=$("$NCP" protocol-nested "$NC" "$n" PROTOCOL.HOST 2>/dev/null | head -1) + pport=$("$NCP" protocol-nested "$NC" "$n" PROTOCOL.PORT 2>/dev/null | head -1) + isserver=$("$NCP" protocol-nested "$NC" "$n" PROTOCOL.ISSERVER 2>/dev/null | head -1) + + # Clean empty-brace markers + phost=$(printf '%s' "$phost" | sed 's/^{}$//') + pport=$(printf '%s' "$pport" | sed 's/^{}$//') + + if [ "$isserver" = "1" ]; then + klass="tcp-listen" + elif [ "$obib" = "1" ]; then + klass="icl-or-file" + else + continue # not inbound — skip + fi + + # Mode filter + case "$MODE" in + tcp-listen) [ "$klass" = "tcp-listen" ] || continue ;; + icl-or-file) [ "$klass" = "icl-or-file" ] || continue ;; + esac + + printf "%s\t%s\t%s\t%s\t%s\t%s\n" \ + "$n" "${pname:-}" "$klass" "${pport:-}" "${phost:-}" "${ptype:-}" + done +} + +case "$FMT" in + tsv) + printf "name\tprocess\tclass\tport\thost\ttype\n" + collect + ;; + jsonl) + collect | awk -F'\t' ' + function esc(s) { gsub(/\\/, "\\\\", s); gsub(/"/, "\\\"", s); return s } + { + printf "{\"name\":\"%s\",\"process\":\"%s\",\"class\":\"%s\",\"port\":\"%s\",\"host\":\"%s\",\"type\":\"%s\"}\n", + esc($1), esc($2), esc($3), esc($4), esc($5), esc($6) + }' + ;; + table) + { + printf "name\tprocess\tclass\tport\thost\ttype\n" + collect + } | awk -F'\t' ' + { for (i=1;i<=NF;i++){ if (length($i)>w[i]) w[i]=length($i); cell[NR,i]=$i }; rows=NR; cols=NF } + END { + for (r=1; r<=rows; r++) { + for (c=1; c<=cols; c++) printf "%-*s ", w[c], cell[r,c] + printf "\n" + if (r==1) { for (c=1; c<=cols; c++) for (k=0;k # newest-first +# larry-rollback.sh --session # whole session +# larry-rollback.sh --entry # one specific write +# +# Usage: +# nc-insert-protocol.sh insert [--mode end|after|before --anchor NAME] +# nc-insert-protocol.sh add-route +# +# path to a file containing the TCL `protocol NAME { … }` block to insert +# path to a file containing the route entry (just the inner `{ … }`) +# +# Exit codes: 0 OK, 2 usage, 3 target not found, 4 already exists / wouldn't change +set -o pipefail + +NC_SELF="$0" +LIB_DIR="$(cd "$(dirname "$NC_SELF")" && pwd)" +NCP="$LIB_DIR/nc-parse.sh" +JOURNAL="$LIB_DIR/journal.sh" + +die() { printf 'nc-insert-protocol: %s\n' "$*" >&2; exit 1; } + +# Source journal so we can call journal_write directly +# shellcheck disable=SC1090 +. "$JOURNAL" + +cmd_insert() { + local nc="$1" block_file="$2" + shift 2 + local mode="end" anchor="" + while [ $# -gt 0 ]; do + case "$1" in + --mode) shift; mode="$1" ;; + --anchor) shift; anchor="$1" ;; + *) die "unknown flag for insert: $1" ;; + esac + shift + done + + [ -f "$nc" ] || { die "no such NetConfig: $nc"; } + [ -f "$block_file" ] || { die "no such block file: $block_file"; } + + # Extract block name to detect collisions + local block_name + block_name=$(awk '/^protocol [A-Za-z0-9_]+/ {print $2; exit}' "$block_file") + [ -n "$block_name" ] || die "block file does not start with 'protocol NAME {' — bad input" + + if "$NCP" list-protocols "$nc" 2>/dev/null | grep -qx "$block_name"; then + die "protocol '$block_name' already exists in $nc — refusing to insert. Use a different name or update the existing block via a different tool." + fi + + local tmp; tmp=$(mktemp) + + case "$mode" in + end) + cat "$nc" > "$tmp" + # Ensure trailing newline before appending + [ -n "$(tail -c1 "$tmp")" ] && printf '\n' >> "$tmp" + printf '\n' >> "$tmp" + cat "$block_file" >> "$tmp" + printf '\n' >> "$tmp" + ;; + after) + [ -n "$anchor" ] || die "--mode after needs --anchor NAME" + local end_line + end_line=$("$NCP" list-protocols "$nc" >/dev/null 2>&1 + # Compute end line via the same logic as protocol-block + awk -v target="$anchor" ' + BEGIN { depth=0; in_block=0; name="" } + /^protocol [A-Za-z0-9_]+ \{$/ && !in_block { + name=$2; depth=1; in_block=1 + if (name==target) start=NR + next + } + in_block { + n_open=gsub(/\{/,"{",$0); n_close=gsub(/\}/,"}",$0) + depth += n_open - n_close + if (depth==0) { + if (name==target) { print NR; exit } + in_block=0; name="" + } + }' "$nc") + [ -n "$end_line" ] || die "anchor protocol not found: $anchor" + head -n "$end_line" "$nc" > "$tmp" + printf '\n' >> "$tmp" + cat "$block_file" >> "$tmp" + printf '\n' >> "$tmp" + tail -n +$((end_line + 1)) "$nc" >> "$tmp" + ;; + before) + [ -n "$anchor" ] || die "--mode before needs --anchor NAME" + local start_line + start_line=$("$NCP" protocol-line "$nc" "$anchor" 2>/dev/null) + [ -n "$start_line" ] || die "anchor protocol not found: $anchor" + head -n $((start_line - 1)) "$nc" > "$tmp" + cat "$block_file" >> "$tmp" + printf '\n' >> "$tmp" + tail -n +"$start_line" "$nc" >> "$tmp" + ;; + *) die "bad --mode: $mode (use end|after|before)" ;; + esac + + # Hand off to journal for atomic backup+write + local entry_id + entry_id=$(journal_write "$nc" "$tmp") + rm -f "$tmp" + printf 'inserted protocol %s into %s (mode=%s)\n' "$block_name" "$nc" "$mode" + printf 'journal entry: %s\n' "$entry_id" + printf 'rollback: larry-rollback.sh --entry %s OR larry-rollback.sh --target %s\n' "$entry_id" "$nc" +} + +cmd_add_route() { + local nc="$1" prot="$2" route_file="$3" + [ -f "$nc" ] || die "no such NetConfig: $nc" + [ -f "$route_file" ] || die "no such route file: $route_file" + + # Find the protocol's line range + local start end + start=$("$NCP" protocol-line "$nc" "$prot" 2>/dev/null) + [ -n "$start" ] || die "no such protocol: $prot" + + # Compute end-line of the protocol block + end=$(awk -v s="$start" ' + NR == s { depth = 1; in_block = 1; next } + in_block { + n_open = gsub(/\{/, "{", $0) + n_close = gsub(/\}/, "}", $0) + depth += n_open - n_close + if (depth == 0) { print NR; exit } + } + ' "$nc") + [ -n "$end" ] || die "could not determine end of protocol block: $prot" + + # Find the DATAXLATE inner block boundaries (just inside the protocol). + # Lines look like: + # { DATAXLATE { + # {existing route 1} + # {existing route 2} + # } } + local dx_start dx_end + dx_start=$(awk -v s="$start" -v e="$end" ' + NR>s && NR } } + # If empty, replace with a populated block on a single line plus the new route. + local dx_empty_line + dx_empty_line=$(awk -v s="$start" -v e="$end" ' + NR>s && NRs && NR=s && NR<=e { + n_open = gsub(/\{/, "{", $0) + n_close = gsub(/\}/, "}", $0) + if (NR == s) { + # The DATAXLATE-open line; the outer "{ DATAXLATE {" has 2 opens + depth = 0 + } + depth += n_open - n_close + if (NR > s && depth <= -2) { # closing "} }" brings us down 2 levels + print NR; exit + } + } + ' "$nc") + # If our naive count failed, try a simpler approach: find next line that is + # exactly the closing pattern of DATAXLATE ( } } at the appropriate indent). + if [ -z "$dx_end" ]; then + dx_end=$(awk -v s="$dx_start" -v e="$end" ' + NR>s && NR<=e && /^[[:space:]]+\}[[:space:]]\}[[:space:]]*$/ { print NR; exit } + ' "$nc") + fi + [ -n "$dx_end" ] || die "could not locate end of DATAXLATE block in protocol $prot" + + # Indent the route content to match the DATAXLATE inner indentation (8 spaces typical). + local indent=" " + local indented_route; indented_route=$(awk -v IND="$indent" '{print IND $0}' "$route_file") + + # Build the new file: + # lines 1..dx_start + # route content (indented) + # lines dx_start+1..end of file + # Skip the simple "empty" case: if dx_start was a "{ DATAXLATE {} }" single line, + # we need to split it. Detect by reading that line. + local dx_start_line; dx_start_line=$(sed -n "${dx_start}p" "$nc") + local tmp; tmp=$(mktemp) + + if [[ "$dx_start_line" =~ \{[[:space:]]DATAXLATE[[:space:]]\{[[:space:]]*\}[[:space:]]*\}[[:space:]]*$ ]]; then + # Single-line "{ DATAXLATE { } }" — replace with multi-line form + head -n $((dx_start - 1)) "$nc" > "$tmp" + printf ' { DATAXLATE {\n' >> "$tmp" + printf '%s\n' "$indented_route" >> "$tmp" + printf ' } }\n' >> "$tmp" + tail -n +$((dx_start + 1)) "$nc" >> "$tmp" + else + head -n "$dx_start" "$nc" > "$tmp" + printf '%s\n' "$indented_route" >> "$tmp" + tail -n +$((dx_start + 1)) "$nc" >> "$tmp" + fi + + local entry_id + entry_id=$(journal_write "$nc" "$tmp") + rm -f "$tmp" + printf 'added route to protocol %s in %s (DATAXLATE block at line %s)\n' "$prot" "$nc" "$dx_start" + printf 'journal entry: %s\n' "$entry_id" + printf 'rollback: larry-rollback.sh --entry %s OR larry-rollback.sh --target %s\n' "$entry_id" "$nc" +} + +# ───────────────────────────────────────────────────────────────────────────── +SUB="${1:-help}" +case "$SUB" in + insert) + [ $# -ge 3 ] || { echo "usage: $0 insert [--mode end|after|before --anchor NAME]" >&2; exit 2; } + cmd_insert "$2" "$3" "${@:4}" ;; + add-route) + [ $# -ge 4 ] || { echo "usage: $0 add-route " >&2; exit 2; } + cmd_add_route "$2" "$3" "$4" ;; + help|-h|--help) sed -n '2,30p' "$NC_SELF" ;; + *) echo "unknown subcommand: $SUB" >&2; exit 2 ;; +esac diff --git a/lib/nc-make-jump.sh b/lib/nc-make-jump.sh new file mode 100755 index 0000000..b564913 --- /dev/null +++ b/lib/nc-make-jump.sh @@ -0,0 +1,424 @@ +#!/usr/bin/env bash +# nc-make-jump.sh — generate the 3-thread jump pattern for cross-environment +# data replay during a Cloverleaf migration. Matches Bryan's house pattern +# (linux__out / windows__in / windows__out). +# +# Topology: +# +# OLD env (e.g. windows) NEW env (linux) +# ── adt site (existing) ── ── server_jump site (new) ── ── adt site (cloned, unchanged) ── +# ┌─────────────────────┐ ┌────────────────────────────┐ ┌─────────────────────────┐ +# │ │ │ windows__in (NEW) │ │ │ +# │ ├─→ existing dests │ │ tcpip-server, ISSERVER=1 │ │ listens on ORIG_PORT │ +# │ └─→ linux__ │ ──TCP──→ │ PORT = jump_port │ │ (UNCHANGED — no route │ +# │ out (NEW) │ │ │ internal route │ │ changes here) │ +# │ tcpip-client │ │ ▼ │ │ │ +# │ → jump_port │ │ windows__out (NEW) │ │ ▲ │ +# └─────────────────────┘ │ tcpip-client │ │ │ │ +# │ HOST=127.0.0.1 │ │ │ TCP localhost │ +# │ PORT=ORIG_PORT ─────────┼───────┘ │ ──────────────────── │ +# └────────────────────────────┘ │ +# │ +# (so OLD's existing inbound gets ONE new route; NEW's existing inbound is UNTOUCHED) +# +# Three new protocol blocks: +# 1. `linux__out` → add to OLD env NetConfig (same process as original inbound). +# 2. `windows__in` → add to NEW env server_jump/NetConfig. +# 3. `windows__out` → add to NEW env server_jump/NetConfig. +# +# Plus one route-add snippet for OLD's existing inbound's DATAXLATE block. +# +# Tag = inbound thread name (auto-derived per Bryan's preference). +# +# Usage: +# nc-make-jump.sh --inbound NAME --new-host HOST --jump-port PORT +# [--inbound-host HOST] # default 127.0.0.1 +# [--process-jump PROC] # process for NEW-side threads, default server_jump +# [--encoding ENC] # default = ENCODING from existing inbound +# [--out-prefix PREFIX] # write files instead of stdout +set -u +set -o pipefail + +NC_SELF="$0" +LIB_DIR="$(cd "$(dirname "$NC_SELF")" && pwd)" +NCP="$LIB_DIR/nc-parse.sh" + +die() { printf 'nc-make-jump: %s\n' "$*" >&2; exit 1; } + +NC="" +INBOUND="" +NEW_HOST="" +JUMP_PORT="" +INBOUND_HOST="127.0.0.1" +PROC_JUMP="server_jump" +ENC_OVERRIDE="" +OUT_PREFIX="" + +while [ $# -gt 0 ]; do + case "$1" in + --inbound) shift; INBOUND="$1" ;; + --new-host) shift; NEW_HOST="$1" ;; + --jump-port) shift; JUMP_PORT="$1" ;; + --inbound-host) shift; INBOUND_HOST="$1" ;; + --process-jump) shift; PROC_JUMP="$1" ;; + --encoding) shift; ENC_OVERRIDE="$1" ;; + --out-prefix) shift; OUT_PREFIX="$1" ;; + -h|--help) sed -n '2,50p' "$NC_SELF"; exit 0 ;; + -*) die "unknown flag: $1" ;; + *) [ -z "$NC" ] && NC="$1" || die "extra arg: $1" ;; + esac + shift +done + +[ -n "$NC" ] || die "usage: see --help" +[ -n "$INBOUND" ] || die "missing --inbound" +[ -n "$NEW_HOST" ] || die "missing --new-host" +[ -n "$JUMP_PORT" ] || die "missing --jump-port" +[ -f "$NC" ] || die "not a file: $NC" + +# Read fields from the existing inbound +T_PROCESS=$("$NCP" protocol-field "$NC" "$INBOUND" PROCESSNAME 2>/dev/null | head -1) +[ -n "$T_PROCESS" ] || die "no such protocol in $NC: $INBOUND" + +T_ENC=$("$NCP" protocol-field "$NC" "$INBOUND" ENCODING 2>/dev/null | head -1) +[ -z "$T_ENC" ] && T_ENC="ASCII" +ENC="${ENC_OVERRIDE:-$T_ENC}" + +ORIG_PORT=$("$NCP" protocol-nested "$NC" "$INBOUND" PROTOCOL.PORT 2>/dev/null | head -1) +[ -n "$ORIG_PORT" ] || die "could not read PROTOCOL.PORT of inbound $INBOUND (is it a TCP listener? if it's a file/ICL inbound, this pattern may not apply directly)" + +# tag = the inbound name itself (Bryan's "auto-derived" preference) +TAG="$INBOUND" + +OLD_OUT_NAME="linux_${TAG}_out" +NEW_IN_NAME="windows_${TAG}_in" +NEW_OUT_NAME="windows_${TAG}_out" + +# ───────────────────────────────────────────────────────────────────────────── +# Common helpers — emit the standard set of fields that every protocol block +# carries. Differences between the three threads are isolated below. +# ───────────────────────────────────────────────────────────────────────────── +emit_dataformat_passthrough() { + cat <<'EOF' + { DATAFORMAT { + { FRLTYPE offlen } + { OFFLEN { { LEN 0 } { OFF 0 } } } + { TYPE frl } + } } +EOF +} + +emit_edibatch_empty() { + cat <<'EOF' + { EDIBATCH { + { IN_DATA { { TYPE {} } { VERSION {} } } } + { OUT_DATA { { HEADER {} } { TRIGGER { { COUNT {} } { SCHEDULER {} } { TIMER {} } } } { TYPE {} } { VERSION {} } } } + } } +EOF +} + +emit_errdbtps_default() { + cat <<'EOF' + { ERRDBTPS { + { ERRTPSPROCS { { ARGS {} } { PROCS {} } { PROCSCONTROL {} } } } + { RETRIES -1 } + } } +EOF +} + +emit_proc_blocks_empty() { + cat <<'EOF' + { RECVCONTROL { { ACKCONTROL { { ARGS {} } { PROCS {} } { PROCSCONTROL {} } } } { EOMSG {} } { HOLDMSGS 0 } { MSGPRIO 5120 } } } + { SAVECONTROL { { ARGS {} } { PROCS {} } { PROCSCONTROL {} } } } + { TPS_INBOUND { { ARGS {} } { PROCS {} } { PROCSCONTROL {} } } } + { TPS_OUTBOUND { { ARGS {} } { PROCS {} } { PROCSCONTROL {} } } } + { TRACING 0 } +EOF +} + +emit_inner_protocol_tcp_client() { + local host="$1" port="$2" + cat <_out +# Outbound TCP client, same process as the existing inbound. +# No DATAXLATE (pass-through). Receives data via the route-add on the original inbound. +# ───────────────────────────────────────────────────────────────────────────── +emit_old_out() { + printf 'protocol %s {\n' "$OLD_OUT_NAME" + cat <_in +# Inbound TCP server. Routes internally to windows__out (same site). +# ───────────────────────────────────────────────────────────────────────────── +emit_new_in() { + printf 'protocol %s {\n' "$NEW_IN_NAME" + cat <_out +# Outbound TCP client, connects to localhost on the ORIGINAL inbound port. +# Receives via internal route from windows__in. +# ───────────────────────────────────────────────────────────────────────────── +emit_new_out() { + printf 'protocol %s {\n' "$NEW_OUT_NAME" + cat < "${OUT_PREFIX}.old_out.tcl" + emit_new_in > "${OUT_PREFIX}.new_in.tcl" + emit_new_out > "${OUT_PREFIX}.new_out.tcl" + emit_route_add > "${OUT_PREFIX}.route_add.tcl" + printf '%s\n%s\n%s\n%s\n' \ + "${OUT_PREFIX}.old_out.tcl" \ + "${OUT_PREFIX}.new_in.tcl" \ + "${OUT_PREFIX}.new_out.tcl" \ + "${OUT_PREFIX}.route_add.tcl" +else + cat < [--after EXPR] [--before EXPR] +# [--field PATH=VALUE] # repeatable filter, AND semantics +# [--type DATA|ACK] +# [--limit N] # default 100 +# [--format text|json|count|raw] +# [--sitedir DIR] # default $HCISITEDIR +# [--db PATH] # explicit smatdb path (overrides locate) +# +# Time expressions (--after, --before): +# "3 days ago", "12 hours ago", "30 minutes ago" +# "2026-05-20", "2026-05-20 14:30:00" +# unix epoch in seconds (e.g. 1772100000) +# +# Examples: +# nc-msgs.sh to_3m --after "3 days ago" --field PID.18=623000286 +# nc-msgs.sh ADTto_3m --field MSH.9.2=A08 --limit 5 +# nc-msgs.sh ADTto_3m --format count +set -u +set -o pipefail + +NC_SELF="$0" +LIB_DIR="$(cd "$(dirname "$NC_SELF")" && pwd)" +HL7F="$LIB_DIR/hl7-field.sh" + +die() { printf 'nc-msgs: %s\n' "$*" >&2; exit 1; } + +THREAD="" +AFTER="" +BEFORE="" +FILTERS=() +TYPE="" +LIMIT=100 +FORMAT="text" +SITEDIR="${HCISITEDIR:-}" +DB_OVERRIDE="" + +while [ $# -gt 0 ]; do + case "$1" in + --after) shift; AFTER="$1" ;; + --before) shift; BEFORE="$1" ;; + --field) shift; FILTERS+=("$1") ;; + --type) shift; TYPE="$1" ;; + --limit) shift; LIMIT="$1" ;; + --format) shift; FORMAT="$1" ;; + --sitedir) shift; SITEDIR="$1" ;; + --db) shift; DB_OVERRIDE="$1" ;; + -h|--help) sed -n '2,30p' "$NC_SELF"; exit 0 ;; + -*) die "unknown flag: $1" ;; + *) [ -z "$THREAD" ] && THREAD="$1" || die "extra arg: $1" ;; + esac + shift +done + +[ -n "$THREAD" ] || die "usage: nc-msgs.sh [...flags]" +case "$FORMAT" in text|json|count|raw) ;; *) die "bad --format: $FORMAT" ;; esac +command -v sqlite3 >/dev/null 2>&1 || die "sqlite3 not on PATH (universally available on Cloverleaf hosts; install via your distro otherwise)" + +# Locate smatdb +locate_smatdb() { + if [ -n "$DB_OVERRIDE" ]; then + [ -f "$DB_OVERRIDE" ] || die "no such db: $DB_OVERRIDE" + printf '%s\n' "$DB_OVERRIDE" + return + fi + [ -n "$SITEDIR" ] || die "no \$HCISITEDIR and no --sitedir; pass one or set the env var" + [ -d "$SITEDIR" ] || die "sitedir not a directory: $SITEDIR" + # Standard layout: $SITEDIR/exec/processes//.smatdb + local found + found=$(find "$SITEDIR/exec/processes" -maxdepth 2 -type f -name "${THREAD}.smatdb" 2>/dev/null | head -1) + if [ -z "$found" ]; then + # Sometimes lives one level deeper or under a different layout + found=$(find "$SITEDIR" -type f -name "${THREAD}.smatdb" 2>/dev/null | head -1) + fi + [ -n "$found" ] || die "no smatdb found for thread $THREAD under $SITEDIR (looked for ${THREAD}.smatdb)" + printf '%s\n' "$found" +} + +# Parse time expression -> unix ms +parse_time_ms() { + local expr="$1" + [ -z "$expr" ] && return 0 + # If it's purely numeric and >= 10 digits, treat as already-ms + if [[ "$expr" =~ ^[0-9]+$ ]]; then + if [ "${#expr}" -ge 12 ]; then printf '%s' "$expr"; return; fi + if [ "${#expr}" -le 10 ]; then printf '%s' "$((expr * 1000))"; return; fi + fi + # GNU date and BSD date differ. Try GNU first (-d EXPR), fall back to BSD (-jf or -v). + local ts="" + if ts=$(date -d "$expr" +%s 2>/dev/null); then + printf '%s' "$((ts * 1000))"; return + fi + # BSD date — try `-v` shorthand for relative times + if echo "$expr" | grep -qE '^[0-9]+ (second|minute|hour|day|week|month|year)s? ago$'; then + local n unit + n=$(echo "$expr" | awk '{print $1}') + unit=$(echo "$expr" | awk '{print $2}' | sed 's/s$//') + local flag + case "$unit" in + second) flag="S" ;; + minute) flag="M" ;; + hour) flag="H" ;; + day) flag="d" ;; + week) flag="d"; n=$((n * 7)) ;; + month) flag="m" ;; + year) flag="y" ;; + esac + ts=$(date -v "-${n}${flag}" +%s 2>/dev/null) && { printf '%s' "$((ts * 1000))"; return; } + fi + # BSD date with -jf + if ts=$(date -jf "%Y-%m-%d %H:%M:%S" "$expr" +%s 2>/dev/null); then + printf '%s' "$((ts * 1000))"; return + fi + if ts=$(date -jf "%Y-%m-%d" "$expr" +%s 2>/dev/null); then + printf '%s' "$((ts * 1000))"; return + fi + die "could not parse time expression: $expr" +} + +AFTER_MS=$(parse_time_ms "$AFTER") +BEFORE_MS=$(parse_time_ms "$BEFORE") + +# Build WHERE clause +WHERE="1=1" +[ -n "$AFTER_MS" ] && WHERE="$WHERE AND Time >= $AFTER_MS" +[ -n "$BEFORE_MS" ] && WHERE="$WHERE AND Time <= $BEFORE_MS" +if [ -n "$TYPE" ]; then + # Escape single quotes + ESC_TYPE=$(printf '%s' "$TYPE" | sed "s/'/''/g") + WHERE="$WHERE AND Type = '$ESC_TYPE'" +fi + +# Coarse LIKE pre-filter for any --field VALUEs (substring presence) +# This is just an SQL fast-path; the precise field match happens via hl7-field.sh below. +for filt in "${FILTERS[@]}"; do + val="${filt#*=}" + if [ -n "$val" ] && [ "$val" != "$filt" ]; then + ESC_VAL=$(printf '%s' "$val" | sed "s/'/''/g") + WHERE="$WHERE AND MessageContent LIKE '%${ESC_VAL}%'" + fi +done + +SMATDB=$(locate_smatdb) +[ "$FORMAT" = "count" ] || printf 'nc-msgs: querying %s\n' "$SMATDB" >&2 + +# Pull the data +TMP_OUT=$(mktemp -d) +trap 'rm -rf "$TMP_OUT"' EXIT + +SQL="SELECT Time, Type, SourceConn, DestConn, MessageContent FROM smat_msgs WHERE $WHERE ORDER BY Time DESC LIMIT $LIMIT" +sqlite3 -ascii "$SMATDB" "$SQL" > "$TMP_OUT/raw.bin" 2>"$TMP_OUT/err" +if [ -s "$TMP_OUT/err" ]; then + cat "$TMP_OUT/err" >&2 + exit 1 +fi + +# Split rows (0x1e) into individual files, parse fields per row (0x1f) +awk -v RS=$'\x1e' -v FS=$'\x1f' -v outdir="$TMP_OUT" ' + NF >= 5 { + n++ + fpath = outdir "/msg_" sprintf("%05d", n) ".bin" + print $5 > fpath + close(fpath) + metafpath = outdir "/meta_" sprintf("%05d", n) ".tsv" + printf "%s\t%s\t%s\t%s\n", $1, $2, $3, $4 > metafpath + close(metafpath) + } +' "$TMP_OUT/raw.bin" + +MSG_COUNT=$(ls "$TMP_OUT"/msg_*.bin 2>/dev/null | wc -l | tr -d ' ') +KEPT=0 + +# Apply --field filters precisely via hl7-field.sh +match_filters() { + local msg_file="$1" + for filt in "${FILTERS[@]}"; do + path="${filt%%=*}" + expected="${filt#*=}" + [ "$path" = "$expected" ] && continue # skip if "=" missing + # exact match: any repetition equal to expected + actual=$("$HL7F" "$path" "$msg_file" 2>/dev/null) + matched=0 + if [ -n "$actual" ]; then + while IFS= read -r rep; do + [ "$rep" = "$expected" ] && { matched=1; break; } + done <<< "$actual" + fi + [ "$matched" = "1" ] || return 1 + done + return 0 +} + +# Emit +case "$FORMAT" in + count) + # Count after filter + if [ ${#FILTERS[@]} -eq 0 ]; then + echo "$MSG_COUNT" + else + for f in "$TMP_OUT"/msg_*.bin; do + match_filters "$f" && KEPT=$((KEPT+1)) + done + echo "$KEPT" + fi + ;; + raw) + for f in "$TMP_OUT"/msg_*.bin; do + if [ ${#FILTERS[@]} -eq 0 ] || match_filters "$f"; then + cat "$f"; printf '\x1c' # File separator between messages (rare in HL7) + KEPT=$((KEPT+1)) + fi + done + ;; + text) + i=0 + for f in "$TMP_OUT"/msg_*.bin; do + i=$((i+1)) + if [ ${#FILTERS[@]} -eq 0 ] || match_filters "$f"; then + KEPT=$((KEPT+1)) + meta=$(cat "${TMP_OUT}/meta_$(printf '%05d' "$i").tsv") + tm=$(printf '%s' "$meta" | awk -F'\t' '{print $1}') + typ=$(printf '%s' "$meta" | awk -F'\t' '{print $2}') + src=$(printf '%s' "$meta" | awk -F'\t' '{print $3}') + dst=$(printf '%s' "$meta" | awk -F'\t' '{print $4}') + # Render time + if [ "$tm" -gt 100000000000 ] 2>/dev/null; then + tm_h=$(date -r $((tm/1000)) 2>/dev/null || date -d "@$((tm/1000))" 2>/dev/null || echo "$tm") + else + tm_h="$tm" + fi + printf '===== msg %d time=%s type=%s src=%s dst=%s =====\n' "$KEPT" "$tm_h" "$typ" "$src" "$dst" + tr '\r' '\n' < "$f" + printf '\n' + fi + done + printf 'nc-msgs: %d msgs scanned, %d match filters\n' "$MSG_COUNT" "$KEPT" >&2 + ;; + json) + printf '[' + first=1 + i=0 + for f in "$TMP_OUT"/msg_*.bin; do + i=$((i+1)) + if [ ${#FILTERS[@]} -eq 0 ] || match_filters "$f"; then + KEPT=$((KEPT+1)) + [ "$first" = "1" ] && first=0 || printf ',' + meta=$(cat "${TMP_OUT}/meta_$(printf '%05d' "$i").tsv") + tm=$(printf '%s' "$meta" | awk -F'\t' '{print $1}') + typ=$(printf '%s' "$meta" | awk -F'\t' '{print $2}') + src=$(printf '%s' "$meta" | awk -F'\t' '{print $3}') + dst=$(printf '%s' "$meta" | awk -F'\t' '{print $4}') + # Replace \r with \n in message content for JSON-safety, then JSON-escape + msg_text=$(tr '\r' '\n' < "$f" | jq -Rs .) + printf '{"time_ms":%s,"type":"%s","source":"%s","dest":"%s","content":%s}' \ + "$tm" "$typ" "$src" "$dst" "$msg_text" + fi + done + printf ']\n' + ;; +esac diff --git a/lib/nc-parse.sh b/lib/nc-parse.sh new file mode 100755 index 0000000..c95744d --- /dev/null +++ b/lib/nc-parse.sh @@ -0,0 +1,373 @@ +#!/usr/bin/env bash +# nc-parse.sh — first-class native Cloverleaf NetConfig parser for Larry-Anywhere v3. +# Pure bash + awk. No external tools. No v1/v2 dependencies. +# +# The NetConfig is a TCL-style nested-block file with two top-level declarations: +# - process { ... } — process containers +# - protocol { ... } — threads (the operational unit) +# +# This parser exposes structured access to those blocks. +# +# Usage: +# nc-parse.sh [args...] +# +# Subcommands: +# list-protocols — one protocol name per line +# list-processes — one process name per line +# protocol-line — line number where `protocol NAME {` appears +# protocol-block — emit the full TCL block for NAME +# protocol-field — emit value of top-level field for NAME +# (e.g. PROCESSNAME, OUTBOUNDONLY, OBWORKASIB) +# protocol-nested — drill into nested block, e.g. "PROTOCOL.PORT" +# protocol-summary [--all|--filter R] — TSV summary of all protocols with key fields +# destinations — list DEST values from DATAXLATE routing block +# xlate-refs [] — list xlate .xlt files referenced +# route-block — emit the DATAXLATE block (the routing config) +# help — this help +# +# Exit codes: 0 OK, 1 usage error, 2 not found, 3 parse error. +set -u +set -o pipefail + +NC_SELF="$0" + +die() { printf 'nc-parse: %s\n' "$*" >&2; exit 1; } + +require_file() { + [ -f "$1" ] || { printf 'nc-parse: not a file: %s\n' "$1" >&2; exit 2; } +} + +# ───────────────────────────────────────────────────────────────────────────── +# Core: emit each top-level block as record `TYPE\tNAME\tSTART_LINE\tEND_LINE` +# Robust to braces nested arbitrarily. +# ───────────────────────────────────────────────────────────────────────────── +_blocks() { + local nc="$1" + awk ' + BEGIN { depth=0; in_block=0; type=""; name=""; start=0 } + { + line = $0 + if (!in_block && line ~ /^(process|protocol) [A-Za-z0-9_]+ \{$/) { + split(line, a, " ") + type = a[1] + name = a[2] + start = NR + depth = 1 + in_block = 1 + next + } + if (in_block) { + # count unescaped { and } on this line + n_open = gsub(/\{/, "{", line) + n_close = gsub(/\}/, "}", line) + depth += n_open - n_close + if (depth == 0) { + printf "%s\t%s\t%d\t%d\n", type, name, start, NR + in_block = 0; type=""; name=""; start=0 + } + } + } + ' "$nc" +} + +cmd_list_protocols() { + local nc="$1" + require_file "$nc" + _blocks "$nc" | awk -F'\t' '$1=="protocol"{print $2}' +} + +cmd_list_processes() { + local nc="$1" + require_file "$nc" + _blocks "$nc" | awk -F'\t' '$1=="process"{print $2}' +} + +cmd_protocol_line() { + local nc="$1" name="$2" + require_file "$nc" + _blocks "$nc" | awk -F'\t' -v n="$name" '$1=="protocol" && $2==n {print $3}' +} + +cmd_protocol_block() { + local nc="$1" name="$2" + require_file "$nc" + local range; range=$(_blocks "$nc" | awk -F'\t' -v n="$name" '$1=="protocol" && $2==n {print $3","$4}') + [ -z "$range" ] && { printf 'nc-parse: no such protocol: %s\n' "$name" >&2; exit 2; } + awk -v range="$range" 'BEGIN{split(range,r,",")} NR>=r[1] && NR<=r[2]' "$nc" +} + +# Top-level fields are lines like: ` { FIELD value }` at depth 1 inside the protocol block. +# Strip surrounding `{ ... }` and emit value(s). +cmd_protocol_field() { + local nc="$1" name="$2" field="$3" + require_file "$nc" + cmd_protocol_block "$nc" "$name" \ + | awk -v F="$field" ' + BEGIN { depth = 0 } + { + line = $0 + n_open = gsub(/\{/, "{", line) + n_close = gsub(/\}/, "}", line) + # before applying deltas, the previous depth is what we use to test + # for "this line is a depth-1 field-statement" + prev = depth + depth += n_open - n_close + + # A top-level field is at indent depth==1 BEFORE this line opens any + # further blocks. It looks like: { FIELD value } + # (entire content on one line). We match exactly that. + if (prev == 1 && line ~ "^[[:space:]]+\\{ " F " ") { + # strip leading " { F " and trailing " }" + sub("^[[:space:]]+\\{ " F " ", "", line) + sub(" \\}$", "", line) + print line + } + } + ' +} + +# Drill into nested blocks. e.g. protocol-nested NAME PROTOCOL.PORT +# Walks the nested { KEY { ... } } structure. +cmd_protocol_nested() { + local nc="$1" name="$2" path="$3" + require_file "$nc" + IFS='.' read -ra parts <<< "$path" + local block; block=$(cmd_protocol_block "$nc" "$name") || return 1 + local current="$block" + local i + for ((i=0; i<${#parts[@]}; i++)); do + local key="${parts[$i]}" + if [ $((i+1)) -eq ${#parts[@]} ]; then + # Last part: extract scalar value. + # Baseline depth depends on whether we drilled (body has no wrapper, prev=0) + # or we're at the protocol-block level (has `protocol NAME {` wrapper, prev=1). + local baseline=1 + [ $i -gt 0 ] && baseline=0 + printf '%s\n' "$current" | awk -v K="$key" -v BASE="$baseline" ' + BEGIN { depth = 0 } + { + line = $0 + n_open = gsub(/\{/, "{", line) + n_close = gsub(/\}/, "}", line) + prev = depth + depth += n_open - n_close + if (prev == BASE && line ~ "^[[:space:]]+\\{ " K " ") { + sub("^[[:space:]]+\\{ " K " ", "", line) + sub(" \\}$", "", line) + print line + } + } + ' + else + # Drill: find `{ KEY {` opening, capture body until matching `} }` + current=$(printf '%s\n' "$current" | awk -v K="$key" ' + BEGIN { depth=0; capturing=0; cap_depth=0 } + { + line = $0 + n_open = gsub(/\{/, "{", line) + n_close = gsub(/\}/, "}", line) + if (!capturing && line ~ "^[[:space:]]+\\{ " K " \\{$") { + capturing = 1 + cap_depth = depth + 1 # the new opening { just hit + depth += n_open - n_close + next + } + if (capturing) { + depth += n_open - n_close + if (depth < cap_depth) { + capturing = 0 + exit + } + print + } else { + depth += n_open - n_close + } + } + ') + [ -z "$current" ] && { printf 'nc-parse: no nested key %s under %s\n' "$key" "$name" >&2; exit 2; } + fi + done +} + +# Compact one-line summary per protocol. TSV. +cmd_protocol_summary() { + local nc="$1"; shift + local filter="" + while [ $# -gt 0 ]; do + case "$1" in + --all) filter="" ;; + --filter) shift; filter="$1" ;; + *) die "unknown summary flag: $1" ;; + esac + shift + done + require_file "$nc" + + # Print TSV header + printf "name\tprocess\tdirection\tport\thost\ttype\tisserver\toutonly\tobworkasib\ticlserverport\n" + + local names + names=$(cmd_list_protocols "$nc") + local n + for n in $names; do + if [ -n "$filter" ] && ! printf '%s' "$n" | grep -Eq -- "$filter"; then + continue + fi + local pname obib outonly iclserv ptype phost pport isserver direction + pname=$(cmd_protocol_field "$nc" "$n" PROCESSNAME | head -1) + obib=$(cmd_protocol_field "$nc" "$n" OBWORKASIB | head -1) + outonly=$(cmd_protocol_field "$nc" "$n" OUTBOUNDONLY | head -1) + iclserv=$(cmd_protocol_field "$nc" "$n" ICLSERVERPORT | head -1) + ptype=$(cmd_protocol_nested "$nc" "$n" PROTOCOL.TYPE 2>/dev/null | head -1) + phost=$(cmd_protocol_nested "$nc" "$n" PROTOCOL.HOST 2>/dev/null | head -1) + pport=$(cmd_protocol_nested "$nc" "$n" PROTOCOL.PORT 2>/dev/null | head -1) + isserver=$(cmd_protocol_nested "$nc" "$n" PROTOCOL.ISSERVER 2>/dev/null | head -1) + + # Direction inference + if [ "$isserver" = "1" ]; then + direction="inbound-tcp-listen" + elif [ "$obib" = "1" ]; then + direction="inbound-icl-or-file" + elif [ "$outonly" = "1" ]; then + direction="outbound" + else + direction="unknown" + fi + + # Clean braces from values + phost=$(printf '%s' "$phost" | sed 's/^{}$//; s/^{//; s/}$//') + pport=$(printf '%s' "$pport" | sed 's/^{}$//; s/^{//; s/}$//') + + printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n" \ + "$n" "${pname:-}" "$direction" "${pport:-}" "${phost:-}" \ + "${ptype:-}" "${isserver:-}" "${outonly:-}" "${obib:-}" "${iclserv:-}" + done +} + +# Destinations: walk DATAXLATE > ROUTE_DETAILS > { DEST } +cmd_destinations() { + local nc="$1" name="$2" + require_file "$nc" + cmd_protocol_block "$nc" "$name" \ + | awk ' + /\{ DEST [A-Za-z0-9_]+ \}/ { + sub(/.*\{ DEST /, "") + sub(/ \}.*$/, "") + print + } + ' | sort -u +} + +# Xlate refs: every X.xlt name appearing in the protocol's block (or all if no name) +cmd_xlate_refs() { + local nc="$1" name="${2:-}" + require_file "$nc" + if [ -n "$name" ]; then + cmd_protocol_block "$nc" "$name" | grep -oE '[A-Za-z0-9_]+\.xlt' | sort -u + else + grep -oE '[A-Za-z0-9_]+\.xlt' "$nc" | sort -u + fi +} + +# Sources: every protocol that has `{ DEST }` in its body. +# Slower than _blocks because it scans each protocol's body, but for a 48-thread +# site it's still sub-second. +cmd_sources() { + local nc="$1" target="$2" + require_file "$nc" + local names; names=$(cmd_list_protocols "$nc") + local n + for n in $names; do + [ "$n" = "$target" ] && continue + if cmd_protocol_block "$nc" "$n" 2>/dev/null | grep -qE "\\{ DEST $target \\}"; then + printf '%s\n' "$n" + fi + done +} + +# Tclproc references — extract every TCL proc name referenced in this protocol's +# block (DATAFORMAT.PROC singletons + PROCS clauses with one or more names). +# Excludes empty {} and the bare keyword PROCSCONTROL. +cmd_tclproc_refs() { + local nc="$1" name="${2:-}" + require_file "$nc" + local body + if [ -n "$name" ]; then + body=$(cmd_protocol_block "$nc" "$name" 2>/dev/null) + else + body=$(cat "$nc") + fi + printf '%s\n' "$body" | awk ' + { + line = $0 + # PROC (singleton, e.g. DATAFORMAT.PROC) + if (match(line, /\{ PROC [A-Za-z_][A-Za-z0-9_]*/)) { + v = substr(line, RSTART + 7, RLENGTH - 7) + print v + } + # PROCS (singleton) + if (match(line, /\{ PROCS [A-Za-z_][A-Za-z0-9_]*/)) { + v = substr(line, RSTART + 8, RLENGTH - 8) + print v + } + # PROCS { name1 name2 ... } (list — rare but possible) + if (match(line, /\{ PROCS \{ [^}]+\}/)) { + v = substr(line, RSTART + 9, RLENGTH - 9) + sub(/ *\}$/, "", v) + n = split(v, arr, /[ \t]+/) + for (i=1; i<=n; i++) if (arr[i] != "") print arr[i] + } + } + ' | sort -u | grep -v '^$' +} + +cmd_route_block() { + local nc="$1" name="$2" + require_file "$nc" + cmd_protocol_block "$nc" "$name" \ + | awk ' + BEGIN { depth=0; capturing=0; cap_depth=0 } + { + line = $0 + n_open = gsub(/\{/, "{", line) + n_close = gsub(/\}/, "}", line) + if (!capturing && line ~ /^[[:space:]]+\{ DATAXLATE \{$/) { + capturing = 1 + cap_depth = depth + 1 + print + depth += n_open - n_close + next + } + if (capturing) { + print + depth += n_open - n_close + if (depth < cap_depth) exit + } else { + depth += n_open - n_close + } + } + ' +} + +cmd_help() { sed -n '2,30p' "$NC_SELF"; } + +# ───────────────────────────────────────────────────────────────────────────── +# Dispatch +# ───────────────────────────────────────────────────────────────────────────── +SUB="${1:-help}" +case "$SUB" in + list-protocols) [ $# -ge 2 ] || die "usage: $0 list-protocols "; cmd_list_protocols "$2" ;; + list-processes) [ $# -ge 2 ] || die "usage: $0 list-processes "; cmd_list_processes "$2" ;; + protocol-line) [ $# -ge 3 ] || die "usage: $0 protocol-line "; cmd_protocol_line "$2" "$3" ;; + protocol-block) [ $# -ge 3 ] || die "usage: $0 protocol-block "; cmd_protocol_block "$2" "$3" ;; + protocol-field) [ $# -ge 4 ] || die "usage: $0 protocol-field "; cmd_protocol_field "$2" "$3" "$4" ;; + protocol-nested) [ $# -ge 4 ] || die "usage: $0 protocol-nested "; cmd_protocol_nested "$2" "$3" "$4" ;; + protocol-summary) [ $# -ge 2 ] || die "usage: $0 protocol-summary [--filter REGEX]"; cmd_protocol_summary "$2" "${@:3}" ;; + destinations) [ $# -ge 3 ] || die "usage: $0 destinations "; cmd_destinations "$2" "$3" ;; + sources) [ $# -ge 3 ] || die "usage: $0 sources "; cmd_sources "$2" "$3" ;; + xlate-refs) [ $# -ge 2 ] || die "usage: $0 xlate-refs [name]"; cmd_xlate_refs "$2" "${3:-}" ;; + tclproc-refs) [ $# -ge 2 ] || die "usage: $0 tclproc-refs [name]"; cmd_tclproc_refs "$2" "${3:-}" ;; + route-block) [ $# -ge 3 ] || die "usage: $0 route-block "; cmd_route_block "$2" "$3" ;; + help|-h|--help) cmd_help ;; + *) die "unknown subcommand: $SUB (try '$0 help')" ;; +esac diff --git a/lib/nc-regression.sh b/lib/nc-regression.sh new file mode 100755 index 0000000..c35f611 --- /dev/null +++ b/lib/nc-regression.sh @@ -0,0 +1,332 @@ +#!/usr/bin/env bash +# nc-regression.sh — Example 6 orchestrator: end-to-end regression testing +# between two Cloverleaf environments. +# +# Phases: +# 1. discover → list inbound threads in scope (uses nc-find-inbound) +# 2. sample → grab N messages per inbound from env-A smatdb → input .msgs files +# 3. route-A → run route_test on env-A for each inbound → captured outputs/env-a/... +# 4. route-B → run route_test on env-B with same inputs → captured outputs/env-b/... +# 5. diff → hl7-diff every paired output file with --ignore MSH.7 → per-pair report +# 6. summary → one master regression-summary.md compiling everything +# +# Phases 3 and 4 require Cloverleaf's route_test command on each box. The +# command is parameterized via --route-test-cmd with placeholders: +# {THREAD} → the inbound thread name +# {INPUT} → absolute path to the .msgs input file +# {OUTPUT_DIR} → absolute path where output files should land +# {HCIROOT} → the env's HCIROOT +# {HCISITE} → the env's HCISITE +# Default: not set — you must pass it once for your shop's invocation pattern. +# +# A common pattern: a wrapper script that sources the Cloverleaf profile and +# runs ` route_test `, with output redirected to OUTPUT_DIR. +# Example template you might pass: +# --route-test-cmd 'cd {HCIROOT}/{HCISITE} && . ./.profile && {THREAD} route_test {INPUT} && cp *.out.* {OUTPUT_DIR}/' +# +# Usage: +# nc-regression.sh --scope +# --count N +# --env-a HCIROOT_A --site-a SITENAME +# --env-b HCIROOT_B --site-b SITENAME +# --out DIR +# --route-test-cmd 'TEMPLATE' +# [--ignore "FIELDS"] +# [--include-fields "FIELDS"] +# [--phase 1|2|3|4|5|6|all] +# [--dry-run] +# [--inbound-mode tcp-listen|icl-or-file|all] +# [--env-b-host HOST] [--env-b-user USER] # for scp of inputs +# +# Scope formats: +# thread:NAME one specific thread +# threads:N1,N2,N3 comma-separated list +# site every inbound in the configured site +# server every inbound in every site under HCIROOT +set -o pipefail + +NC_SELF="$0" +LIB_DIR="$(cd "$(dirname "$NC_SELF")" && pwd)" +NCP="$LIB_DIR/nc-parse.sh" +NCI="$LIB_DIR/nc-inbound.sh" +NCM="$LIB_DIR/nc-msgs.sh" +HL7DIFF="$LIB_DIR/hl7-diff.sh" + +die() { printf 'nc-regression: %s\n' "$*" >&2; exit 1; } +say() { printf 'nc-regression: %s\n' "$*" >&2; } + +SCOPE="" +COUNT=10 +ENV_A="" +SITE_A="" +ENV_B="" +SITE_B="" +OUT="" +ROUTE_TEST_CMD="" +IGNORE="MSH.7" +INCLUDE="" +PHASE="all" +DRY_RUN=0 +INBOUND_MODE="all" +ENV_B_HOST="" +ENV_B_USER="" + +while [ $# -gt 0 ]; do + case "$1" in + --scope) shift; SCOPE="$1" ;; + --count) shift; COUNT="$1" ;; + --env-a) shift; ENV_A="$1" ;; + --site-a) shift; SITE_A="$1" ;; + --env-b) shift; ENV_B="$1" ;; + --site-b) shift; SITE_B="$1" ;; + --out) shift; OUT="$1" ;; + --route-test-cmd) shift; ROUTE_TEST_CMD="$1" ;; + --ignore) shift; IGNORE="$1" ;; + --include-fields) shift; INCLUDE="$1" ;; + --phase) shift; PHASE="$1" ;; + --dry-run) DRY_RUN=1 ;; + --inbound-mode) shift; INBOUND_MODE="$1" ;; + --env-b-host) shift; ENV_B_HOST="$1" ;; + --env-b-user) shift; ENV_B_USER="$1" ;; + -h|--help) sed -n '2,55p' "$NC_SELF"; exit 0 ;; + -*) die "unknown flag: $1" ;; + *) die "extra arg: $1" ;; + esac + shift +done + +[ -n "$SCOPE" ] || die "missing --scope (thread:NAME | threads:N1,N2 | site | server)" +[ -n "$ENV_A" ] || die "missing --env-a HCIROOT_A" +[ -n "$ENV_B" ] || die "missing --env-b HCIROOT_B" +[ -n "$OUT" ] || die "missing --out DIR" +[ -d "$ENV_A" ] || die "env-a is not a directory: $ENV_A" +case "$PHASE" in 1|2|3|4|5|6|all) ;; *) die "bad --phase" ;; esac +[ "$DRY_RUN" = "1" ] || [ -n "$ROUTE_TEST_CMD" ] || say "WARNING: --route-test-cmd is unset; phases 3 and 4 will be skipped (you can run them manually using the generated input files)" + +mkdir -p "$OUT" "$OUT/inputs" "$OUT/outputs/env-a" "$OUT/outputs/env-b" "$OUT/diff" 2>/dev/null + +# ───────────────────────────────────────────────────────────────────────────── +# Phase 1: discover inbound threads in scope +# ───────────────────────────────────────────────────────────────────────────── +discover_inbounds() { + case "$SCOPE" in + thread:*) echo "${SCOPE#thread:}" ;; + threads:*) echo "${SCOPE#threads:}" | tr ',' '\n' ;; + site) + [ -n "$SITE_A" ] || die "scope=site requires --site-a" + "$NCI" "$ENV_A/$SITE_A/NetConfig" --mode "$INBOUND_MODE" --format tsv \ + | awk -F'\t' 'NR>1 {print $1}' + ;; + server) + while IFS= read -r nc; do + "$NCI" "$nc" --mode "$INBOUND_MODE" --format tsv \ + | awk -F'\t' 'NR>1 {print $1}' + done < <(find "$ENV_A" -maxdepth 2 -name NetConfig -type f 2>/dev/null) + ;; + *) die "bad --scope: $SCOPE" ;; + esac +} + +phase_1() { + say "=== PHASE 1: discover inbounds ===" + local inbounds; inbounds=$(discover_inbounds | sort -u | grep -v '^$') + if [ -z "$inbounds" ]; then + say "no inbounds found in scope $SCOPE" + return 1 + fi + printf '%s\n' "$inbounds" > "$OUT/inbounds.txt" + local count; count=$(printf '%s\n' "$inbounds" | wc -l | tr -d ' ') + say "discovered $count inbound thread(s) → $OUT/inbounds.txt" + printf '%s\n' "$inbounds" | sed 's/^/ - /' +} + +# ───────────────────────────────────────────────────────────────────────────── +# Phase 2: sample N messages per inbound from env-A's smatdbs +# ───────────────────────────────────────────────────────────────────────────── +phase_2() { + say "=== PHASE 2: sample $COUNT messages per inbound from env-A ===" + [ -f "$OUT/inbounds.txt" ] || { say "no inbounds file from phase 1; running phase 1 first"; phase_1 || return 1; } + local thread sitedir + local count=0 + while IFS= read -r thread; do + [ -z "$thread" ] && continue + # Locate the site for this thread + sitedir="" + if [ -n "$SITE_A" ]; then + sitedir="$ENV_A/$SITE_A" + else + # Search across all sites under ENV_A for the smatdb + sitedir=$(find "$ENV_A" -maxdepth 4 -name "${thread}.smatdb" -type f 2>/dev/null | head -1 | xargs -I{} dirname {} | sed "s#/exec/processes/.*##") + [ -z "$sitedir" ] && { say " skip $thread: smatdb not found under $ENV_A"; continue; } + fi + local input="$OUT/inputs/${thread}.msgs" + if [ "$DRY_RUN" = "1" ]; then + say " [dry-run] would sample $COUNT msgs from $thread → $input" + else + HCISITEDIR="$sitedir" "$NCM" "$thread" --limit "$COUNT" --format raw > "$input" 2>/dev/null + local got; got=$(tr -cd $'\x1c' < "$input" | wc -c | tr -d ' ') + say " sampled $thread → $input ($got messages)" + fi + count=$((count+1)) + done < "$OUT/inbounds.txt" + say "phase 2 done: $count thread(s) processed" +} + +# ───────────────────────────────────────────────────────────────────────────── +# Phase 3 / 4: execute route_test on each env +# ───────────────────────────────────────────────────────────────────────────── +render_cmd() { + local tmpl="$1" thread="$2" input="$3" outdir="$4" hciroot="$5" hcisite="$6" + local cmd="$tmpl" + cmd="${cmd//\{THREAD\}/$thread}" + cmd="${cmd//\{INPUT\}/$input}" + cmd="${cmd//\{OUTPUT_DIR\}/$outdir}" + cmd="${cmd//\{HCIROOT\}/$hciroot}" + cmd="${cmd//\{HCISITE\}/$hcisite}" + printf '%s' "$cmd" +} + +phase_routes() { + local label="$1" hciroot="$2" hcisite="$3" + say "=== PHASE ${label}: route_test on env-${label} ===" + if [ -z "$ROUTE_TEST_CMD" ]; then + say "no --route-test-cmd; skipping phase ${label}" + say "to run manually, use the input files at $OUT/inputs/*.msgs" + return 0 + fi + local thread + while IFS= read -r thread; do + [ -z "$thread" ] && continue + local input="$OUT/inputs/${thread}.msgs" + local outdir="$OUT/outputs/env-${label}/${thread}" + [ -f "$input" ] || { say " skip $thread: no input file"; continue; } + mkdir -p "$outdir" + local cmd; cmd=$(render_cmd "$ROUTE_TEST_CMD" "$thread" "$input" "$outdir" "$hciroot" "$hcisite") + if [ "$DRY_RUN" = "1" ]; then + say " [dry-run] $thread:" + say " \$ $cmd" + else + say " $thread:" + say " \$ $cmd" + bash -c "$cmd" 2>&1 | sed 's/^/ /' || say " (route_test exit non-zero — continuing)" + fi + done < "$OUT/inbounds.txt" +} + +phase_3() { phase_routes "a" "$ENV_A" "$SITE_A"; } + +phase_4() { + # Phase 4: copy inputs to env-b host (if remote), then route_test on env-B. + if [ -n "$ENV_B_HOST" ]; then + say "copying input files to ${ENV_B_USER:-$USER}@${ENV_B_HOST}:${OUT}/inputs/" + if [ "$DRY_RUN" = "1" ]; then + say " [dry-run] scp -r $OUT/inputs/ ${ENV_B_USER:-$USER}@${ENV_B_HOST}:${OUT}/" + else + ssh "${ENV_B_USER:-$USER}@${ENV_B_HOST}" "mkdir -p $OUT/inputs $OUT/outputs/env-b" || true + scp -r "$OUT/inputs/" "${ENV_B_USER:-$USER}@${ENV_B_HOST}:${OUT}/" || say "scp failed; you'll need to copy manually" + fi + fi + phase_routes "b" "$ENV_B" "$SITE_B" +} + +# ───────────────────────────────────────────────────────────────────────────── +# Phase 5: diff outputs pair-by-pair +# ───────────────────────────────────────────────────────────────────────────── +phase_5() { + say "=== PHASE 5: diff env-a vs env-b outputs ===" + local diff_index="$OUT/diff/_index.md" + { + printf '# Regression diff index\n\n' + printf '- env-A: `%s`\n- env-B: `%s`\n- scope: `%s`\n- count: %s msgs per inbound\n- ignore: `%s`\n%s\n\n' \ + "$ENV_A" "$ENV_B" "$SCOPE" "$COUNT" "$IGNORE" \ + "$([ -n "$INCLUDE" ] && printf -- '- include-only: `%s`' "$INCLUDE")" + printf '| thread | dest | diffs | report |\n|---|---|---|---|\n' + } > "$diff_index" + + local total_diff=0 total_pairs=0 + local thread destfile destname + while IFS= read -r thread; do + [ -z "$thread" ] && continue + local a_dir="$OUT/outputs/env-a/${thread}" + local b_dir="$OUT/outputs/env-b/${thread}" + [ -d "$a_dir" ] || { say " skip $thread: no env-a outputs"; continue; } + [ -d "$b_dir" ] || { say " skip $thread: no env-b outputs"; continue; } + # Pair up by filename + while IFS= read -r destfile; do + destname=$(basename "$destfile") + local b_pair="$b_dir/$destname" + total_pairs=$((total_pairs+1)) + if [ ! -f "$b_pair" ]; then + echo "| \`$thread\` | \`$destname\` | (missing on env-b) | — |" >> "$diff_index" + continue + fi + local report="$OUT/diff/${thread}.${destname}.md" + local count + if [ "$DRY_RUN" = "1" ]; then + echo " [dry-run] would diff $destfile vs $b_pair → $report" + echo "| \`$thread\` | \`$destname\` | [dry-run] | — |" >> "$diff_index" + continue + fi + local diff_args=(--ignore "$IGNORE") + [ -n "$INCLUDE" ] && diff_args+=(--include-fields "$INCLUDE") + count=$("$HL7DIFF" "${diff_args[@]}" --format count "$destfile" "$b_pair" 2>/dev/null || echo "?") + { + printf '# Diff: %s → %s\n\n- env-A: `%s`\n- env-B: `%s`\n\n' "$thread" "$destname" "$destfile" "$b_pair" + "$HL7DIFF" "${diff_args[@]}" --format text "$destfile" "$b_pair" 2>/dev/null || true + } > "$report" + echo "| \`$thread\` | \`$destname\` | $count | [report](./$(basename "$report")) |" >> "$diff_index" + total_diff=$((total_diff + count)) + say " $thread → $destname: $count diff(s)" + done < <(find "$a_dir" -maxdepth 1 -type f 2>/dev/null) + done < "$OUT/inbounds.txt" + + { + printf '\n## Summary\n\n- pairs compared: %s\n- total field differences (post-ignore): %s\n' "$total_pairs" "$total_diff" + } >> "$diff_index" + say "phase 5 done: $total_pairs pairs compared, $total_diff total diffs" + say "index: $diff_index" +} + +# ───────────────────────────────────────────────────────────────────────────── +# Phase 6: master summary +# ───────────────────────────────────────────────────────────────────────────── +phase_6() { + say "=== PHASE 6: master regression-summary.md ===" + local summary="$OUT/regression-summary.md" + { + printf '# Regression test summary\n\n' + printf 'Generated: %s\n\n' "$(date -Iseconds 2>/dev/null || date)" + printf '## Configuration\n\n' + printf '- scope: `%s`\n' "$SCOPE" + printf '- count: %s messages per inbound\n' "$COUNT" + printf '- env-A: `%s` (site=%s)\n' "$ENV_A" "${SITE_A:-auto}" + printf '- env-B: `%s` (site=%s)\n' "$ENV_B" "${SITE_B:-auto}" + printf '- ignore: `%s`\n' "$IGNORE" + [ -n "$INCLUDE" ] && printf '- include-only: `%s`\n' "$INCLUDE" + printf '\n## Inbounds tested\n\n' + [ -f "$OUT/inbounds.txt" ] && awk '{print "- `" $0 "`"}' "$OUT/inbounds.txt" + printf '\n## Inputs\n\n' + find "$OUT/inputs" -maxdepth 1 -type f 2>/dev/null \ + | awk '{print "- `" $0 "`"}' || true + printf '\n## Output directories\n\n' + printf -- '- env-A: `%s`\n' "$OUT/outputs/env-a" + printf -- '- env-B: `%s`\n' "$OUT/outputs/env-b" + printf '\n## Diff details\n\n' + printf 'See [diff/_index.md](./diff/_index.md) for the per-pair table.\n' + } > "$summary" + say "summary: $summary" +} + +# ───────────────────────────────────────────────────────────────────────────── +# Dispatch +# ───────────────────────────────────────────────────────────────────────────── +case "$PHASE" in + 1) phase_1 ;; + 2) phase_1 && phase_2 ;; + 3) phase_3 ;; + 4) phase_4 ;; + 5) phase_5 ;; + 6) phase_6 ;; + all) phase_1 && phase_2 && phase_3 && phase_4 && phase_5 && phase_6 ;; +esac +say "regression run done. Output root: $OUT"