From 3eb88f86c8af0d57370f972c69497c7c3c7fa194 Mon Sep 17 00:00:00 2001 From: Bryan Johnson Date: Tue, 26 May 2026 11:05:19 -0700 Subject: [PATCH] v0.4.1: each / each-site / len2nl / csv-to-table / table-to-csv MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five small Unix-style loop & format helpers, fully offline: lib/each.sh - replaces v1 `each` - run a CMD per item: args, stdin lines, or {}-placeholder substitution - example: tbn adt | awk '{print $2}' | each.sh 'route_test {}' lib/each-site.sh - replaces v1 each_site / each_site_hdr / each_site_tcl patterns - iterates every site under $HCIROOT with HCISITE/HCISITEDIR auto-exported - --filter REGEX limits which sites; --hdr prints a header before each lib/len2nl.sh - replaces v1 `len2nl` - strict superset: handles length-prefixed (digits before MSH), MLLP (\x0b...\x1c\x0d), and segment CRs (→ LF) - works as stdin filter or with file arg lib/csv-to-table.sh - 2-column CSV → Cloverleaf .tbl format - emits proper prologue (who, date, bidir, type, version) - --has-header --default VALUE --bidir 0|1 --in-delim CHAR --user NAME --out PATH lib/table-to-csv.sh - reverse: .tbl → CSV - --with-header --delim CHAR --include-meta - confirmed clean round-trip: CSV → table → CSV byte-identical for the data rows All 5 are pipeable, have --help, zero external deps beyond bash+awk+sed. Co-Authored-By: Claude Opus 4.7 --- VERSION | 2 +- install-larry.sh | 5 +++ larry.sh | 2 +- lib/csv-to-table.sh | 102 ++++++++++++++++++++++++++++++++++++++++++++ lib/each-site.sh | 54 +++++++++++++++++++++++ lib/each.sh | 45 +++++++++++++++++++ lib/len2nl.sh | 38 +++++++++++++++++ lib/table-to-csv.sh | 70 ++++++++++++++++++++++++++++++ 8 files changed, 316 insertions(+), 2 deletions(-) create mode 100755 lib/csv-to-table.sh create mode 100755 lib/each-site.sh create mode 100755 lib/each.sh create mode 100755 lib/len2nl.sh create mode 100755 lib/table-to-csv.sh diff --git a/VERSION b/VERSION index 1d0ba9e..267577d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.4.0 +0.4.1 diff --git a/install-larry.sh b/install-larry.sh index caa8368..54932e2 100755 --- a/install-larry.sh +++ b/install-larry.sh @@ -94,6 +94,11 @@ fetch lib/oauth.sh "$LARRY_HOME/lib/oauth.sh" fetch lib/lessons.sh "$LARRY_HOME/lib/lessons.sh" fetch lib/hl7-sanitize.sh "$LARRY_HOME/lib/hl7-sanitize.sh" fetch lib/hl7-desanitize.sh "$LARRY_HOME/lib/hl7-desanitize.sh" +fetch lib/each.sh "$LARRY_HOME/lib/each.sh" +fetch lib/each-site.sh "$LARRY_HOME/lib/each-site.sh" +fetch lib/len2nl.sh "$LARRY_HOME/lib/len2nl.sh" +fetch lib/csv-to-table.sh "$LARRY_HOME/lib/csv-to-table.sh" +fetch lib/table-to-csv.sh "$LARRY_HOME/lib/table-to-csv.sh" fetch lib/nc-parse.sh "$LARRY_HOME/lib/nc-parse.sh" fetch lib/nc-inbound.sh "$LARRY_HOME/lib/nc-inbound.sh" fetch lib/nc-make-jump.sh "$LARRY_HOME/lib/nc-make-jump.sh" diff --git a/larry.sh b/larry.sh index 0732d90..946d894 100755 --- a/larry.sh +++ b/larry.sh @@ -32,7 +32,7 @@ set -o pipefail # ───────────────────────────────────────────────────────────────────────────── # Config # ───────────────────────────────────────────────────────────────────────────── -LARRY_VERSION="0.4.0" +LARRY_VERSION="0.4.1" LARRY_HOME="${LARRY_HOME:-$HOME/.larry}" LARRY_UPDATE_URL="${LARRY_UPDATE_URL:-https://raw.githubusercontent.com/bojj27/cloverleaf-larry/main/larry.sh}" LARRY_AGENTS_URL="${LARRY_AGENTS_URL:-https://raw.githubusercontent.com/bojj27/cloverleaf-larry/main/agents}" diff --git a/lib/csv-to-table.sh b/lib/csv-to-table.sh new file mode 100755 index 0000000..0416e44 --- /dev/null +++ b/lib/csv-to-table.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +# csv-to-table.sh — convert a 2-column CSV into Cloverleaf .tbl format. +# +# Input CSV (with or without header): +# input,output +# 12345,M +# 12346,F +# ... +# +# Output Cloverleaf .tbl: +# # Generated by csv-to-table.sh +# prologue +# who: +# date: +# outname: output +# inname: input +# bidir: 0 +# type: tbl +# version: 7.0 +# end_prologue +# dflt=__non-existent__ +# # +# 12345 +# M +# encoded=0,0 +# # +# 12346 +# F +# ... +# +# Usage: +# csv-to-table.sh [FILE] [--default VALUE] [--bidir 0|1] [--has-header] +# [--in-delim CHAR] # default ',' +# [--user NAME] # default $USER +# [--out PATH] # default stdout +set -o pipefail + +usage() { sed -n '2,30p' "$0"; exit 0; } + +INPUT="" +DEFAULT="__non-existent__" +BIDIR=0 +HAS_HEADER=0 +DELIM=',' +USER_NAME="${USER:-larry}" +OUT_FILE="" + +while [ $# -gt 0 ]; do + case "$1" in + --default) shift; DEFAULT="$1" ;; + --bidir) shift; BIDIR="$1" ;; + --has-header) HAS_HEADER=1 ;; + --in-delim) shift; DELIM="$1" ;; + --user) shift; USER_NAME="$1" ;; + --out) shift; OUT_FILE="$1" ;; + -h|--help) usage ;; + -*) echo "csv-to-table: unknown flag: $1" >&2; exit 2 ;; + *) INPUT="$1" ;; + esac + shift +done + +[ -z "$INPUT" ] || [ -f "$INPUT" ] || { echo "csv-to-table: no such file: $INPUT" >&2; exit 2; } + +emit() { + cat </dev/null || date) + outname: output + inname: input + bidir: ${BIDIR} + type: tbl + version: 7.0 +end_prologue +# +dflt=${DEFAULT} +# +EOF + awk -F"$DELIM" -v has_header="$HAS_HEADER" ' + NR == 1 && has_header == 1 { next } + NF >= 2 { + # Trim whitespace + gsub(/^[ \t"]+|[ \t"]+$/, "", $1) + gsub(/^[ \t"]+|[ \t"]+$/, "", $2) + if ($1 == "") next + print $1 + print $2 + print "encoded=0,0" + print "#" + } + ' "${INPUT:-/dev/stdin}" +} + +if [ -n "$OUT_FILE" ]; then + mkdir -p "$(dirname "$OUT_FILE")" 2>/dev/null + emit > "$OUT_FILE" + printf 'csv-to-table: wrote %s\n' "$OUT_FILE" >&2 +else + emit +fi diff --git a/lib/each-site.sh b/lib/each-site.sh new file mode 100755 index 0000000..da45e7b --- /dev/null +++ b/lib/each-site.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# each-site.sh — run a command for each site under $HCIROOT. +# Replaces v1 `each_site` / `each_site_hdr` / `each_site_tcl` patterns. +# +# Usage: +# each-site.sh [--hciroot DIR] [--hdr] [--filter REGEX] +# +# The current site is exposed to CMD as the env vars HCISITE and HCISITEDIR. +# The literal token {} in CMD, if present, is replaced with the site name. +# +# Examples: +# each-site.sh 'lib/nc-find.sh --name adt --netconfigs "$HCISITEDIR/NetConfig"' +# each-site.sh --hdr 'echo "Hey from $HCISITE; netconfig=$HCISITEDIR/NetConfig"' +# each-site.sh --filter '^(ancout|epic)$' 'wc -l "$HCISITEDIR/NetConfig"' +# each-site.sh 'lib/nc-parse.sh list-protocols "$HCISITEDIR/NetConfig" | wc -l' +# +# --hdr prints a "===== =====" header before each run. +set -o pipefail + +HCIROOT_OVR="" +HEADER=0 +FILTER="" + +while [ $# -gt 0 ]; do + case "$1" in + --hciroot) shift; HCIROOT_OVR="$1" ;; + --hdr) HEADER=1 ;; + --filter) shift; FILTER="$1" ;; + -h|--help) sed -n '2,18p' "$0"; exit 0 ;; + --) shift; break ;; + -*) echo "each-site: unknown flag: $1" >&2; exit 2 ;; + *) break ;; + esac + shift +done +[ $# -ge 1 ] || { echo "each-site: needs a CMD" >&2; exit 2; } +CMD="$*" + +ROOT="${HCIROOT_OVR:-${HCIROOT:-}}" +[ -n "$ROOT" ] && [ -d "$ROOT" ] || { echo "each-site: no \$HCIROOT (or --hciroot)" >&2; exit 2; } + +# Site = subdir of ROOT that contains a NetConfig file. +SITES=$(find "$ROOT" -mindepth 1 -maxdepth 2 -name NetConfig -type f 2>/dev/null \ + | xargs -I{} dirname {} 2>/dev/null \ + | xargs -I{} basename {} 2>/dev/null \ + | sort -u) +[ -n "$SITES" ] || { echo "each-site: no sites with NetConfig under $ROOT" >&2; exit 1; } + +while IFS= read -r site; do + [ -z "$site" ] && continue + if [ -n "$FILTER" ] && ! printf '%s' "$site" | grep -Eq -- "$FILTER"; then continue; fi + [ "$HEADER" = "1" ] && printf '\n===== %s =====\n' "$site" + HCISITE="$site" HCISITEDIR="$ROOT/$site" eval "${CMD//\{\}/$site}" +done <<< "$SITES" diff --git a/lib/each.sh b/lib/each.sh new file mode 100755 index 0000000..a6b9d74 --- /dev/null +++ b/lib/each.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# each.sh — run a command for each item. Replaces v1 `each`. +# +# Two input modes: +# each.sh [ARG2 ... ARGN] — args as items +# | each.sh — stdin lines as items (one per line) +# +# The literal token {} in CMD, if present, gets replaced with the current item. +# Otherwise the item is appended as the last arg. +# +# Examples: +# each.sh 'echo got:' /etc/hosts /etc/passwd +# tbn adt | awk 'NR>1{print $2}' | each.sh 'echo route_test for' {} +# nc-find.sh --process codametrix --format tsv \ +# | awk -F'\t' 'NR>1{print $2}' \ +# | each.sh 'lib/nc-msgs.sh' {} '--limit' '1' '--format' 'count' +# +# Each invocation is a separate process; failures are not aggregated. To +# stop on first error, prepend `set -e` in your own wrapper. +set -o pipefail + +usage() { sed -n '2,20p' "$0"; exit 0; } +[ $# -ge 1 ] || usage +case "$1" in -h|--help) usage ;; esac + +CMD="$1"; shift + +run_one() { + local item="$1" + if [[ "$CMD" == *"{}"* ]]; then + eval "${CMD//\{\}/\"\$item\"}" + else + # Append item as last arg + eval "$CMD \"\$item\"" + fi +} + +if [ $# -gt 0 ]; then + for item in "$@"; do run_one "$item"; done +else + while IFS= read -r item; do + [ -z "$item" ] && continue + run_one "$item" + done +fi diff --git a/lib/len2nl.sh b/lib/len2nl.sh new file mode 100755 index 0000000..e70b9f2 --- /dev/null +++ b/lib/len2nl.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# len2nl.sh — convert length-prefixed and/or MLLP-framed HL7 to newline-readable. +# +# Handles both common framing patterns: +# 1. Length-prefix: `MSH|...` → strip leading digits before MSH +# 2. MLLP: `\x0b...\x1c\x0d` → strip VT and FS+CR end-block +# 3. Segment CRs: `\r\r...` → replace \r with \n +# +# After conversion, each segment is on its own line (LF), making the message +# greppable / pageable. +# +# Usage: +# len2nl.sh [FILE] # file or stdin +# cat raw.hl7 | len2nl.sh +# tcpdump ... | len2nl.sh +# +# This is a strict superset of the v1 `len2nl` behavior. +set -o pipefail + +usage() { sed -n '2,18p' "$0"; exit 0; } +case "${1:-}" in -h|--help) usage ;; esac + +INPUT="${1:-}" +if [ -n "$INPUT" ] && [ ! -f "$INPUT" ]; then + echo "len2nl: no such file: $INPUT" >&2; exit 2 +fi + +if [ -n "$INPUT" ]; then + CMD=(cat "$INPUT") +else + CMD=(cat) +fi + +"${CMD[@]}" \ + | tr -d '\013' `# strip MLLP VT (0x0B = start-block)` \ + | sed 's/\x1c\x0d//g' `# strip MLLP FS+CR (0x1C 0x0D = end-block)` \ + | sed -E 's/[0-9]+MSH\|/MSH|/g' `# strip numeric length-prefix before MSH` \ + | tr '\r' '\n' `# CR-separated segments → LF` diff --git a/lib/table-to-csv.sh b/lib/table-to-csv.sh new file mode 100755 index 0000000..f5b6d4c --- /dev/null +++ b/lib/table-to-csv.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# table-to-csv.sh — convert a Cloverleaf .tbl file to CSV. +# +# Reverses csv-to-table.sh. Skips the prologue, dflt= line, encoded= meta, +# comment lines, and #-separator lines; emits "input,output" rows. +# +# Usage: +# table-to-csv.sh [FILE] # file or stdin +# table-to-csv.sh --with-header [FILE] # emit "input,output" header +# table-to-csv.sh --delim ';' [FILE] # output delimiter (default ',') +# table-to-csv.sh --include-meta [FILE] # also emit prologue/default rows +set -o pipefail + +usage() { sed -n '2,12p' "$0"; exit 0; } + +INPUT="" +WITH_HEADER=0 +DELIM=',' +INCLUDE_META=0 + +while [ $# -gt 0 ]; do + case "$1" in + --with-header) WITH_HEADER=1 ;; + --delim) shift; DELIM="$1" ;; + --include-meta) INCLUDE_META=1 ;; + -h|--help) usage ;; + -*) echo "table-to-csv: unknown flag: $1" >&2; exit 2 ;; + *) INPUT="$1" ;; + esac + shift +done + +[ -z "$INPUT" ] || [ -f "$INPUT" ] || { echo "table-to-csv: no such file: $INPUT" >&2; exit 2; } + +[ "$WITH_HEADER" = "1" ] && printf 'input%soutput\n' "$DELIM" + +awk -v D="$DELIM" -v META="$INCLUDE_META" ' + function csv_escape(s, needs_q) { + needs_q = (index(s, D) > 0 || index(s, "\"") > 0 || index(s, "\n") > 0) + gsub(/"/, "\"\"", s) + if (needs_q) return "\"" s "\"" + return s + } + /^[[:space:]]*$/ { next } + /^[[:space:]]*#/ { next } + /^prologue$/ { in_prologue=1; next } + /^end_prologue$/ { in_prologue=0; next } + in_prologue { next } + /^dflt=/ { + if (META == "1") { + val = $0; sub(/^dflt=/, "", val) + print "__dflt__" D csv_escape(val) + } + next + } + /^encoded=/ { next } + { + line = $0 + gsub(/^[[:space:]]+|[[:space:]]+$/, "", line) + if (line == "") next + if (waiting_for_output) { + print csv_escape(pending_input) D csv_escape(line) + pending_input = "" + waiting_for_output = 0 + } else { + pending_input = line + waiting_for_output = 1 + } + } +' "${INPUT:-/dev/stdin}"