cloverleaf-larry/lib/nc-paths.sh

845 lines
40 KiB
Bash
Executable File

#!/usr/bin/env bash
# nc-paths.sh — deterministic route-chain path ENUMERATOR for Larry-Anywhere v3.
#
# This is the SINGLE walker backend for Cloverleaf message routing. It replaces
# the old dark `nc-parse.sh chain` BFS-node-set command (which only ever
# returned a flat set of reachable nodes, never enumerated paths, and was never
# wired into the LLM). It ports the v2 `paths` semantics
# (cloverleaf_tools/cli/legacy_workflow_commands.py paths_cmd + the three
# _enumerate_* helpers, lines 315-464) faithfully:
#
# - Downstream DFS from a start thread, following the DATAXLATE DEST list
# (find_outgoing). A leaf (no outgoing) OR a cycle hit terminates that path
# and the terminal node is included in the emitted chain.
# - Upstream DFS (mirror), following incoming threads (find_incoming).
# - All-mode: enumerate from every entry point (a thread with no incoming),
# deduped — gives the whole-site chain inventory (v2 list_full_routes).
#
# INTRA-SITE ROUTING RESOLUTION: within a single site the next hop is resolved
# ONLY from the DATAXLATE { DEST <name> } list (via nc-parse.sh destinations /
# sources). It NEVER walks via ICLSERVERPORT inside a site. The DEST list is
# present on every routing thread regardless of direction and simply yields
# nothing (no crash) when a thread has no routes. DO NOT reintroduce an
# ICLSERVERPORT-based hop for INTRA-site routing.
#
# CROSS-SITE BY DESTINATION BLOCK (v0.8.20, corrected on the real integrator):
# Cloverleaf links sites through named `destination` blocks — the inter-cloverleaf
# (ICL) routing table — NOT by blindly matching ports. A `destination <name> {...}`
# top-level block declares { SITE <site> } { THREAD <thread> } { PORT <port> }: it
# names a remote inbound thread in another site and the port the link connects on.
# A protocol's DATAXLATE DEST list may name EITHER (a) a LOCAL protocol (intra-site
# hop) OR (b) a destination block — and a DEST naming a destination block is the
# cross-site hop, resolved AUTHORITATIVELY to (SITE,THREAD). The PORT equals the
# remote thread's listen/ICL port (verifiable), but the link is name-resolved, so
# it is exact: e.g. mux thread ADTfr_epic_964700 has { DEST OB_ADT_ancS }; the
# destination block OB_ADT_ancS is { SITE ancout } { THREAD IB_ADT_muxS }
# { PORT 62043 } — so the chain continues into ancout's IB_ADT_muxS.
#
# WHY NOT PURE PORT-MATCHING (the rejected v0.8.20-draft mechanism): an earlier
# draft inferred the link by matching an outbound's PROTOCOL.PORT to an inbound's
# server/ICL port. That was (1) slow and (2) WRONG — it missed real feeders whose
# cross-site link is expressed only via a destination block (the mux feeder of
# IB_ADT_muxS above is reached through DEST OB_ADT_ancS, not through any thread
# whose PROTOCOL.PORT == 62043). ICLSERVERPORT is still read GUARDED in the index
# (absent / `{}` on most threads → skipped, never an error — the un-guarded keylget
# is exactly what crashed the old paths.tcl), but it is used only to corroborate a
# destination block's PORT, never as the primary link key.
#
# The whole route graph (protocol DEST edges + destination-block resolution +
# reverse-source maps) is built ONCE per run from a single awk pass per NetConfig
# (`nc-parse.sh index`) into in-memory associative arrays. Cross-site DOWNSTREAM: a
# DEST naming a destination block continues into its (site,thread). Cross-site
# UPSTREAM feeders of (site,thread): every destination block (any site) resolving
# to it, and the threads in that block's site that DEST to the block name — all
# in-memory lookups, no per-site chain enumeration (fixes Vera's m3 AND the old
# O(threads x parse-cost) per-hop subprocess blowup). Pass --site-only to scope the
# walk to a single site.
#
# Robust cycle detection across sites: every walk carries the full ancestor set
# keyed by "site\037thread"; revisiting any (site,thread) ancestor terminates the
# path (the terminal node is still emitted), so the enumeration always
# terminates. A global max-depth cap (default 128, matching v2) is a second
# backstop.
#
# DEFAULT OUTPUT = v1 CHAINS (one path per line, site/thread nodes, typed arrows):
# mux/ADTfr_epic_964700 --> mux/OB_ADT_ancS ==> ancout/IB_ADT_muxS --> ancout/ADTto_CodaMetrix
# - every NODE is rendered "site/thread" (slash join)
# - "-->" = an INTRA-site DATAXLATE route hop (a thread's DEST that names a
# LOCAL protocol — including the local OUTBOUND SENDER node, which is
# the destination-block name living in this site)
# - "==>" = a CROSS-site hop (the destination block's link: FROM the local
# outbound sender node TO the remote inbound thread it names)
# - one path per line; a branching thread yields N lines.
# This matches Bryan's v1 ground-truth paths.tcl: at every cross-site boundary the
# chain reads …local_inbound --> local_outbound_sender ==> remote_inbound --> … —
# the sender (= the destination-block name) is ALWAYS shown, never collapsed.
#
# The v1 line is PIPE-FIRST / field-extractable: `paths X | awkcut 1` yields the
# root node (field 1 = chain root, e.g. mux/ADTfr_epic_964700). The output is also
# valid INPUT: a "site/thread" node can be fed back in (paths X → extract root →
# paths <root>). `--format nodes` emits just the site/thread nodes (no arrows) one
# per line so piping never fights the arrow tokens.
#
# OTHER FORMATS (--format):
# table — the SITE/THREAD/HOPS/PATH aligned table (Bryan: kept, opt-in).
# THREAD = the start/anchor (ROOT) node of the row (first node in PATH);
# HOPS = number of nodes in the chain; PATH = the typed v1 chain.
# tsv — site<TAB>thread<TAB>hops<TAB>path (path = the typed v1 chain)
# jsonl — one JSON object per path {site,thread,hops,path}
# nodes — node-only: each path's "site/thread" nodes, one per line, blank line
# between paths (no arrows — clean for re-piping into `paths`).
# NOTE (Vera m2): for UPSTREAM (--up) chains the root is the feeder ROOT (the
# most-upstream source) and the queried thread is the chain TERMINUS.
#
# Usage:
# nc-paths.sh --netconfig <file> <thread> [flags] # explicit NetConfig
# nc-paths.sh <thread> <site> [flags] # resolve site under $HCIROOT
# nc-paths.sh <site>/<thread> [flags] # site/thread (v1 node form)
# nc-paths.sh --all [--site <name>] [flags] # whole-site entry chains
#
# Flags:
# --upstream | --up only the upstream chains feeding the thread
# --downstream | --down only the downstream chains from the thread
# (neither flag = full paths containing the thread,
# v2 default, falling back to downstream-from-thread)
# --all enumerate from every entry point (no thread arg)
# --site <name> scope all-mode (or site resolution) to one site
# --site-only do NOT cross site boundaries (downstream only)
# --hciroot <dir> override $HCIROOT for site/cross-site discovery
# --netconfig <file> operate on one explicit NetConfig (implies the site is
# basename(dirname(file)); cross-site still scans $HCIROOT)
# --max-depth N recursion cap (default 128)
# --format v1|table|tsv|jsonl|nodes default: v1 (the ground-truth chain form)
#
# Exit codes: 0 OK, 1 usage error, 2 not found.
set -u
set -o pipefail
NC_SELF="$0"
LIB_DIR="$(cd "$(dirname "$NC_SELF")" && pwd)"
NCP="$LIB_DIR/nc-parse.sh"
die() { printf 'nc-paths: %s\n' "$*" >&2; exit 1; }
# ─────────────────────────────────────────────────────────────────────────────
# Arg parsing
# ─────────────────────────────────────────────────────────────────────────────
THREAD=""
SITE_ARG=""
NETCONFIG=""
HCIROOT_OVERRIDE=""
DIR_MODE="full" # full | up | down
ALL_MODE=0
SITE_ONLY=0
MAX_DEPTH=128
FORMAT="v1"
POSITIONAL=()
while [ $# -gt 0 ]; do
case "$1" in
--upstream|--up) DIR_MODE="up" ;;
--downstream|--down) DIR_MODE="down" ;;
--all) ALL_MODE=1 ;;
--site) shift; SITE_ARG="${1:-}" ;;
--site-only) SITE_ONLY=1 ;;
--hciroot) shift; HCIROOT_OVERRIDE="${1:-}" ;;
--netconfig) shift; NETCONFIG="${1:-}" ;;
--max-depth) shift; MAX_DEPTH="${1:-128}" ;;
--format) shift; FORMAT="${1:-v1}" ;;
-h|--help) sed -n '2,113p' "$NC_SELF" | sed 's/^# \{0,1\}//'; exit 0 ;;
--*) die "unknown flag: $1" ;;
*) POSITIONAL+=("$1") ;;
esac
shift
done
case "$FORMAT" in v1|tsv|table|jsonl|nodes) ;; *) die "bad --format: $FORMAT (v1|table|tsv|jsonl|nodes)" ;; esac
# Positional shapes:
# <thread> (manual: thread only; site from $HCISITE/$HCISITEDIR)
# <thread> <site> (manual muscle-memory: thread + site)
# <site>/<thread> (v1 node form — the output IS valid input; pipe-first)
# PIPE-FIRST: a single positional containing a "/" is parsed as site/thread, so
# the v1 output (root node = "site/thread") can be fed straight back into paths.
if [ "${#POSITIONAL[@]}" -ge 1 ]; then THREAD="${POSITIONAL[0]}"; fi
if [ "${#POSITIONAL[@]}" -ge 2 ] && [ -z "$SITE_ARG" ]; then SITE_ARG="${POSITIONAL[1]}"; fi
if [ "${#POSITIONAL[@]}" -gt 2 ]; then die "too many positional args: ${POSITIONAL[*]}"; fi
# Accept the v1 "site/thread" node form as a single positional. A bare thread with
# no embedded slash (the legacy form) is left untouched. Only split on the FIRST
# slash so thread names are preserved verbatim. An explicit --site/2nd positional
# wins over a slash-embedded site only if they agree; otherwise the slash form is
# authoritative for the site (it came from our own output).
if [ -n "$THREAD" ] && [ -z "$NETCONFIG" ]; then
case "$THREAD" in
*/*) _slash_site="${THREAD%%/*}"; _slash_thr="${THREAD#*/}"
if [ -n "$_slash_site" ] && [ -n "$_slash_thr" ]; then
THREAD="$_slash_thr"; SITE_ARG="$_slash_site"
fi ;;
esac
fi
if [ "$ALL_MODE" = "0" ] && [ -z "$THREAD" ]; then
die "no thread given (and --all not set). Try: nc-paths.sh <thread> <site> OR nc-paths.sh --all --site <name>"
fi
ROOT="${HCIROOT_OVERRIDE:-${HCIROOT:-}}"
# ─────────────────────────────────────────────────────────────────────────────
# Site discovery — map every discovered NetConfig to a site name.
# Two parallel arrays (portable to bash 3.2 on macOS; no associative-array dep).
# SITE_NAMES[i] = site (basename of NetConfig's parent dir)
# SITE_NCS[i] = absolute NetConfig path
# An explicit --netconfig is always included; cross-site scanning still walks
# $HCIROOT so a terminal can hop into another site.
# ─────────────────────────────────────────────────────────────────────────────
SITE_NAMES=()
SITE_NCS=()
_add_site() {
local name="$1" nc="$2" i
[ -f "$nc" ] || return 0
# de-dupe by NetConfig path
for ((i=0; i<${#SITE_NCS[@]}; i++)); do
[ "${SITE_NCS[$i]}" = "$nc" ] && return 0
done
SITE_NAMES+=("$name")
SITE_NCS+=("$nc")
}
_discover_sites() {
# explicit NetConfig first (its site name is the parent dir basename)
if [ -n "$NETCONFIG" ]; then
[ -f "$NETCONFIG" ] || die "not a file: $NETCONFIG"
_add_site "$(basename "$(dirname "$NETCONFIG")")" "$NETCONFIG"
fi
# When --site-only with an explicit NetConfig, do not scan further.
if [ "$SITE_ONLY" = "1" ] && [ -n "$NETCONFIG" ]; then
return 0
fi
# Otherwise discover all sites under $HCIROOT (for cross-site joins / site
# resolution / all-mode), same walk nc-find.sh uses.
if [ -n "$ROOT" ]; then
local nc sname
while IFS= read -r nc; do
sname=$(basename "$(dirname "$nc")")
# When --site-only (no explicit NetConfig) and a site was named, keep only it.
if [ "$SITE_ONLY" = "1" ] && [ -n "$SITE_ARG" ] && [ "$sname" != "$SITE_ARG" ]; then
continue
fi
_add_site "$sname" "$nc"
done < <(find "$ROOT" -maxdepth 2 -name NetConfig -type f 2>/dev/null | sort)
fi
}
US=$'\037' # unit separator — safe field delimiter for site/thread keys
# ─────────────────────────────────────────────────────────────────────────────
# IN-MEMORY ROUTE GRAPH (v0.8.20 perf rearchitecture).
#
# The old walker invoked nc-parse.sh ONCE PER HOP PER CANDIDATE (destinations /
# sources / protocol-nested / protocol-field / list-protocols), and EACH of those
# re-ran _blocks + cmd_protocol_block — two full awk passes over the (16K-line)
# NetConfig. On the real 24-site integrator that is O(threads x parse-cost) =
# minutes (84s --site-only, 164s full for a single thread). Even intra-site was a
# bottleneck because `sources` scans every protocol body.
#
# Now we PARSE EACH NEEDED NetConfig EXACTLY ONCE (`nc-parse.sh index`, a single
# awk pass — see cmd_index) and load the result into bash associative arrays. The
# walkers then do pure O(1) in-memory lookups: NO subprocess and NO re-parse per
# hop. Indexing all 24 live NetConfigs is <1s; a single-thread trace is now a
# few seconds and a full-tree run is well under a minute.
#
# CROSS-SITE LINK (corrected): Cloverleaf links sites through named `destination`
# blocks (the ICL routing table), NOT by blindly matching ports. A protocol's
# DATAXLATE DEST may name either (a) a LOCAL protocol (intra-site hop) or (b) a
# `destination` block, which resolves to { SITE <site> } { THREAD <thread> }
# { PORT <port> } — the authoritative remote target. The PORT is the connecting
# port (it equals the remote thread's listen/ICL port — verifiable), but the SITE
# and THREAD come straight from the destination block, so the hop is exact and
# name-resolved. (The old port-only heuristic was BOTH slow AND missed real
# feeders whose link is expressed via a destination block — e.g. the mux feeder of
# ancout's IB_ADT_muxS via destination OB_ADT_ancS.)
#
# Associative arrays (bash 4+; matches the rest of this repo, and Git-Bash /
# Cygwin on Windows ship bash 4+/5+). Keys use US ("site\037thread") so names with
# unusual characters never collide with the field delimiter.
# G_PROTO[site\037thread] = 1 membership: thread exists in site
# G_DESTS[site\037thread] = "d1\nd2..." raw DATAXLATE DEST targets (newline)
# G_LISTEN[site\037thread] = "p1 p2" listen ports (server + ICL), space-sep
# G_OUT[site\037thread] = "port" outbound/tcpip-client dest port
# G_DESTBLK[site\037destname] = "tsite\037tthread\037tport" destination-block resolution
# G_INSRC[site\037thread] = "s1\ns2..." reverse intra-site DEST edges (sources)
# G_DESTBLK_REV[tsite\037tthread] = "fsite\037fname\n..." destination blocks (any site)
# pointing AT (tsite,tthread); fname is the dest
# block name, used to find its upstream feeders.
# G_LOADED tracks which NetConfigs have already been indexed (idempotent).
# ─────────────────────────────────────────────────────────────────────────────
declare -A G_PROTO G_DESTS G_LISTEN G_OUT G_DESTBLK G_INSRC G_DESTBLK_REV G_LOADED
# Load ONE NetConfig's index into the in-memory graph (idempotent per nc path).
_load_nc() {
local site="$1" nc="$2"
[ -n "${G_LOADED[$nc]:-}" ] && return 0
G_LOADED[$nc]=1
local tag a b c d e key
while IFS=$'\t' read -r tag a b c d e; do
case "$tag" in
P) key="${site}${US}${a}"; G_PROTO[$key]=1 ;;
D) key="${site}${US}${a}"
if [ -z "${G_DESTS[$key]:-}" ]; then G_DESTS[$key]="$b"; else G_DESTS[$key]="${G_DESTS[$key]}"$'\n'"$b"; fi ;;
L) key="${site}${US}${a}"
if [ -z "${G_LISTEN[$key]:-}" ]; then G_LISTEN[$key]="$b"; else G_LISTEN[$key]="${G_LISTEN[$key]} $b"; fi ;;
O) key="${site}${US}${a}"; G_OUT[$key]="$b" ;;
X) # X <destname> <tsite> <tthread> <tport>
key="${site}${US}${a}"; G_DESTBLK[$key]="${b}${US}${c}${US}${d}"
local rkey="${b}${US}${c}"
local rval="${site}${US}${a}"
if [ -z "${G_DESTBLK_REV[$rkey]:-}" ]; then G_DESTBLK_REV[$rkey]="$rval"; else G_DESTBLK_REV[$rkey]="${G_DESTBLK_REV[$rkey]}"$'\n'"$rval"; fi ;;
esac
done < <("$NCP" index "$nc" 2>/dev/null)
}
# Build the reverse intra-site DEST edges (sources) for every loaded site. Called
# once after all needed NetConfigs are loaded. For each thread A with DEST B in
# the SAME site, record A as a source of B (only when B is a local protocol —
# DEST targets that are destination blocks are handled as cross-site, not here).
_build_in_sources() {
local key src site dst dkey
for key in "${!G_DESTS[@]}"; do
site="${key%%$US*}"; src="${key#*$US}"
while IFS= read -r dst; do
[ -z "$dst" ] && continue
dkey="${site}${US}${dst}"
[ -n "${G_PROTO[$dkey]:-}" ] || continue # only local protocols are intra-site sources
if [ -z "${G_INSRC[$dkey]:-}" ]; then G_INSRC[$dkey]="$src"; else G_INSRC[$dkey]="${G_INSRC[$dkey]}"$'\n'"$src"; fi
done <<< "${G_DESTS[$key]}"
done
}
# Ensure the WHOLE tree is loaded (all discovered sites) — needed for cross-site
# resolution and reverse-source maps. Idempotent.
GRAPH_BUILT=0
_build_graph() {
[ "$GRAPH_BUILT" = "1" ] && return 0
GRAPH_BUILT=1
local i
for ((i=0; i<${#SITE_NCS[@]}; i++)); do
_load_nc "${SITE_NAMES[$i]}" "${SITE_NCS[$i]}"
done
_build_in_sources
}
# Given a thread name, find the FIRST discovered site that declares it (in-memory).
# Emits "site" or returns 1.
_locate_thread() {
local want="$1" i sname
for ((i=0; i<${#SITE_NAMES[@]}; i++)); do
sname="${SITE_NAMES[$i]}"
[ -n "${G_PROTO[${sname}${US}${want}]:-}" ] && { printf '%s' "$sname"; return 0; }
done
return 1
}
# ─────────────────────────────────────────────────────────────────────────────
# One-hop primitives — now pure in-memory lookups (no subprocess, no re-parse).
# INTRA-site routing follows the DATAXLATE DEST list only (never ICLSERVERPORT).
# A DEST that names a destination block is NOT an intra-site dest (it is the
# cross-site link, handled in the walkers).
# ─────────────────────────────────────────────────────────────────────────────
# Intra-site downstream: DEST targets that are LOCAL protocols in this site.
_outgoing() { # site thread
local site="$1" thr="$2" key="${1}${US}${2}" d dkey
[ -n "${G_DESTS[$key]:-}" ] || return 0
while IFS= read -r d; do
[ -z "$d" ] && continue
dkey="${site}${US}${d}"
[ -n "${G_PROTO[$dkey]:-}" ] && printf '%s\n' "$d"
done <<< "${G_DESTS[$key]}"
}
# Intra-site upstream: local protocols that DEST to this thread.
_incoming() { local key="${1}${US}${2}"; [ -n "${G_INSRC[$key]:-}" ] && printf '%s\n' "${G_INSRC[$key]}"; }
# Is <thread> an entry point (no incoming) in <site>?
_is_entry_in() { [ -z "${G_INSRC[${1}${US}${2}]:-}" ]; }
# Cross-site DOWNSTREAM targets: a DEST of (cur_site,cur_thread) that is NOT a
# local protocol but IS a destination block. The destination-block NAME (d) is the
# LOCAL OUTBOUND SENDER node, living in cur_site — v1 shows it and we must NOT
# collapse it. The block resolves to the remote inbound (tsite,tthread). Emit each
# as "sender\037tsite\037tthread" (sender = the dest-block name in cur_site). The
# walker then renders: cur_thread --(intra)--> cur_site/sender ==(cross)==> tsite/tthread.
# Authoritative name-resolved link (PORT is just confirmation).
_xsite_down_targets() {
local cur_site="$1" cur_thread="$2" key="${1}${US}${2}" d dbkey resolved
[ -n "${G_DESTS[$key]:-}" ] || return 0
while IFS= read -r d; do
[ -z "$d" ] && continue
[ -n "${G_PROTO[${cur_site}${US}${d}]:-}" ] && continue # local protocol → intra-site, not here
dbkey="${cur_site}${US}${d}"
resolved="${G_DESTBLK[$dbkey]:-}"
[ -z "$resolved" ] && continue # not a known destination block → skip
local tsite="${resolved%%$US*}" rest="${resolved#*$US}"
local tthr="${rest%%$US*}"
printf '%s%s%s%s%s\n' "$d" "$US" "$tsite" "$US" "$tthr"
done <<< "${G_DESTS[$key]}"
}
# Cross-site UPSTREAM feeders: who feeds (cur_site,cur_thread) from another site?
# Any destination block (in any site) that resolves to (cur_site,cur_thread); its
# upstream feeders are the threads in the destination block's OWN site that DEST to
# that destination-block NAME (dbname). The block NAME is the LOCAL OUTBOUND SENDER
# node, living in fsite between the feeder and this remote inbound — v1 shows it,
# so we carry it. Emit each as "fsite\037fthread\037dbname". The walker then renders
# the upstream prefix: fsite/feeder --(intra)--> fsite/dbname ==(cross)==> cur_site/cur_thread.
# Pure in-memory lookup — no per-site chain enumeration.
_xsite_up_feeders() {
local cur_site="$1" cur_thread="$2" rkey="${1}${US}${2}" dbref
[ -n "${G_DESTBLK_REV[$rkey]:-}" ] || return 0
while IFS= read -r dbref; do
[ -z "$dbref" ] && continue
# dbref = fsite\037destblockname
local fsite="${dbref%%$US*}" dbname="${dbref#*$US}" feeder
# feeders = local protocols in fsite whose DEST names dbname
local fkey
for fkey in "${!G_DESTS[@]}"; do
[ "${fkey%%$US*}" = "$fsite" ] || continue
case $'\n'"${G_DESTS[$fkey]}"$'\n' in
(*$'\n'"$dbname"$'\n'*)
feeder="${fkey#*$US}"
printf '%s%s%s%s%s\n' "$fsite" "$US" "$feeder" "$US" "$dbname" ;;
esac
done
done <<< "${G_DESTBLK_REV[$rkey]}"
}
# ─────────────────────────────────────────────────────────────────────────────
# Path enumeration. Emitted paths are written to $OUT_PATHS as one line each:
# site<TAB>chain where chain = the rendered v1 typed chain (site/thread nodes
# joined by --> / ==>).
#
# CHAIN ENCODING (EDGE-TYPED). We carry the running chain as a space-joined list
# of TOKENS. The FIRST token is a bare node key "site\037thread". Every SUBSEQUENT
# token is "EDGE\035site\037thread" where the leading 1-char EDGE code records how
# this node connects to the PREVIOUS node:
# i = INTRA-site DATAXLATE hop → rendered "-->"
# x = CROSS-site destination-link → rendered "==>"
# \035 (GS) separates the edge code from the node key; \037 (US) separates site
# from thread. Node names are [A-Za-z0-9_]+ so neither separator can collide, and
# tokens stay space-tokenizable (the full-mode awk join still splits on spaces).
# The ancestor set (cycle detection) remains a newline-joined list of plain node
# keys "site\037thread" — edge codes are never part of it.
# ─────────────────────────────────────────────────────────────────────────────
GS=$'\035' # group separator — delimits the edge code from the node key
OUT_PATHS=$(mktemp)
trap 'rm -f "$OUT_PATHS"' EXIT
# Append a node to a keychain with an explicit edge type.
# _chain_push CHAIN EDGE NODEKEY (EDGE = i|x; first push ignores EDGE)
# Emits the new chain on stdout.
_chain_push() {
local chain="$1" edge="$2" node="$3"
if [ -z "$chain" ]; then printf '%s' "$node"; else printf '%s %s%s%s' "$chain" "$edge" "$GS" "$node"; fi
}
# Prepend a node (upstream walk builds a prefix). The edge code lives on the node
# that follows it; when we prepend a NEW root we must move the edge code onto the
# OLD first node and leave the new root bare.
# _chain_unshift CHAIN EDGE NODEKEY
_chain_unshift() {
local chain="$1" edge="$2" node="$3"
if [ -z "$chain" ]; then printf '%s' "$node"; return 0; fi
# The current chain's first token is a bare node key (no edge code). Re-tag it
# with EDGE (its connection to the new root we are prepending), then prefix the
# bare new root.
local first="${chain%% *}" rest=""
case "$chain" in *' '*) rest=" ${chain#* }" ;; esac
printf '%s %s%s%s%s' "$node" "$edge" "$GS" "$first" "$rest"
}
# _emit_chain ANCHOR_SITE KEYCHAIN
# KEYCHAIN = the edge-typed token list described above.
# Renders to "anchor_site<TAB>site/thread --> site/thread ==> ..." (v1 form).
_emit_chain() {
local anchor_site="$1" keychain="$2"
local out="" tok edge node site thr first=1
for tok in $keychain; do
if [ "$first" = "1" ]; then
node="$tok"; edge=""
else
edge="${tok%%$GS*}"; node="${tok#*$GS}"
fi
site="${node%%$US*}"; thr="${node#*$US}"
if [ "$first" = "1" ]; then
out="${site}/${thr}"; first=0
else
case "$edge" in
x) out="$out ==> ${site}/${thr}" ;;
*) out="$out --> ${site}/${thr}" ;;
esac
fi
done
printf '%s\t%s\n' "$anchor_site" "$out"
}
# Cycle test against the newline-joined ancestor set — pure bash, no grep
# subprocess (this used to fork `grep -qxF` per hop). seen lines are US-keyed.
_seen_has() {
case $'\n'"$1"$'\n' in (*$'\n'"$2"$'\n'*) return 0 ;; esac
return 1
}
# Downstream DFS. Mirrors v2 _enumerate_downstream_paths + cross-site hop.
# All lookups are in-memory (the graph is keyed by SITE; no NetConfig path / no
# subprocess per hop).
# $1 anchor_site — site to report in the SITE column for these rows
# $2 cur_site — site of current thread
# $3 cur_thread — current thread name
# $4 keychain — edge-typed ancestor chain NOT including current
# $5 seen — newline-joined ancestor node keys (for cycle detection)
# $6 depth
# $7 edge_in — edge connecting the previous node to cur (i|x; "" for root)
_walk_down() {
local anchor_site="$1" cur_site="$2" cur_thread="$3"
local keychain="$4" seen="$5" depth="$6" edge_in="${7:-}"
local curkey="${cur_site}${US}${cur_thread}"
local newchain
newchain="$(_chain_push "$keychain" "${edge_in:-i}" "$curkey")"
# cycle / depth cap → terminate, include current node (v2 semantics)
if [ "$depth" -gt "$MAX_DEPTH" ] || _seen_has "$seen" "$curkey"; then
_emit_chain "$anchor_site" "$newchain"
return 0
fi
# gather outgoing within the current site (DEST targets that are local protocols)
local outgoing=()
local d
while IFS= read -r d; do
[ -z "$d" ] && continue
outgoing+=("$d")
done < <(_outgoing "$cur_site" "$cur_thread")
local nseen="$seen"$'\n'"$curkey"
local branched=0
if [ "${#outgoing[@]}" -gt 0 ]; then
branched=1
for d in "${outgoing[@]}"; do
# intra-site route hop (-->)
_walk_down "$anchor_site" "$cur_site" "$d" "$newchain" "$nseen" $((depth+1)) i
done
fi
# CROSS-SITE HOP via destination block (v0.8.20, corrected; v0.8.20 output fix:
# SHOW THE SENDER NODE). A DEST of this thread that names a destination block is
# the LOCAL OUTBOUND SENDER node (the block name, in cur_site) followed by the
# remote inbound (tsite,tthread). v1 renders BOTH:
# cur_thread --(intra -->)--> cur_site/sender ==(cross ==>)==> tsite/tthread
# so we (1) push the sender node with an INTRA edge, then (2) recurse into the
# remote inbound with a CROSS edge. NEVER collapse the sender. This is in ADDITION
# to any intra-site branches above (a thread can route both locally and cross-site).
if [ "$SITE_ONLY" = "0" ]; then
local tgt sender osite othr okey sendkey sendchain
while IFS= read -r tgt; do
[ -z "$tgt" ] && continue
# tgt = sender\037tsite\037tthread
sender="${tgt%%$US*}"; local rest="${tgt#*$US}"
osite="${rest%%$US*}"; othr="${rest#*$US}"
okey="${osite}${US}${othr}"
_seen_has "$seen" "$okey" && continue # cycle guard across sites
branched=1
# (1) the local outbound sender node, intra-site edge from cur_thread
sendkey="${cur_site}${US}${sender}"
sendchain="$(_chain_push "$newchain" i "$sendkey")"
# (2) cross-site edge from the sender into the remote inbound; continue there
_walk_down "$anchor_site" "$osite" "$othr" "$sendchain" "$nseen" $((depth+1)) x
done < <(_xsite_down_targets "$cur_site" "$cur_thread")
fi
# true terminal (no intra- or cross-site continuation) — emit the chain
[ "$branched" = "0" ] && _emit_chain "$anchor_site" "$newchain"
return 0
}
# Upstream DFS. Mirrors v2 _enumerate_upstream_paths. Builds the chain as a PREFIX
# (sources come before current). Cross-site feeders are resolved via destination
# blocks (see _xsite_up_feeders) — in-memory, no per-site enumeration.
# $7 edge_in — edge connecting cur to the node that FOLLOWS it (already in
# keychain). i|x; "" for the terminus (nothing follows yet).
_walk_up() {
local anchor_site="$1" cur_site="$2" cur_thread="$3"
local keychain="$4" seen="$5" depth="$6" edge_in="${7:-}"
local curkey="${cur_site}${US}${cur_thread}"
local newchain
newchain="$(_chain_unshift "$keychain" "${edge_in:-i}" "$curkey")"
if [ "$depth" -gt "$MAX_DEPTH" ] || _seen_has "$seen" "$curkey"; then
_emit_chain "$anchor_site" "$newchain"
return 0
fi
local incoming=()
local s
while IFS= read -r s; do
[ -z "$s" ] && continue
incoming+=("$s")
done < <(_incoming "$cur_site" "$cur_thread")
local nseen="$seen"$'\n'"$curkey"
local branched=0
if [ "${#incoming[@]}" -gt 0 ]; then
branched=1
for s in "${incoming[@]}"; do
# intra-site source feeds cur via a route hop (-->)
_walk_up "$anchor_site" "$cur_site" "$s" "$newchain" "$nseen" $((depth+1)) i
done
fi
# CROSS-SITE UPSTREAM FEEDERS via destination block (v0.8.20, corrected; output
# fix: SHOW THE SENDER NODE). Any destination block (any site) resolving to THIS
# (site,thread); the block NAME is the LOCAL OUTBOUND SENDER node in the feeder's
# site, and the feeders are the threads in that site that DEST to the block name.
# v1 renders the upstream prefix as:
# fsite/feeder --(intra -->)--> fsite/sender ==(cross ==>)==> cur_site/cur_thread
# so we (1) prepend the sender node with a CROSS edge (sender ==> cur), then
# (2) recurse up into the feeder with an INTRA edge (feeder --> sender). In-memory.
if [ "$SITE_ONLY" = "0" ]; then
local fdr fsite othr okey sender sendkey sendchain
while IFS= read -r fdr; do
[ -z "$fdr" ] && continue
# fdr = fsite\037fthread\037dbname
fsite="${fdr%%$US*}"; local rest="${fdr#*$US}"
othr="${rest%%$US*}"; sender="${rest#*$US}"
okey="${fsite}${US}${othr}"
_seen_has "$seen" "$okey" && continue
branched=1
# (1) the local outbound sender node, CROSS edge into cur
sendkey="${fsite}${US}${sender}"
sendchain="$(_chain_unshift "$newchain" x "$sendkey")"
# (2) recurse up into the feeder, INTRA edge into the sender
_walk_up "$anchor_site" "$fsite" "$othr" "$sendchain" "$nseen" $((depth+1)) i
done < <(_xsite_up_feeders "$cur_site" "$cur_thread")
fi
[ "$branched" = "0" ] && _emit_chain "$anchor_site" "$newchain"
return 0
}
# ─────────────────────────────────────────────────────────────────────────────
# Drivers
# ─────────────────────────────────────────────────────────────────────────────
# In-memory list of a site's protocol names (membership keys are "site\037thread").
_protos_in_site() {
local site="$1" key
for key in "${!G_PROTO[@]}"; do
[ "${key%%$US*}" = "$site" ] && printf '%s\n' "${key#*$US}"
done
}
# Enumerate every full path in a site by starting from each entry point.
# Cross-site continuation happens naturally inside _walk_down. Dedup by the
# rendered "site\tchain" line. All in-memory — no subprocess.
_enumerate_all_in_site() {
local site="$1"
local entry entries=() any_entry=0 all=()
while IFS= read -r entry; do
[ -z "$entry" ] && continue
all+=("$entry")
if _is_entry_in "$site" "$entry"; then
entries+=("$entry"); any_entry=1
fi
done < <(_protos_in_site "$site")
# if no entry points (every thread has an incoming, e.g. a pure cycle),
# fall back to all protocols as start points (v2 fallback)
if [ "$any_entry" = "0" ]; then
entries=("${all[@]}")
fi
for entry in "${entries[@]}"; do
_walk_down "$site" "$site" "$entry" "" "" 0
done
}
main_enumerate() {
_discover_sites
[ "${#SITE_NCS[@]}" -gt 0 ] || die "no NetConfig found (set \$HCIROOT, or pass --netconfig / --hciroot)"
# PARSE ONCE: build the whole in-memory route graph (single awk pass per
# NetConfig + reverse-source maps). The walkers then run entirely in memory.
# With --site-only and an explicit thread we still build the full graph (it is
# <1s for 24 sites); cross-site hops are simply suppressed by the SITE_ONLY guard.
_build_graph
local raw
raw=$(mktemp)
trap 'rm -f "$OUT_PATHS" "$raw"' EXIT
if [ "$ALL_MODE" = "1" ]; then
# whole-site entry chains; scope to --site if given (else every site)
local i sname
for ((i=0; i<${#SITE_NAMES[@]}; i++)); do
sname="${SITE_NAMES[$i]}"
if [ -n "$SITE_ARG" ] && [ "$sname" != "$SITE_ARG" ]; then continue; fi
_enumerate_all_in_site "$sname" >> "$raw"
done
else
# locate the thread's home site (in-memory membership lookup)
local home_site
if [ -n "$NETCONFIG" ]; then
home_site="$(basename "$(dirname "$NETCONFIG")")"
[ -n "${G_PROTO[${home_site}${US}${THREAD}]:-}" ] \
|| die "thread not found in $NETCONFIG: $THREAD"
elif [ -n "$SITE_ARG" ]; then
home_site="$SITE_ARG"
[ -n "${G_PROTO[${home_site}${US}${THREAD}]:-}" ] \
|| die "thread not found in site $SITE_ARG: $THREAD"
else
home_site="$(_locate_thread "$THREAD")" || die "thread not found in any discovered site: $THREAD"
fi
case "$DIR_MODE" in
up) _walk_up "$home_site" "$home_site" "$THREAD" "" "" 0 >> "$raw" ;;
down) _walk_down "$home_site" "$home_site" "$THREAD" "" "" 0 >> "$raw" ;;
full)
# v2 default: every full ROOT-TO-LEAF path CONTAINING the thread.
#
# v0.8.20 (rearchitected): do NOT scan every site's entry chains (the old
# O(sites x threads) loop). The complete chain = the thread's UPSTREAM
# feeder chains (each ending AT the thread: root -> ... -> thread) JOINED at
# the thread to its DOWNSTREAM chains (each starting AT the thread:
# thread -> ... -> leaf). Both walks are in-memory and follow cross-site
# links via destination blocks, so the join naturally spans sites
# (e.g. mux/ADTfr_epic_964700 --> ... ==> ancout/IB_ADT_muxS --> ancout/ADTto_CodaMetrix).
# The cartesian join over the (usually tiny) up x down sets is done in awk.
# Both halves are the RENDERED v1 chain (site/thread nodes; --> / ==> arrows).
# The upstream prefix ENDS with the queried node (home_site/THREAD); the
# downstream chain STARTS with it. We strip the leading queried node from the
# downstream — KEEPING the arrow that follows it (--> or ==>) so the cross-site
# boundary type is preserved — and graft the remaining suffix onto the prefix.
local up_tmp down_tmp qnode
up_tmp=$(mktemp); down_tmp=$(mktemp)
qnode="${home_site}/${THREAD}"
_walk_up "$home_site" "$home_site" "$THREAD" "" "" 0 > "$up_tmp"
_walk_down "$home_site" "$home_site" "$THREAD" "" "" 0 > "$down_tmp"
# join: for each upstream prefix x each downstream chain, emit
# prefix <arrow> <downstream minus leading queried-node-and-its-arrow>.
awk -F'\t' -v q="$qnode" '
FNR==NR { usite[NR]=$1; up[NR]=$2; nu=NR; next }
{
dn=$2
# split the downstream into the leading queried node, the arrow that
# follows it, and the remaining suffix. arrow is " --> " or " ==> ".
arrow=""; suffix=""
if (index(dn, q " --> ") == 1) { arrow=" --> "; suffix=substr(dn, length(q " --> ")+1) }
else if (index(dn, q " ==> ") == 1) { arrow=" ==> "; suffix=substr(dn, length(q " ==> ")+1) }
else { arrow=""; suffix="" } # downstream was just the node
for (i=1; i<=nu; i++) {
chain = up[i]
if (suffix != "") chain = up[i] arrow suffix
print usite[i] "\t" chain
}
}
' "$up_tmp" "$down_tmp" | sort -u >> "$raw"
rm -f "$up_tmp" "$down_tmp"
;;
esac
fi
# dedup the raw "site<TAB>chain" lines, preserving first-seen order
awk '!seen[$0]++' "$raw" > "$OUT_PATHS"
rm -f "$raw"
trap 'rm -f "$OUT_PATHS"' EXIT
}
# ─────────────────────────────────────────────────────────────────────────────
# Render: OUT_PATHS holds "site<TAB>chain" lines, where chain is the v1 rendered
# form (site/thread nodes joined by " --> " / " ==> "). All derived columns split
# the chain on the TYPED-ARROW regex " (--|==)> " so HOPS counts NODES and the
# root (field 1) is the first node — independent of the boundary type.
# THREAD = first node of the chain (the anchor/root for this row)
# HOPS = number of nodes in the chain
# ─────────────────────────────────────────────────────────────────────────────
render() {
if [ ! -s "$OUT_PATHS" ]; then
# No-paths goes to stderr for data/pipe formats so stdout stays clean for
# downstream field extraction (awkcut / cut never sees a prose line).
case "$FORMAT" in
v1|nodes|tsv|jsonl) printf 'No paths found.\n' >&2 ;;
*) printf 'No paths found.\n' ;;
esac
return 0
fi
case "$FORMAT" in
v1)
# The ground-truth chain, one path per line. PIPE-FIRST: field 1 (split on
# the arrow tokens, e.g. `awkcut 1`) is the root node "site/thread".
awk -F'\t' '{ print $2 }' "$OUT_PATHS"
return 0
;;
nodes)
# node-only extraction: each path's site/thread nodes one per line, a blank
# line between paths. No arrows — clean for re-piping into `paths`.
awk -F'\t' '
NR>1 { print "" }
{
chain=$2
n=split(chain, parts, / (--|==)> /)
for (i=1; i<=n; i++) print parts[i]
}' "$OUT_PATHS"
return 0
;;
esac
# produce a 4-col TSV: site thread hops path (path = the v1 typed chain)
local tsv
tsv=$(awk -F'\t' '
{
site=$1; chain=$2
# first node = chain up to the first typed arrow
first=chain
sub(/ (--|==)> .*/, "", first)
# hop count = number of nodes = typed-arrow separators + 1
n=split(chain, parts, / (--|==)> /)
printf "%s\t%s\t%d\t%s\n", site, first, n, chain
}' "$OUT_PATHS")
case "$FORMAT" in
tsv)
printf 'site\tthread\thops\tpath\n'
printf '%s\n' "$tsv"
;;
jsonl)
printf '%s\n' "$tsv" | awk -F'\t' '
function esc(s){ gsub(/\\/,"\\\\",s); gsub(/"/,"\\\"",s); return s }
{ printf "{\"site\":\"%s\",\"thread\":\"%s\",\"hops\":%s,\"path\":\"%s\"}\n",
esc($1),esc($2),$3,esc($4) }'
;;
table)
{
printf 'SITE\tTHREAD\tHOPS\tPATH\n'
printf '%s\n' "$tsv"
} | awk -F'\t' '
{ for (i=1;i<=NF;i++){ if (length($i)>w[i]) w[i]=length($i); cell[NR,i]=$i }; rows=NR; cols=NF }
END {
for (r=1; r<=rows; r++) {
for (c=1; c<=cols; c++) printf "%-*s ", w[c], cell[r,c]
printf "\n"
if (r==1) { for (c=1; c<=cols; c++) { for (k=0;k<w[c];k++) printf "-"; printf " " }; printf "\n" }
}
}'
;;
esac
if [ "$FORMAT" = "table" ]; then
local n
n=$(printf '%s\n' "$tsv" | grep -c . )
printf '\n%d path(s)\n' "${n:-0}" >&2
fi
return 0
}
main_enumerate
render