Bryan Johnson
578cefcc35
Closes the last gap in the rate-limit-diagnosis pipeline: anthropic-ratelimit-*
headers captured on the MobaXterm work-box now flow to the Mac memory daemon
(Tier 4 Hindsight + Tier 7 mem0) automatically.
- lib/headers-sync.sh: incremental, offset-tracked, idempotent push of
headers.log to ~/.cloverleaf/headers-<hostname>.jsonl on the Mac, riding the
existing authenticated SSH ControlMaster. No new auth; password never in
argv/env. No-op when nothing new; re-seed on local rotation/shrink. Fully
graceful (no target / closed master / transport error → warn + continue;
never crashes the session).
- /headers-sync on|off|status|target <alias>|now slash command + TAB-completion
+ /help. Config persisted to $LARRY_HOME/.env. Auto-sync fires on REPL exit.
- Security: headers.log carries only anthropic-* headers + status lines — NO
PHI per Vera audit V7; transport reused unchanged (not weakened).
Layered cleanly on top of Clover #8's v0.8.5 (4f1ea86) — edits isolated to new
lib + help/array/trap/dispatch hunks; no overlap with the streaming parser,
retry/backoff, error-display, or phi-notice regions.
Co-Authored-By: Clover (Claude Opus 4.7) <noreply@anthropic.com>
368 lines
15 KiB
Bash
Executable File
368 lines
15 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# headers-sync.sh — incremental work-box → Mac sync of Clover-Larry's
|
|
# headers.log, so Bryan's Mac memory daemon ingests the work-box's
|
|
# anthropic-ratelimit-* headers automatically.
|
|
#
|
|
# WHY (tsk-2026-05-27-023)
|
|
# ------------------------
|
|
# headers.log (the rate-limit-header capture written by larry.sh's
|
|
# _parse_response_headers) lives at $LARRY_HOME/log/headers.log ON THE
|
|
# WORK-BOX (Bryan's MobaXterm Cloud PC, where the testing happens). The
|
|
# Mac's memory daemon (scripts/memory_daemon.sh) watches the Mac's
|
|
# $LARRY_HOME/log/ and ~/.cloverleaf/ and fires headers_log_ingest.py on
|
|
# any write — but NOTHING moves the work-box file to the Mac. This tool
|
|
# closes that gap, unblocking the rate_limit_error diagnosis: the
|
|
# anthropic-ratelimit-* headers captured on the work-box flow to the Mac
|
|
# and into Tier 4 (Hindsight) + Tier 7 (mem0).
|
|
#
|
|
# DESIGN
|
|
# ------
|
|
# * Transport: rides the EXISTING authenticated SSH ControlMaster that
|
|
# ssh-helper.sh opens (/ssh-setup <alias>). No new key, no new auth, no
|
|
# password ever in argv/env. We reuse the same `ssh -S <sock>` multiplex.
|
|
# * Incremental + offset-tracked: headers.log is append-only. We track the
|
|
# last byte offset we pushed in $LARRY_HOME/.headers-sync-offset and send
|
|
# ONLY the new bytes since then (dd skip=offset), appending them remotely
|
|
# with `cat >> remote`. We never re-send the whole file.
|
|
# * Idempotent: if no new bytes, it's a no-op. If the local file shrank
|
|
# (rotation/truncation), the offset resets to 0 and the next sync re-seeds
|
|
# the remote file (truncate-then-append). The Mac-side ingester
|
|
# (headers_log_ingest.py) is ALSO offset-tracked + idempotent, so a
|
|
# re-seed is tolerated (it resets its own offset when the source shrinks).
|
|
# * Target file on the Mac: a per-host filename so the work-box stream never
|
|
# collides with the Mac's own local headers.log. Default basename is
|
|
# `headers-<workbox-hostname>.jsonl`, placed under the Mac's
|
|
# ~/.cloverleaf/ (a daemon-watched dir). The daemon's fswatch arm matches
|
|
# both `headers.log` and `headers-*.jsonl`; headers_log_ingest.py resolves
|
|
# the candidate (the v0.8.6 daemon patch adds the glob so the synced file
|
|
# is actually picked up).
|
|
# * Graceful: if no target is configured, or the master isn't open, or the
|
|
# transport fails, we log a warn to $LARRY_HOME/log/sync.log and return
|
|
# non-fatally — we NEVER crash the larry session.
|
|
#
|
|
# SECURITY (Vera PHI audit V7)
|
|
# ----------------------------
|
|
# headers.log holds only anthropic-* response headers (rate-limit metadata +
|
|
# org id) and HTTP status lines. Per Vera's V7 audit it carries NO message
|
|
# body and NO PHI, so syncing it is safe. The transport is the existing
|
|
# key/password-authenticated ControlMaster — we do not weaken it. (Defensive:
|
|
# we never echo file contents to stdout; only byte counts + paths.)
|
|
#
|
|
# CONFIG (persisted in $LARRY_HOME/.env via simple KEY=VALUE lines)
|
|
# -----------------------------------------------------------------
|
|
# LARRY_HEADERS_SYNC=1 auto-sync enabled (default off)
|
|
# LARRY_HEADERS_SYNC_TARGET=ALIAS Mac SSH alias (see /ssh-status)
|
|
# LARRY_HEADERS_SYNC_DEST=PATH remote dest path (default
|
|
# ~/.cloverleaf/headers-<hostname>.jsonl)
|
|
#
|
|
# SUBCOMMANDS
|
|
# on enable auto-sync (persist LARRY_HEADERS_SYNC=1)
|
|
# off disable auto-sync (persist LARRY_HEADERS_SYNC=0)
|
|
# target <alias> set the Mac SSH alias (persist)
|
|
# status show enabled? + target + dest + last-sync + bytes pushed
|
|
# now run one incremental sync immediately (ignores on/off)
|
|
# sync internal: run sync IF enabled (called by larry.sh hooks)
|
|
# help print this header
|
|
#
|
|
# EXIT CODES
|
|
# 0 ok / no-op / disabled (graceful)
|
|
# 1 usage error
|
|
# 2 sync attempted but failed (logged; larry.sh treats as non-fatal)
|
|
|
|
set -u
|
|
set -o pipefail
|
|
|
|
LARRY_HOME="${LARRY_HOME:-$HOME/.larry}"
|
|
HS_LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" 2>/dev/null && pwd)"
|
|
HS_ENV_FILE="$LARRY_HOME/.env"
|
|
HS_OFFSET_FILE="$LARRY_HOME/.headers-sync-offset"
|
|
HS_STATE_FILE="$LARRY_HOME/.headers-sync-state" # last-sync ts + bytes
|
|
HS_SYNC_LOG="$LARRY_HOME/log/sync.log"
|
|
HS_HEADERS_LOG="$LARRY_HOME/log/headers.log"
|
|
HS_HELPER="$HS_LIB_DIR/ssh-helper.sh"
|
|
SSH_HOSTS_FILE="$LARRY_HOME/.ssh-hosts.tsv"
|
|
SSH_SOCKETS_DIR="$LARRY_HOME/.ssh-sockets"
|
|
|
|
# Shared CR-defense primitives (coerce_int / strip_cr). Fall back to inline
|
|
# defs if the lib isn't alongside (defensive — should always be present).
|
|
if [ -r "$HS_LIB_DIR/cygwin-safe.sh" ]; then
|
|
# shellcheck disable=SC1090,SC1091
|
|
. "$HS_LIB_DIR/cygwin-safe.sh"
|
|
else
|
|
coerce_int() { local r="${1:-}" d="${2:-0}" c; c=$(printf '%s' "$r" | tr -cd '0-9'); printf '%s' "${c:-$d}"; }
|
|
strip_cr() { printf '%s' "${1:-}" | tr -d '\r'; }
|
|
fi
|
|
|
|
hs_warn() { printf 'headers-sync: warn: %s\n' "$*" >&2; }
|
|
hs_say() { printf 'headers-sync: %s\n' "$*"; }
|
|
hs_err() { printf 'headers-sync: error: %s\n' "$*" >&2; }
|
|
|
|
# Append a timestamped audit line to $LARRY_HOME/log/sync.log.
|
|
_hs_log() {
|
|
mkdir -p "$LARRY_HOME/log" 2>/dev/null || true
|
|
printf '%s %s\n' "$(date -Iseconds 2>/dev/null || date)" "$*" >> "$HS_SYNC_LOG" 2>/dev/null || true
|
|
}
|
|
|
|
# Load persisted config (KEY=VALUE) from $LARRY_HOME/.env without exporting
|
|
# the rest of the file into our process beyond the keys we care about.
|
|
_hs_load_env() {
|
|
[ -f "$HS_ENV_FILE" ] || return 0
|
|
local line k v
|
|
while IFS= read -r line || [ -n "$line" ]; do
|
|
line=$(strip_cr "$line")
|
|
case "$line" in
|
|
LARRY_HEADERS_SYNC=*|LARRY_HEADERS_SYNC_TARGET=*|LARRY_HEADERS_SYNC_DEST=*)
|
|
k=${line%%=*}; v=${line#*=}
|
|
# strip surrounding quotes if present
|
|
v=${v#\"}; v=${v%\"}; v=${v#\'}; v=${v%\'}
|
|
export "$k=$v"
|
|
;;
|
|
esac
|
|
done < "$HS_ENV_FILE"
|
|
}
|
|
|
|
# Persist a single KEY=VALUE into $LARRY_HOME/.env idempotently (replace the
|
|
# line if the key exists, else append). Preserves all other lines + mode 0600.
|
|
_hs_persist_env() {
|
|
local key="$1" val="$2"
|
|
mkdir -p "$LARRY_HOME" 2>/dev/null || true
|
|
umask 077
|
|
local tmp; tmp=$(mktemp 2>/dev/null || echo "$HS_ENV_FILE.tmp.$$")
|
|
if [ -f "$HS_ENV_FILE" ]; then
|
|
grep -v "^${key}=" "$HS_ENV_FILE" 2>/dev/null > "$tmp" || true
|
|
else
|
|
: > "$tmp"
|
|
fi
|
|
printf '%s=%s\n' "$key" "$val" >> "$tmp"
|
|
mv "$tmp" "$HS_ENV_FILE" 2>/dev/null || { hs_err "could not write $HS_ENV_FILE"; rm -f "$tmp"; return 1; }
|
|
chmod 600 "$HS_ENV_FILE" 2>/dev/null || true
|
|
return 0
|
|
}
|
|
|
|
# Resolve ADDR/PORT/SOCK for the target alias; verify the master is open.
|
|
# Sets _HS_ADDR _HS_PORT _HS_SOCK. Returns non-zero (graceful) if not ready.
|
|
_hs_resolve_master() {
|
|
local alias="$1"
|
|
[ -n "$alias" ] || return 1
|
|
[ -f "$SSH_HOSTS_FILE" ] || return 1
|
|
local addr_port
|
|
addr_port=$(awk -F'\t' -v a="$alias" 'NR>1 && $1==a { print $2 "\t" $3; exit }' < "$SSH_HOSTS_FILE")
|
|
[ -n "$addr_port" ] || return 1
|
|
_HS_ADDR=$(printf '%s' "$addr_port" | cut -f1)
|
|
_HS_PORT=$(printf '%s' "$addr_port" | cut -f2)
|
|
[ -n "$_HS_PORT" ] || _HS_PORT=22
|
|
_HS_SOCK="$SSH_SOCKETS_DIR/$alias.sock"
|
|
[ -S "$_HS_SOCK" ] || return 2
|
|
ssh -S "$_HS_SOCK" -O check -p "$_HS_PORT" "$_HS_ADDR" 2>/dev/null || return 2
|
|
return 0
|
|
}
|
|
|
|
# Compute the default remote destination path for this work-box.
|
|
# ~/.cloverleaf/headers-<hostname>.jsonl (a daemon-watched dir on the Mac)
|
|
# We use a tilde-relative remote path so we don't need to know the Mac's home;
|
|
# the remote shell expands ~ to the Mac user's home.
|
|
_hs_default_dest() {
|
|
local h; h=$(hostname 2>/dev/null | tr -cd 'A-Za-z0-9_.-')
|
|
[ -n "$h" ] || h="workbox"
|
|
printf '~/.cloverleaf/headers-%s.jsonl' "$h"
|
|
}
|
|
|
|
_hs_load_offset() {
|
|
coerce_int "$(cat "$HS_OFFSET_FILE" 2>/dev/null)" 0
|
|
}
|
|
_hs_save_offset() {
|
|
umask 077
|
|
printf '%s' "$(coerce_int "$1" 0)" > "$HS_OFFSET_FILE" 2>/dev/null || true
|
|
}
|
|
|
|
# _hs_write_state TS BYTES_TOTAL — record last-sync info for `status`.
|
|
_hs_write_state() {
|
|
umask 077
|
|
printf 'last_sync=%s\ntotal_bytes_pushed=%s\n' "$1" "$(coerce_int "$2" 0)" \
|
|
> "$HS_STATE_FILE" 2>/dev/null || true
|
|
}
|
|
_hs_read_state_field() {
|
|
local field="$1"
|
|
[ -f "$HS_STATE_FILE" ] || { printf ''; return; }
|
|
awk -F= -v f="$field" '$1==f { print $2; exit }' "$HS_STATE_FILE" 2>/dev/null
|
|
}
|
|
|
|
# ── core: incremental push ───────────────────────────────────────────────────
|
|
# _hs_do_sync — push the new bytes of headers.log to the configured Mac target.
|
|
# Returns 0 on success/no-op, 2 on a (logged, non-fatal) failure.
|
|
_hs_do_sync() {
|
|
_hs_load_env
|
|
local alias="${LARRY_HEADERS_SYNC_TARGET:-}"
|
|
local dest="${LARRY_HEADERS_SYNC_DEST:-$(_hs_default_dest)}"
|
|
|
|
if [ -z "$alias" ]; then
|
|
hs_warn "no target configured — run /headers-sync target <alias> first"
|
|
_hs_log "SKIP no-target"
|
|
return 0
|
|
fi
|
|
if [ ! -f "$HS_HEADERS_LOG" ]; then
|
|
hs_warn "no local headers.log yet ($HS_HEADERS_LOG) — nothing to sync"
|
|
_hs_log "SKIP no-local-headers"
|
|
return 0
|
|
fi
|
|
if [ ! -x "$HS_HELPER" ]; then
|
|
hs_warn "ssh-helper.sh not installed at $HS_HELPER"
|
|
_hs_log "SKIP no-helper"
|
|
return 0
|
|
fi
|
|
|
|
# Verify the ControlMaster is open. Graceful skip if not.
|
|
if ! _hs_resolve_master "$alias"; then
|
|
case $? in
|
|
1) hs_warn "alias '$alias' not configured (run /ssh-add $alias ...)";;
|
|
*) hs_warn "master for '$alias' is not open — run /ssh-setup $alias";;
|
|
esac
|
|
_hs_log "SKIP master-not-ready alias=$alias"
|
|
return 0
|
|
fi
|
|
|
|
local size offset
|
|
size=$(coerce_int "$(wc -c < "$HS_HEADERS_LOG" 2>/dev/null)" 0)
|
|
offset=$(_hs_load_offset)
|
|
|
|
local reseed=0
|
|
if [ "$offset" -gt "$size" ]; then
|
|
# local file shrank (rotation/truncation) — re-seed the remote file.
|
|
reseed=1
|
|
offset=0
|
|
fi
|
|
if [ "$offset" -eq "$size" ]; then
|
|
hs_say "no new bytes (offset=$offset size=$size) — no-op"
|
|
_hs_log "NOOP alias=$alias offset=$offset size=$size dest=$dest"
|
|
return 0
|
|
fi
|
|
|
|
local new_bytes=$((size - offset))
|
|
|
|
# Ensure the remote dir exists. We pass the dest through the remote shell so
|
|
# ~ expands there; quote-escape only the dirname computation.
|
|
local remote_dir_cmd
|
|
remote_dir_cmd="d=\$(dirname \"$dest\"); mkdir -p \"\$d\" 2>/dev/null; "
|
|
if [ "$reseed" -eq 1 ]; then
|
|
# truncate the remote file before appending from offset 0
|
|
remote_dir_cmd+=": > \"$dest\"; "
|
|
fi
|
|
|
|
# Stream ONLY the new bytes (dd skip=offset) over the open master and append
|
|
# remotely. dd bs=1 is correct-but-slow; headers.log is tiny per-turn, so
|
|
# bs=1 is fine and avoids partial-block math. We pipe through ssh -S.
|
|
local rc
|
|
if dd if="$HS_HEADERS_LOG" bs=1 skip="$offset" count="$new_bytes" 2>/dev/null \
|
|
| ssh -S "$_HS_SOCK" -p "$_HS_PORT" -o BatchMode=yes "$_HS_ADDR" \
|
|
"$remote_dir_cmd cat >> \"$dest\""; then
|
|
rc=0
|
|
else
|
|
rc=$?
|
|
fi
|
|
|
|
if [ "$rc" -ne 0 ]; then
|
|
hs_err "transport failed (rc=$rc) pushing $new_bytes bytes to $alias:$dest"
|
|
_hs_log "FAIL alias=$alias dest=$dest new_bytes=$new_bytes rc=$rc (offset NOT advanced)"
|
|
return 2
|
|
fi
|
|
|
|
# Verify remote size advanced by at least new_bytes (idempotency check).
|
|
local remote_size
|
|
remote_size=$(coerce_int "$(ssh -S "$_HS_SOCK" -p "$_HS_PORT" -o BatchMode=yes "$_HS_ADDR" \
|
|
"wc -c < \"$dest\" 2>/dev/null" 2>/dev/null)" 0)
|
|
|
|
# Advance the offset ONLY after a confirmed push.
|
|
_hs_save_offset "$size"
|
|
local now; now=$(date -Iseconds 2>/dev/null || date)
|
|
_hs_write_state "$now" "$size"
|
|
hs_say "pushed $new_bytes new byte(s) → $alias:$dest (offset now $size; remote=$remote_size bytes)"
|
|
_hs_log "OK alias=$alias dest=$dest pushed=$new_bytes offset=$size remote=$remote_size reseed=$reseed"
|
|
return 0
|
|
}
|
|
|
|
# ── subcommands ──────────────────────────────────────────────────────────────
|
|
cmd_on() {
|
|
_hs_persist_env LARRY_HEADERS_SYNC 1 || return 1
|
|
export LARRY_HEADERS_SYNC=1
|
|
hs_say "auto-sync ENABLED (persisted to $HS_ENV_FILE). Headers.log will sync on larry exit (and via /headers-sync now)."
|
|
_hs_load_env
|
|
[ -z "${LARRY_HEADERS_SYNC_TARGET:-}" ] && hs_warn "no target set yet — run /headers-sync target <alias>"
|
|
}
|
|
|
|
cmd_off() {
|
|
_hs_persist_env LARRY_HEADERS_SYNC 0 || return 1
|
|
export LARRY_HEADERS_SYNC=0
|
|
hs_say "auto-sync DISABLED (persisted). /headers-sync now still works on demand."
|
|
}
|
|
|
|
cmd_target() {
|
|
local alias="${1:-}"
|
|
[ -n "$alias" ] || { hs_err "usage: /headers-sync target <alias>"; return 1; }
|
|
_hs_persist_env LARRY_HEADERS_SYNC_TARGET "$alias" || return 1
|
|
export LARRY_HEADERS_SYNC_TARGET="$alias"
|
|
hs_say "target set: $alias (persisted). Confirm the master is open with /ssh-status; open it with /ssh-setup $alias."
|
|
# Best-effort readiness hint.
|
|
if _hs_resolve_master "$alias"; then
|
|
hs_say "master for '$alias' is OPEN — ready to sync."
|
|
else
|
|
hs_warn "master for '$alias' not open yet — run /ssh-setup $alias before the next sync."
|
|
fi
|
|
}
|
|
|
|
cmd_status() {
|
|
_hs_load_env
|
|
local enabled="${LARRY_HEADERS_SYNC:-0}"
|
|
local alias="${LARRY_HEADERS_SYNC_TARGET:-}"
|
|
local dest="${LARRY_HEADERS_SYNC_DEST:-$(_hs_default_dest)}"
|
|
local offset; offset=$(_hs_load_offset)
|
|
local size="(no local headers.log)"
|
|
[ -f "$HS_HEADERS_LOG" ] && size=$(coerce_int "$(wc -c < "$HS_HEADERS_LOG" 2>/dev/null)" 0)
|
|
local last_sync; last_sync=$(_hs_read_state_field last_sync)
|
|
local total; total=$(_hs_read_state_field total_bytes_pushed)
|
|
[ -n "$last_sync" ] || last_sync="(never)"
|
|
[ -n "$total" ] || total="0"
|
|
|
|
printf 'headers-sync status\n'
|
|
printf ' enabled: %s\n' "$([ "$enabled" = "1" ] && echo yes || echo no)"
|
|
printf ' target alias: %s\n' "${alias:-(unset — /headers-sync target <alias>)}"
|
|
printf ' remote dest: %s\n' "$dest"
|
|
printf ' local headers: %s (%s bytes)\n' "$HS_HEADERS_LOG" "$size"
|
|
printf ' pushed offset: %s bytes\n' "$offset"
|
|
printf ' last sync: %s\n' "$last_sync"
|
|
printf ' total pushed: %s bytes (at last sync)\n' "$total"
|
|
printf ' audit log: %s\n' "$HS_SYNC_LOG"
|
|
if [ -n "$alias" ]; then
|
|
if _hs_resolve_master "$alias"; then
|
|
printf ' master: OPEN (%s)\n' "$alias"
|
|
else
|
|
printf ' master: not open — run /ssh-setup %s\n' "$alias"
|
|
fi
|
|
fi
|
|
}
|
|
|
|
cmd_now() {
|
|
_hs_do_sync
|
|
}
|
|
|
|
# `sync` — the auto hook: run only if enabled. larry.sh calls this on exit.
|
|
cmd_sync_if_enabled() {
|
|
_hs_load_env
|
|
if [ "${LARRY_HEADERS_SYNC:-0}" != "1" ]; then
|
|
return 0
|
|
fi
|
|
_hs_do_sync
|
|
}
|
|
|
|
cmd_help() { sed -n '2,75p' "$0"; }
|
|
|
|
case "${1:-help}" in
|
|
on) shift; cmd_on "$@" ;;
|
|
off) shift; cmd_off "$@" ;;
|
|
target) shift; cmd_target "$@" ;;
|
|
status) shift; cmd_status "$@" ;;
|
|
now) shift; cmd_now "$@" ;;
|
|
sync) shift; cmd_sync_if_enabled "$@" ;;
|
|
-h|--help|help) cmd_help ;;
|
|
*) hs_err "unknown subcommand: ${1:-} (try: on|off|target <alias>|status|now)"; exit 1 ;;
|
|
esac
|