incident: SFTPGo outage 2026-09-02 — CT299 tailscaled death + stale lxc-attach recovery
- New incident page documenting CT299 SFTPGo outage (02:18 AWST) - Root cause: swap exhaustion (43Gi, 100%) killed tailscaled; 5 stale lxc-attach PIDs blocked pct start - Resolution: killed PIDs 3037951/3334928/3352203/3402511/4004148, pct stop/start 299 - Updated current-state.md with Recent Changes (2026-09-02), bumped updated to 2026-09-04 - Updated log.md with incident entry - Updated index.md last-updated date No secrets written. Verified end-to-end: tailscale direct, SFTPGo WebAdmin HTTP 401, SFTP banner SSH-2.0-SFTPGo_2.7.0.
This commit is contained in:
Tony0410
@@ -1,134 +0,0 @@
|
||||
#!/bin/bash
|
||||
# wiki-maintain.sh — General Agent Estate Wiki freshness maintenance.
|
||||
# Regenerates machine-derivable pages from live state, audits every page for
|
||||
# staleness, rebuilds the Quartz site, and deploys. Run on a schedule so the
|
||||
# wiki never silently drifts from reality.
|
||||
#
|
||||
# Usage: ./wiki-maintain.sh [--no-build]
|
||||
# --no-build regenerate content + audit but skip quartz build/deploy
|
||||
set -euo pipefail
|
||||
|
||||
WIKI="/home/hermes/workspace/agent-estate-wiki"
|
||||
QUARTZ="/home/hermes/workspace/quartz-temp"
|
||||
SKILLS="/home/hermes/.hermes/skills"
|
||||
STALE_DAYS=21
|
||||
NO_BUILD=false
|
||||
[[ "${1:-}" == "--no-build" ]] && NO_BUILD=true
|
||||
|
||||
mkdir -p "$WIKI/raw/stocktake"
|
||||
REPORT="$WIKI/raw/stocktake/freshness-$(date +%Y%m%d).txt"
|
||||
|
||||
echo "[$(date -Is)] Agent Estate Wiki maintenance start"
|
||||
echo " wiki=${WIKI}"
|
||||
echo " stale-threshold=${STALE_DAYS}d"
|
||||
|
||||
# ── 1. Regenerate skills-index from live skill tree ────────────────────────
|
||||
python3 - "$SKILLS" "$WIKI/systems/skills-index.md" <<'PY'
|
||||
import sys, re
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
from datetime import date
|
||||
skills_root = Path(sys.argv[1]); out = Path(sys.argv[2])
|
||||
files = sorted(skills_root.rglob('SKILL.md'))
|
||||
skills = defaultdict(list)
|
||||
for f in files:
|
||||
rel = f.relative_to(skills_root); parts = rel.parts
|
||||
if len(parts) >= 3: cat, name = parts[0], parts[1]
|
||||
elif len(parts) == 2: cat, name = 'uncategorized', parts[0]
|
||||
else: continue
|
||||
desc = ''
|
||||
try:
|
||||
m = re.search(r'^description:\s*(.+)$', f.read_text(errors='ignore')[:2000], re.M)
|
||||
if m: desc = m.group(1).strip().strip('"').strip("'")[:120]
|
||||
except Exception: pass
|
||||
skills[cat].append((name, desc))
|
||||
total = sum(len(v) for v in skills.values())
|
||||
L = ['---','title: Installed Skills','type: system','status: active',
|
||||
'created: 2026-07-22', f'updated: {date.today().isoformat()}',
|
||||
f'verified_on: {date.today().isoformat()}','confidence: high',
|
||||
'tags: [skills, hermes]','sources: []','---','','# Installed Skills','',
|
||||
f'**Total installed: {total} skills** in `~/.hermes/skills/`.','',
|
||||
'This page is auto-regenerated from the live skill inventory. Do not hand-edit.','',
|
||||
'## Stock-take notes','- Category = directory structure under `~/.hermes/skills/` (`category/skill/SKILL.md`).',
|
||||
'- Skills directly under `~/.hermes/skills/` group under `uncategorized`.','']
|
||||
priority = ['core-hermes','autonomous-ai-agents','hermes','devops','productivity','creative',
|
||||
'software-development','research','email','mlops','github','apple','frontend','media',
|
||||
'note-taking','web-development','smart-home','mcp','uncategorized']
|
||||
def order(c):
|
||||
try: return priority.index(c)
|
||||
except ValueError: return 100+len(priority)
|
||||
for cat, items in sorted(skills.items(), key=lambda x:(order(x[0]),-len(x[1]))):
|
||||
L.append(f'## {cat.replace("-"," ").title()} ({len(items)})'); L.append('')
|
||||
for name, desc in sorted(items):
|
||||
L.append(f'- `{name}` — {desc}' if desc else f'- `{name}`')
|
||||
L.append('')
|
||||
L += ['## Related','- [[systems/hermes-agent]]','- [[systems/omniroute]]','']
|
||||
out.write_text('\n'.join(L))
|
||||
print(f"[1/3] skills: {total} across {len(skills)} categories -> {out}")
|
||||
PY
|
||||
|
||||
# ── 2. Audit every page for staleness ────────────────────────────────────────
|
||||
python3 - "$WIKI" "$REPORT" "$STALE_DAYS" <<'PY'
|
||||
import sys, re
|
||||
from pathlib import Path
|
||||
from datetime import date
|
||||
wiki = Path(sys.argv[1]); report = Path(sys.argv[2]); threshold = int(sys.argv[3])
|
||||
today = date.today()
|
||||
stale, fresh, nodate = [], [], []
|
||||
for f in sorted(wiki.rglob('*.md')):
|
||||
rel = f.relative_to(wiki)
|
||||
if str(rel).startswith('raw/'): continue
|
||||
content = f.read_text(errors='ignore')
|
||||
upd = re.search(r'^updated:\s*(\S+)', content[:800], re.M)
|
||||
if not upd:
|
||||
nodate.append(str(rel)); continue
|
||||
try: d = date.fromisoformat(upd.group(1))
|
||||
except ValueError: nodate.append(str(rel)); continue
|
||||
age = (today - d).days
|
||||
(stale if age > threshold else fresh).append((str(rel), age, upd.group(1)))
|
||||
stale.sort(key=lambda x:-x[1])
|
||||
lines = [f"# Wiki freshness audit — {today.isoformat()}",
|
||||
f"Threshold: >{threshold} days since `updated`",
|
||||
f"", f"## STALE ({len(stale)})", ""]
|
||||
for rel, age, d in stale:
|
||||
lines.append(f"- [{age:>3}d] {rel} (updated {d})")
|
||||
lines += [f"", f"## FRESH ({len(fresh)})", ""]
|
||||
for rel, age, d in fresh:
|
||||
lines.append(f"- [{age:>3}d] {rel}")
|
||||
lines += [f"", f"## NO updated DATE ({len(nodate)})", ""]
|
||||
for rel in nodate:
|
||||
lines.append(f"- {rel}")
|
||||
lines += ["", f"TOTAL: {len(stale)} stale, {len(fresh)} fresh, {len(nodate)} no-date of {len(stale)+len(fresh)+len(nodate)} pages"]
|
||||
report.write_text('\n'.join(lines))
|
||||
print(f"[2/3] audit: {len(stale)} stale (>={threshold}d), {len(fresh)} fresh, {len(nodate)} no-date -> {report}")
|
||||
PY
|
||||
|
||||
# ── 3. Sync content, build, deploy ─────────────────────────────────────────
|
||||
if $NO_BUILD; then
|
||||
echo "[3/3] --no-build: content + audit done, skipping quartz build"
|
||||
exit 0
|
||||
fi
|
||||
rm -rf "$QUARTZ/content" "$QUARTZ/public"
|
||||
mkdir -p "$QUARTZ/content"
|
||||
python3 - "$WIKI" "$QUARTZ/content" <<'PY'
|
||||
import shutil, sys
|
||||
from pathlib import Path
|
||||
src = Path(sys.argv[1]); dst = Path(sys.argv[2])
|
||||
skip = {'.git','public','node_modules','quartz-site','serve.py','.gitignore',
|
||||
'package.json','package-lock.json','.codewiki-state.json','regenerate-skills-index.sh',
|
||||
'wiki-maintain.sh','raw'}
|
||||
for item in src.iterdir():
|
||||
if item.name in skip: continue
|
||||
if item.is_dir(): shutil.copytree(item, dst/item.name, dirs_exist_ok=True)
|
||||
elif item.suffix == '.md': shutil.copy2(item, dst/item.name)
|
||||
PY
|
||||
cd "$QUARTZ"
|
||||
npx quartz build >/tmp/quartz-build.log 2>&1 || { echo "Quartz build FAILED"; tail -20 /tmp/quartz-build.log; exit 1; }
|
||||
rm -rf "$WIKI/public"
|
||||
mkdir -p "$WIKI/public"
|
||||
cp -a "$QUARTZ/public/." "$WIKI/public/"
|
||||
echo "[3/3] build+deploy: $(find "$WIKI/public" -name '*.html' | wc -l) html files"
|
||||
echo "[$(date -Is)] Wiki maintenance complete"
|
||||
echo ""
|
||||
echo "=== Freshness summary ==="
|
||||
grep -E '^TOTAL:' "$REPORT" || true
|
||||
Reference in New Issue
Block a user