Files
job-dashboard/job_dashboard_server.py

563 lines
25 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Job Dashboard API Server — serves the dashboard + Notion sync API.
Replaces static http.server. Handles:
- Static file serving (dashboard HTML, assets)
- REST API for job CRUD (syncs back to Notion)
- Auto-refresh from Notion every 5 min
"""
import json
import os
import re
import subprocess
import sys
import time
import traceback
import urllib.request
import xml.etree.ElementTree as ET
from datetime import datetime, timezone
from http.server import HTTPServer, SimpleHTTPRequestHandler
from urllib.parse import urlparse, parse_qs
# ── Config ───────────────────────────────────────────────────────────────────
PORT = 9099
DASHBOARD_DIR = os.path.expanduser("~/.hermes/dashboard")
DRAFTS_DIR = os.path.expanduser("~/.hermes/radar-ref/drafts")
DB_ID = "0f90ba2b-8b10-4d02-a62c-6f692d5b1168"
ENV_FILE = os.path.expanduser("~/.hermes/.env")
CACHE_FILE = os.path.join(DASHBOARD_DIR, "jobs_cache.json")
REFRESH_INTERVAL = 300 # 5 min
# ── Cache ────────────────────────────────────────────────────────────────────
_jobs_cache = None
_last_refresh = 0
def get_notion_key():
with open(ENV_FILE) as f:
for line in f:
line = line.strip()
if line.startswith("NOTION_API_KEY="):
return line.split("=", 1)[1]
return None
def notion_api(method, path, data=None):
"""Call Notion REST API directly."""
key = get_notion_key()
if not key:
return {"error": "NOTION_API_KEY not found"}
url = f"https://api.notion.com/v1/{path.lstrip('/')}"
cmd = ["curl", "-s", "-X", method, url,
"-H", f"Authorization: Bearer {key}",
"-H", "Notion-Version: 2022-06-28",
"-H", "Content-Type: application/json"]
if data is not None:
cmd += ["-d", json.dumps(data)]
result = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
return json.loads(result.stdout)
def refresh_cache(force=False):
"""Fetch all jobs from Notion and cache."""
global _jobs_cache, _last_refresh
now = time.time()
if not force and _jobs_cache and (now - _last_refresh) < REFRESH_INTERVAL:
return _jobs_cache
all_results = []
has_more = True
start_cursor = None
while has_more:
payload = {
"page_size": 100,
"sorts": [{"property": "Date Added", "direction": "descending"}],
}
if start_cursor:
payload["start_cursor"] = start_cursor
data = notion_api("POST", f"databases/{DB_ID}/query", payload)
if "error" in data:
print(f"Notion API error: {data['error']}", file=sys.stderr)
if _jobs_cache:
return _jobs_cache
return {"error": data["error"]}
all_results.extend(data.get("results", []))
has_more = data.get("has_more", False)
start_cursor = data.get("next_cursor")
jobs = []
for r in all_results:
props = r["properties"]
job = {
"id": r["id"],
"name": "",
"role": "",
"status": "Backlog",
"location": "",
"source": "",
"strong_match": False,
"ai_sourced": False,
"date_added": "",
"url": "",
"salary": None,
"notes": "",
"priority": "",
}
if props.get("Name", {}).get("title"):
job["name"] = props["Name"]["title"][0].get("plain_text", "")
if props.get("Role", {}).get("rich_text"):
job["role"] = props["Role"]["rich_text"][0].get("plain_text", "")
if props.get("Status", {}).get("select"):
job["status"] = props["Status"]["select"]["name"]
if props.get("Location", {}).get("select"):
job["location"] = props["Location"]["select"]["name"]
if props.get("Source", {}).get("select"):
job["source"] = props["Source"]["select"]["name"]
job["strong_match"] = props.get("Strong Match", {}).get("checkbox", False)
job["ai_sourced"] = props.get("AI Sourced", {}).get("checkbox", False)
if props.get("Date Added", {}).get("date"):
job["date_added"] = props["Date Added"]["date"].get("start", "")
if props.get("URL", {}).get("url"):
job["url"] = props["URL"]["url"]
if props.get("Salary (AUD/yr)", {}).get("number") is not None:
job["salary"] = props["Salary (AUD/yr)"]["number"]
if props.get("Notes / JD Summary", {}).get("rich_text"):
job["notes"] = props["Notes / JD Summary"]["rich_text"][0].get("plain_text", "")
if props.get("Priority", {}).get("select"):
job["priority"] = props["Priority"]["select"]["name"]
if props.get("Closing Date", {}).get("date"):
job["closing_date"] = props["Closing Date"]["date"].get("start", "")
else:
job["closing_date"] = ""
# Closing date calculations
job["closing_status"] = "none"
if job.get("closing_date"):
try:
from datetime import date
cd = datetime.strptime(job["closing_date"], "%Y-%m-%d").date()
today = date.today()
days_left = (cd - today).days
if days_left < 0:
job["closing_status"] = "past"
elif days_left <= 7:
job["closing_status"] = "urgent"
elif days_left <= 14:
job["closing_status"] = "warning"
except ValueError:
pass
if props.get("Cover Letter", {}).get("url"):
job["cover_letter_url"] = props["Cover Letter"]["url"]
else:
job["cover_letter_url"] = ""
if job["date_added"]:
try:
added = datetime.strptime(job["date_added"], "%Y-%m-%d")
now_dt = datetime.now()
job["days_since"] = (now_dt - added).days
except ValueError:
job["days_since"] = 0
else:
job["days_since"] = 0
jobs.append(job)
_jobs_cache = {"jobs": jobs, "total": len(jobs), "refreshed_at": datetime.now().isoformat()}
_last_refresh = now
# Also write the cache file
try:
os.makedirs(DASHBOARD_DIR, exist_ok=True)
with open(CACHE_FILE, "w") as f:
json.dump(_jobs_cache, f)
except Exception:
pass
return _jobs_cache
def update_job_status(job_id, updates):
"""Update a Notion job page with new property values."""
notion_props = {}
if "status" in updates:
notion_props["Status"] = {"select": {"name": updates["status"]}}
if "strong_match" in updates:
notion_props["Strong Match"] = {"checkbox": bool(updates["strong_match"])}
if "priority" in updates:
notion_props["Priority"] = {"select": {"name": updates["priority"]}}
if "notes" in updates:
notion_props["Notes / JD Summary"] = {
"rich_text": [{"type": "text", "text": {"content": updates["notes"]}}]
}
if "cover_letter_url" in updates:
notion_props["Cover Letter"] = {"url": updates["cover_letter_url"]}
if not notion_props:
return {"error": "No valid fields to update"}
result = notion_api("PATCH", f"pages/{job_id}", {"properties": notion_props})
return result
# ── HTTP Handler ─────────────────────────────────────────────────────────────
class DashboardHandler(SimpleHTTPRequestHandler):
def __init__(self, *args, **kwargs):
super().__init__(*args, directory=DASHBOARD_DIR, **kwargs)
def do_GET(self):
parsed = urlparse(self.path)
path = parsed.path
# API routes
if path == "/api/jobs":
return self._json_response(refresh_cache())
elif path == "/api/stats":
data = refresh_cache()
jobs = data.get("jobs", [])
statuses = {}
for j in jobs:
s = j["status"]
statuses[s] = statuses.get(s, 0) + 1
locations = {}
for j in jobs:
l = j["location"] or "Unknown"
locations[l] = locations.get(l, 0) + 1
sources = {}
for j in jobs:
s = j["source"] or "Unknown"
sources[s] = sources.get(s, 0) + 1
salaries = [j["salary"] for j in jobs if j["salary"]]
stats = {
"total": data["total"],
"strong_matches": sum(1 for j in jobs if j["strong_match"]),
"ai_sourced": sum(1 for j in jobs if j["ai_sourced"]),
"this_week": sum(1 for j in jobs if j["days_since"] <= 7),
"stale_14d": sum(1 for j in jobs if j["days_since"] > 14),
"closing_past": sum(1 for j in jobs if j.get("closing_status") == "past"),
"closing_urgent": sum(1 for j in jobs if j.get("closing_status") == "urgent"),
"closing_warning": sum(1 for j in jobs if j.get("closing_status") == "warning"),
"has_closing": sum(1 for j in jobs if j.get("closing_date")),
"statuses": statuses,
"locations": locations,
"sources": sources,
"salary_min": min(salaries) if salaries else None,
"salary_max": max(salaries) if salaries else None,
"refreshed_at": data.get("refreshed_at", ""),
}
return self._json_response(stats)
elif path == "/api/refresh":
data = refresh_cache(force=True)
return self._json_response({"ok": True, "total": data["total"]})
elif path.startswith("/api/jobs/") and path.endswith("/refresh"):
data = refresh_cache(force=True)
return self._json_response({"ok": True, "total": data["total"]})
# GET /api/market-intel — salary distribution + trend analysis
elif path == "/api/market-intel":
data = refresh_cache()
jobs = data.get("jobs", [])
salaries = [j["salary"] for j in jobs if j["salary"]]
brackets = [
{"label": "Under $80k", "min": 0, "max": 80000},
{"label": "$80k$100k", "min": 80000, "max": 100000},
{"label": "$100k$120k", "min": 100000, "max": 120000},
{"label": "$120k$140k", "min": 120000, "max": 140000},
{"label": "$140k+", "min": 140000, "max": 999999},
]
distribution = [{"label": b["label"], "count": sum(1 for s in salaries if b["min"] <= s < b["max"])} for b in brackets]
locs = {}
for j in jobs:
loc = j["location"] or "Unknown"
if j["salary"]:
locs.setdefault(loc, []).append(j["salary"])
loc_stats = {loc: {"avg": round(sum(s)/len(s)), "min": min(s), "max": max(s), "count": len(s)} for loc, s in locs.items()}
months = {}
for j in jobs:
if j["date_added"]:
m = j["date_added"][:7]
months[m] = months.get(m, 0) + 1
sources = {}
for j in jobs:
src = j["source"] or "Unknown"
s = sources.setdefault(src, {"count": 0, "salary_sum": 0, "salary_count": 0})
s["count"] += 1
if j["salary"]:
s["salary_sum"] += j["salary"]
s["salary_count"] += 1
src_stats = {src: {"count": d["count"], "avg_salary": round(d["salary_sum"]/d["salary_count"]) if d["salary_count"] else None} for src, d in sources.items()}
statuses = {}
for j in jobs:
s = j["status"]
statuses[s] = statuses.get(s, 0) + 1
return self._json_response({
"total_tracked": len(jobs),
"with_salary": len(salaries),
"avg_salary": round(sum(salaries)/len(salaries)) if salaries else 0,
"median_salary": sorted(salaries)[len(salaries)//2] if salaries else 0,
"min_salary": min(salaries) if salaries else 0,
"max_salary": max(salaries) if salaries else 0,
"salary_distribution": distribution,
"by_location": loc_stats,
"monthly_trend": dict(sorted(months.items())),
"by_source": src_stats,
"by_status": statuses,
"refreshed_at": data.get("refreshed_at", ""),
})
# GET /api/jobs/<id>/match — match analysis for a job
match_job_match = re.match(r"^/api/jobs/([a-f0-9\-]+)/match$", path)
if match_job_match:
job_id = match_job_match.group(1)
data = refresh_cache()
job = next((j for j in data.get("jobs", []) if j["id"] == job_id), None)
if not job:
return self._json_response({"error": "Job not found"}, 404)
score = 50
signals = []
if job.get("strong_match"):
score += 20; signals.append("Strong match flag")
if job.get("salary"):
score += 5; signals.append("Has salary data")
if job.get("notes") and len(job.get("notes", "")) > 50:
score += 10; signals.append("Detailed JD summary")
elif job.get("notes"):
score += 5; signals.append("Has JD summary")
if job.get("cover_letter_url"):
score += 10; signals.append("Cover letter exists")
if job.get("role") and job.get("name"):
score += 5; signals.append("Company + role populated")
if job.get("url"):
score += 5; signals.append("Job URL available")
score = min(score, 99)
return self._json_response({
"score": score,
"signals": signals,
"strong_match": job.get("strong_match", False),
"has_cl": bool(job.get("cover_letter_url")),
"has_salary": bool(job.get("salary")),
"has_notes": bool(job.get("notes")),
"job_id": job_id,
})
# GET /ops — serve the ops timeline page
elif path == "/api/feed-proxy":
# Fetch multiple RSS feeds and return as JSON
qs = parse_qs(parsed.query)
# Support both ?urls=url1,url2 and multiple &urls=url1&urls=url2
raw_urls = qs.get("urls", [""])
if isinstance(raw_urls, list) and len(raw_urls) > 1:
# Multiple &urls= params — join them
all_urls = ",".join(raw_urls)
else:
all_urls = raw_urls[0] if isinstance(raw_urls, list) else raw_urls
if not all_urls:
return self._json_response({"error": "Provide ?urls=... (comma-separated URLs)"}, 400)
feed_urls = [u.strip() for u in all_urls.split(",") if u.strip()]
results = []
for fu in feed_urls:
try:
req = urllib.request.Request(fu, headers={
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
"Accept": "application/rss+xml, application/xml, text/xml, */*",
"Referer": "https://anthony.martinwa.org/",
})
with urllib.request.urlopen(req, timeout=15) as resp:
raw = resp.read()
root = ET.fromstring(raw)
# RSS 2.0 or Atom
ns = {"content": "http://purl.org/rss/1.0/modules/content/"}
items = []
# Try RSS 2.0
for item in root.iter("item"):
title = item.findtext("title", "")
link = item.findtext("link", "")
pubdate = item.findtext("pubDate", "")
desc = item.findtext("description", "")
# Strip HTML tags from description
desc_clean = re.sub(r"<[^>]+>", "", desc or "")
items.append({"title": title, "link": link, "date": pubdate[:25] if pubdate else "", "summary": desc_clean[:300]})
# Try Atom
if not items:
for entry in root.iter("{http://www.w3.org/2005/Atom}entry"):
title = entry.findtext("{http://www.w3.org/2005/Atom}title", "")
link_el = entry.find("{http://www.w3.org/2005/Atom}link")
link = link_el.get("href", "") if link_el is not None else ""
pubdate = entry.findtext("{http://www.w3.org/2005/Atom}published", "") or entry.findtext("{http://www.w3.org/2005/Atom}updated", "")
desc = entry.findtext("{http://www.w3.org/2005/Atom}summary", "")
items.append({"title": title, "link": link, "date": pubdate[:25], "summary": (desc or "")[:300]})
results.append({"url": fu, "items": items[:15], "count": len(items)})
except Exception as ex:
results.append({"url": fu, "error": str(ex)[:100], "items": []})
return self._json_response({"feeds": results})
elif path == "/ops":
self.path = "/ops.html"
return super().do_GET()
elif path == "/timeline":
self.path = "/timeline.html"
return super().do_GET()
elif path == "/portfolio":
self.path = "/portfolio.html"
return super().do_GET()
# Serve draft files (cover letters) — preserve extension for browser rendering
# Support both /drafts/ and /jobs/drafts/ paths (Tailscale Serve uses /jobs/drafts/ from prompt)
drafts_path = None
if path.startswith("/drafts/"):
safe_path = os.path.normpath(path.lstrip("/"))
drafts_path = os.path.join(DRAFTS_DIR, os.path.relpath(safe_path, "drafts"))
elif path.startswith("/jobs/drafts/"):
safe_path = os.path.normpath(path.lstrip("/"))
drafts_path = os.path.join(DRAFTS_DIR, os.path.relpath(safe_path, "jobs/drafts"))
if drafts_path:
if os.path.isfile(drafts_path) and os.path.realpath(drafts_path).startswith(os.path.realpath(DRAFTS_DIR)):
ext = os.path.splitext(drafts_path)[1].lower()
ctype = "text/plain; charset=utf-8"
if ext == ".html":
ctype = "text/html; charset=utf-8"
elif ext == ".json":
ctype = "application/json; charset=utf-8"
elif ext == ".docx":
ctype = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
self.send_response(200)
self.send_header("Content-Type", ctype)
self.send_header("Access-Control-Allow-Origin", "*")
self.send_header("Content-Disposition", f"inline; filename=\"{os.path.basename(drafts_path)}\"")
self.end_headers()
if ext == ".docx":
with open(drafts_path, "rb") as f:
self.wfile.write(f.read())
else:
with open(drafts_path) as f:
self.wfile.write(f.read().encode("utf-8"))
return
else:
return self._json_response({"error": "Not found"}, 404)
# Default: serve static file
return super().do_GET()
def do_POST(self):
parsed = urlparse(self.path)
path = parsed.path
content_length = int(self.headers.get("Content-Length", 0))
body = self.rfile.read(content_length) if content_length else b"{}"
try:
payload = json.loads(body)
except json.JSONDecodeError:
return self._json_response({"error": "Invalid JSON"}, 400)
# POST /api/refresh — force refresh from Notion
if path == "/api/refresh":
data = refresh_cache(force=True)
return self._json_response({"ok": True, "total": data["total"], "refreshed_at": data["refreshed_at"]})
# POST /api/jobs/<id>/update — update a job
match = re.match(r"^/api/jobs/([a-f0-9\-]+)/update$", path)
if match:
job_id = match.group(1)
result = update_job_status(job_id, payload)
if "error" in result:
return self._json_response(result, 400)
# Refresh cache after update
refresh_cache(force=True)
return self._json_response({"ok": True})
# POST /api/jobs/<id>/cover-letter/regenerate — regenerate cover letter with feedback
match_cl = re.match(r"^/api/jobs/([a-f0-9\-]+)/cover-letter/regenerate$", path)
if match_cl:
job_id = match_cl.group(1)
feedback = payload.get("feedback", "Make it stronger")
script = os.path.expanduser("~/.hermes/scripts/regenerate_cover_letter.py")
result = subprocess.run(
["python3", script, job_id, feedback],
capture_output=True, text=True, timeout=120
)
try:
cl_result = json.loads(result.stdout)
if "error" in cl_result:
return self._json_response(cl_result, 400)
# Update Notion with the new URL
update_result = update_job_status(job_id, {"cover_letter_url": cl_result["url"]})
if "error" in update_result:
return self._json_response({"error": "Draft saved but Notion update failed", "cl_result": cl_result, "notion_error": update_result}, 400)
# Refresh cache
refresh_cache(force=True)
return self._json_response({"ok": True, "url": cl_result["url"], "file": cl_result["file"]})
except json.JSONDecodeError:
return self._json_response({"error": "Regeneration failed", "output": result.stdout[:500]}, 400)
# POST /api/jobs/bulk-update — update multiple jobs
if path == "/api/jobs/bulk-update":
ids = payload.get("ids", [])
updates = payload.get("updates", {})
results = []
for jid in ids:
r = update_job_status(jid, updates)
results.append({"id": jid, "ok": "error" not in r})
refresh_cache(force=True)
return self._json_response({"ok": True, "results": results})
return self._json_response({"error": "Not found"}, 404)
def do_OPTIONS(self):
"""CORS preflight."""
self.send_response(204)
self.send_header("Access-Control-Allow-Origin", "*")
self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
self.send_header("Access-Control-Allow-Headers", "Content-Type")
self.end_headers()
def _json_response(self, data, status=200):
self.send_response(status)
self.send_header("Content-Type", "application/json")
self.send_header("Access-Control-Allow-Origin", "*")
self.send_header("Cache-Control", "no-store")
self.end_headers()
self.wfile.write(json.dumps(data).encode())
def log_message(self, format, *args):
"""Quiet logs — skip static file noise."""
msg = format % args
if "/api/" in msg:
print(f"[API] {msg}", file=sys.stderr)
else:
# Keep it very quiet for static files
pass
# ── Main ─────────────────────────────────────────────────────────────────────
def main():
# Ensure dashboard dir exists
os.makedirs(DASHBOARD_DIR, exist_ok=True)
# Pre-warm the cache
print("Pre-warming job cache from Notion...", file=sys.stderr)
refresh_cache(force=True)
print(f"Cache loaded: {_jobs_cache['total']} jobs", file=sys.stderr)
server = HTTPServer(("0.0.0.0", PORT), DashboardHandler)
print(f"Dashboard server running on http://127.0.0.1:{PORT}", file=sys.stderr)
print(f"Dashboard URL: https://hermes.kangaroo-eel.ts.net/jobs", file=sys.stderr)
print(f"API: GET /api/jobs /api/stats /api/refresh", file=sys.stderr)
print(f"API: POST /api/jobs/<id>/update /api/jobs/bulk-update", file=sys.stderr)
try:
server.serve_forever()
except KeyboardInterrupt:
print("\nShutting down...", file=sys.stderr)
server.server_close()
if __name__ == "__main__":
main()