#!/usr/bin/env python3 """HAGOP Robot Lab: push real TensorBoard scalar metrics to the private Railway API. Run on the GPU Pod in a second terminal, after starting XgoDuck training: ROBOT_LAB_TELEMETRY_TOKEN= \ uv run --with tensorboard python /workspace/robot-lab-reporter.py Never put the telemetry token in public HTML, git, command history or chat. """ import json import os from pathlib import Path import re import socket import sys import time from urllib.request import Request, urlopen API = "https://robot-lab-api-production.up.railway.app/api/monitor/ingest" TOKEN = os.environ.get("ROBOT_LAB_TELEMETRY_TOKEN", "") LOG_ROOT = Path(os.environ.get("ROBOT_LAB_LOG_ROOT", "/workspace/robot-lab/xgoduck_rl/logs/rsl_rl")) RUN_ID = os.environ.get("ROBOT_LAB_RUN_ID", "xgo-" + re.sub(r"[^a-z0-9_-]", "", socket.gethostname().lower())[:32]) POLL_SECONDS = max(5.0, min(30.0, float(os.environ.get("ROBOT_LAB_POLL_SECONDS", "10")))) TARGET_TAGS = re.compile(r"reward|loss|episode|iteration|velocity|error|entropy", re.I) def current_events_folder(): files = list(LOG_ROOT.rglob("events.out.tfevents*")) if LOG_ROOT.exists() else [] if not files: return None return max(files, key=lambda path: path.stat().st_mtime).parent def latest_scalars(folder): from tensorboard.backend.event_processing.event_accumulator import EventAccumulator accumulator = EventAccumulator(str(folder), size_guidance={"scalars": 2}) accumulator.Reload() names = accumulator.Tags().get("scalars", []) ranked = sorted(names, key=lambda tag: (not bool(TARGET_TAGS.search(tag)), tag)) metrics, step = {}, 0 for tag in ranked[:20]: events = accumulator.Scalars(tag) if not events: continue last = events[-1] if not isinstance(last.value, (float, int)): continue metrics[tag[:96]] = float(last.value) step = max(step, int(last.step)) return step, metrics def post_sample(step, metrics): payload = json.dumps({"runId": RUN_ID, "step": step, "metrics": metrics}).encode("utf-8") request = Request(API, data=payload, headers={ "Content-Type": "application/json", "Authorization": "Bearer " + TOKEN, }, method="POST") with urlopen(request, timeout=10) as response: return response.status def main(): if len(TOKEN) < 32: sys.exit("ROBOT_LAB_TELEMETRY_TOKEN is absent or too short. Set it privately on the Pod.") if not re.fullmatch(r"[A-Za-z0-9_-]{1,64}", RUN_ID): sys.exit("Invalid ROBOT_LAB_RUN_ID; use letters, digits, _ and -.") print("Robot Lab telemetry reporter: reading real TensorBoard scalars.") print("Source:", LOG_ROOT) print("Run ID:", RUN_ID) print("No GPU will be provisioned by this reporter.") last_sent = None while True: try: folder = current_events_folder() if folder is None: print("Waiting for PPO TensorBoard event files...", flush=True) else: step, metrics = latest_scalars(folder) if metrics and step != last_sent: status = post_sample(step, metrics) last_sent = step print("Forwarded", len(metrics), "metrics at step", step, "status", status, flush=True) except KeyboardInterrupt: print("Reporter stopped; RunPod Pod must be stopped separately.") break except Exception as error: print("Reporter error:", type(error).__name__, str(error)[:200], flush=True) time.sleep(POLL_SECONDS) if __name__ == "__main__": main()