APPS/ops-dashboard.git - git.mnsoft.org

..	..	@@ -1,3 +1,4 @@
	1	+import asyncio
1	2	import json
2	3	from datetime import datetime, timezone
3	4	from typing import AsyncGenerator, Literal
..	..	@@ -6,7 +7,9 @@
6	7	from fastapi.responses import StreamingResponse
7	8
8	9	from app.auth import verify_token
9		-from app.ops_runner import _BACKUP_TIMEOUT, stream_ops_host
	10	+from app.ops_runner import _BACKUP_TIMEOUT, new_op_id, is_cancelled, clear_cancelled, stream_ops_host
	11	+
	12	+_KEEPALIVE_INTERVAL = 15 # seconds between SSE keepalive pings
10	13
11	14	router = APIRouter()
12	15
..	..	@@ -29,52 +32,101 @@
29	32	Runs on the host via nsenter because ops restore delegates to project CLIs
30	33	that use host Python venvs incompatible with the container's Python.
31	34	"""
32		- base_args = ["restore", project, env]
	35	+ op_id = new_op_id()
	36	+ yield _sse_line({"op_id": op_id})
33	37
34		- # Pass the backup file path to avoid interactive selection prompt
35		- if name:
36		- backup_path = f"/opt/data/backups/{project}/{env}/{name}"
37		- base_args.append(backup_path)
	38	+ try:
	39	+ base_args = ["restore", project, env]
38	40
39		- if dry_run:
40		- base_args.append("--dry-run")
	41	+ # Pass the backup file path to avoid interactive selection prompt
	42	+ if name:
	43	+ backup_path = f"/opt/data/backups/{project}/{env}/{name}"
	44	+ base_args.append(backup_path)
41	45
42		- # Granular restore mode
43		- if mode == "db":
44		- base_args.append("--db-only")
45		- elif mode == "wp":
46		- base_args.append("--wp-only")
	46	+ if dry_run:
	47	+ base_args.append("--dry-run")
47	48
48		- if source == "offsite":
49		- # ops offsite restore <project> <env>
50		- download_args = ["offsite", "restore", project, env]
51		- yield _sse_line({"line": f"Downloading {project}/{env} from offsite...", "timestamp": _now()})
	49	+ # Granular restore mode
	50	+ if mode == "db":
	51	+ base_args.append("--db-only")
	52	+ elif mode == "wp":
	53	+ base_args.append("--wp-only")
52	54
53		- download_ok = True
54		- async for line in stream_ops_host(download_args, timeout=_BACKUP_TIMEOUT):
55		- yield _sse_line({"line": line, "timestamp": _now()})
56		- if line.startswith("[error]"):
57		- download_ok = False
	55	+ if source == "offsite":
	56	+ # ops offsite restore <project> <env>
	57	+ download_args = ["offsite", "restore", project, env]
	58	+ yield _sse_line({"line": f"Downloading {project}/{env} from offsite...", "timestamp": _now()})
58	59
59		- if not download_ok:
60		- yield _sse_line({"done": True, "success": False})
61		- return
	60	+ download_ok = True
	61	+ downloaded_path = None
	62	+ async for line in stream_ops_host(download_args, timeout=_BACKUP_TIMEOUT, op_id=op_id):
	63	+ yield _sse_line({"line": line, "timestamp": _now()})
	64	+ if line.startswith("[error]"):
	65	+ download_ok = False
	66	+ # Capture downloaded file path from offsite.py output
	67	+ if "Downloaded to" in line and "/tmp/" in line:
	68	+ # Parse "Downloaded to: /tmp/filename.tar.gz" or similar
	69	+ for part in line.split():
	70	+ if part.startswith("/tmp/") and part.endswith(".tar.gz"):
	71	+ downloaded_path = part
	72	+ elif line.startswith(" ✓ Downloaded to "):
	73	+ for part in line.split():
	74	+ if part.startswith("/tmp/") and part.endswith(".tar.gz"):
	75	+ downloaded_path = part
62	76
63		- yield _sse_line({"line": "Download complete. Starting restore...", "timestamp": _now()})
	77	+ if is_cancelled(op_id):
	78	+ yield _sse_line({"done": True, "success": False, "cancelled": True})
	79	+ return
64	80
65		- success = True
66		- async for line in stream_ops_host(base_args, timeout=_BACKUP_TIMEOUT):
67		- yield _sse_line({"line": line, "timestamp": _now()})
68		- if line.startswith("[error]"):
69		- success = False
	81	+ if not download_ok:
	82	+ yield _sse_line({"done": True, "success": False})
	83	+ return
70	84
71		- yield _sse_line({"done": True, "success": success})
	85	+ # Use the downloaded offsite file for restore
	86	+ if downloaded_path:
	87	+ base_args.append(downloaded_path)
	88	+ yield _sse_line({"line": f"Download complete. Restoring from {downloaded_path}...", "timestamp": _now()})
	89	+ else:
	90	+ yield _sse_line({"line": "Download complete. Starting restore...", "timestamp": _now()})
	91	+
	92	+ success = True
	93	+ async for item in _stream_with_keepalive(stream_ops_host(base_args, timeout=_BACKUP_TIMEOUT, op_id=op_id)):
	94	+ if item is None:
	95	+ # Keepalive ping — SSE comment to prevent idle timeout
	96	+ yield ": keepalive\n\n"
	97	+ else:
	98	+ yield _sse_line({"line": item, "timestamp": _now()})
	99	+ if item.startswith("[error]"):
	100	+ success = False
	101	+
	102	+ if is_cancelled(op_id):
	103	+ yield _sse_line({"done": True, "success": False, "cancelled": True})
	104	+ else:
	105	+ yield _sse_line({"done": True, "success": success})
	106	+ finally:
	107	+ clear_cancelled(op_id)
72	108
73	109
74	110	def _now() -> str:
75	111	return datetime.now(timezone.utc).isoformat()
76	112
77	113
	114	+async def _stream_with_keepalive(gen: AsyncGenerator[str, None]) -> AsyncGenerator[str \| None, None]:
	115	+ """Wrap an async generator to yield None as keepalive when no data arrives within the interval."""
	116	+ aiter = gen.__aiter__()
	117	+ pending = asyncio.ensure_future(aiter.__anext__())
	118	+ while True:
	119	+ done, _ = await asyncio.wait({pending}, timeout=_KEEPALIVE_INTERVAL)
	120	+ if done:
	121	+ try:
	122	+ yield pending.result()
	123	+ except StopAsyncIteration:
	124	+ break
	125	+ pending = asyncio.ensure_future(aiter.__anext__())
	126	+ else:
	127	+ yield None # keepalive — prevents Traefik idle timeout
	128	+
	129	+
78	130	@router.get("/{project}/{env}", summary="Restore a backup with real-time output")
79	131	async def restore_backup(
80	132	project: str,