feat: promote/sync/rebuild UI, operations page, bidirectional sync, lifecyc...

Matthias Nott

2026-02-22 7d94ec0d18b46893e23680cf8438109a34cc2a10

feat: promote/sync/rebuild UI, operations page, bidirectional sync, lifecycle ops

 app/main.py
..
..
@@ -1,12 +1,16 @@
1
+import hashlib
1
2
 import logging
2
3
 from contextlib import asynccontextmanager
3
4
 from pathlib import Path
4
5
 
5
-from fastapi import FastAPI
6
+from fastapi import FastAPI, Request
6
7
 from fastapi.middleware.cors import CORSMiddleware
8
+from fastapi.responses import HTMLResponse
7
9
 from fastapi.staticfiles import StaticFiles
10
+from starlette.datastructures import MutableHeaders
11
+from starlette.types import ASGIApp, Receive, Scope, Send
8
12
 
9
-from app.routers import backups, restore, services, status, system
13
+from app.routers import backups, promote, rebuild, registry, restore, services, status, sync_data, system
10
14
 
11
15
 logging.basicConfig(
12
16
     level=logging.INFO,
..
..
@@ -15,6 +19,13 @@
15
19
 logger = logging.getLogger(__name__)
16
20
 
17
21
 _STATIC_DIR = Path(__file__).parent.parent / "static"
22
+
23
+
24
+def _content_hash(filepath: Path) -> str:
25
+    """Return first 12 chars of SHA-256 hex digest of a file's contents."""
26
+    if not filepath.exists():
27
+        return "0"
28
+    return hashlib.sha256(filepath.read_bytes()).hexdigest()[:12]
18
29
 
19
30
 
20
31
 @asynccontextmanager
..
..
@@ -49,13 +60,72 @@
49
60
 app.include_router(restore.router, prefix="/api/restore", tags=["restore"])
50
61
 app.include_router(services.router, prefix="/api/services", tags=["services"])
51
62
 app.include_router(system.router, prefix="/api/system", tags=["system"])
63
+app.include_router(promote.router, prefix="/api/promote", tags=["promote"])
64
+app.include_router(sync_data.router, prefix="/api/sync", tags=["sync"])
65
+app.include_router(registry.router, prefix="/api/registry", tags=["registry"])
66
+app.include_router(rebuild.router, prefix="/api/rebuild", tags=["rebuild"])
52
67
 
53
68
 # ---------------------------------------------------------------------------
54
-# Static files – serve CSS/JS at /static and HTML at /
55
-# Mount /static first for explicit asset paths, then / for SPA fallback
69
+# Index route — serves index.html with content-hashed asset URLs.
70
+# The hash changes automatically when JS/CSS files change, so browsers
71
+# always fetch fresh assets after a deploy. No manual version bumping.
72
+# ---------------------------------------------------------------------------
73
+_INDEX_HTML = _STATIC_DIR / "index.html"
74
+
75
+
76
+@app.get("/", response_class=HTMLResponse)
77
+async def serve_index(request: Request):
78
+    """Serve index.html with auto-computed content hashes for cache busting."""
79
+    js_hash = _content_hash(_STATIC_DIR / "js" / "app.js")
80
+    css_hash = _content_hash(_STATIC_DIR / "css" / "style.css")
81
+    html = _INDEX_HTML.read_text()
82
+    # Replace any existing ?v=XX or ?h=XX cache busters with content hash
83
+    import re
84
+    html = re.sub(r'/static/js/app\.js\?[^"]*', f'/static/js/app.js?h={js_hash}', html)
85
+    html = re.sub(r'/static/css/style\.css\?[^"]*', f'/static/css/style.css?h={css_hash}', html)
86
+    return HTMLResponse(content=html, headers={"Cache-Control": "no-cache, must-revalidate"})
87
+
88
+
89
+# ---------------------------------------------------------------------------
90
+# Static files – JS/CSS at /static (cached aggressively — hash busts cache)
56
91
 # ---------------------------------------------------------------------------
57
92
 if _STATIC_DIR.exists():
58
93
     app.mount("/static", StaticFiles(directory=str(_STATIC_DIR)), name="static-assets")
59
-    app.mount("/", StaticFiles(directory=str(_STATIC_DIR), html=True), name="static")
60
94
 else:
61
95
     logger.warning("Static directory not found at %s – frontend will not be served", _STATIC_DIR)
96
+
97
+
98
+# ---------------------------------------------------------------------------
99
+# Cache-Control ASGI wrapper
100
+# - JS/CSS with ?h= hash: cached forever (immutable, 1 year)
101
+# - Everything else: standard headers
102
+# ---------------------------------------------------------------------------
103
+class CacheControlWrapper:
104
+    def __init__(self, inner: ASGIApp):
105
+        self.inner = inner
106
+
107
+    async def __call__(self, scope: Scope, receive: Receive, send: Send):
108
+        if scope["type"] != "http":
109
+            await self.inner(scope, receive, send)
110
+            return
111
+
112
+        path = scope.get("path", "")
113
+        qs = scope.get("query_string", b"").decode()
114
+        has_content_hash = "h=" in qs
115
+
116
+        async def send_with_cache_headers(message):
117
+            if message["type"] == "http.response.start":
118
+                headers = MutableHeaders(scope=message)
119
+                ct = headers.get("content-type", "")
120
+                if has_content_hash and any(t in ct for t in ("javascript", "text/css")):
121
+                    # Content-hashed assets: cache forever, hash changes on any edit
122
+                    headers["cache-control"] = "public, max-age=31536000, immutable"
123
+                elif any(t in ct for t in ("text/html", "javascript", "text/css")):
124
+                    headers["cache-control"] = "no-cache, must-revalidate"
125
+            await send(message)
126
+
127
+        await self.inner(scope, receive, send_with_cache_headers)
128
+
129
+
130
+# Wrap the finished FastAPI app — Uvicorn resolves app.main:app to this
131
+app = CacheControlWrapper(app)
 app/ops_runner.py
..
..
@@ -39,10 +39,11 @@
39
39
         data = json.loads(result["output"])
40
40
         return {"success": True, "data": data, "error": ""}
41
41
     except json.JSONDecodeError as exc:
42
+        raw = result["output"][:500]
42
43
         return {
43
44
             "success": False,
44
45
             "data": None,
45
-            "error": f"Failed to parse JSON: {exc}\nRaw: {result['output'][:500]}",
46
+            "error": f"Failed to parse JSON: {exc}\nRaw: {raw}",
46
47
         }
47
48
 
48
49
 
..
..
@@ -72,12 +73,40 @@
72
73
     return await _run_exec(_NSENTER_PREFIX + [OPS_CLI] + args, timeout=timeout)
73
74
 
74
75
 
76
+async def run_ops_host_json(args: list[str], timeout: int = _DEFAULT_TIMEOUT) -> dict:
77
+    """Run the ops CLI on the host via nsenter with --json and return parsed JSON."""
78
+    result = await run_ops_host(args + ["--json"], timeout=timeout)
79
+    if not result["success"]:
80
+        return {"success": False, "data": None, "error": result["error"] or result["output"]}
81
+    try:
82
+        data = json.loads(result["output"])
83
+        return {"success": True, "data": data, "error": ""}
84
+    except json.JSONDecodeError as exc:
85
+        raw = result["output"][:500]
86
+        return {
87
+            "success": False,
88
+            "data": None,
89
+            "error": f"Failed to parse JSON: {exc}\nRaw: {raw}",
90
+        }
91
+
92
+
75
93
 async def stream_ops_host(args: list[str], timeout: int = _DEFAULT_TIMEOUT) -> AsyncGenerator[str, None]:
76
94
     """Stream ops CLI output from the host via nsenter."""
77
95
     async for line in _stream_exec(_NSENTER_PREFIX + [OPS_CLI] + args, timeout=timeout):
78
96
         yield line
79
97
 
80
98
 
99
+async def run_command_host(args: list[str], timeout: int = _DEFAULT_TIMEOUT) -> dict:
100
+    """Run an arbitrary command on the host via nsenter."""
101
+    return await _run_exec(_NSENTER_PREFIX + args, timeout=timeout)
102
+
103
+
104
+async def stream_command_host(args: list[str], timeout: int = _DEFAULT_TIMEOUT) -> AsyncGenerator[str, None]:
105
+    """Stream arbitrary command output from the host via nsenter."""
106
+    async for line in _stream_exec(_NSENTER_PREFIX + args, timeout=timeout):
107
+        yield line
108
+
109
+
81
110
 # ---------------------------------------------------------------------------
82
111
 # Internal helpers
83
112
 # ---------------------------------------------------------------------------
 app/routers/backups.py
..
..
@@ -1,9 +1,9 @@
1
1
 from typing import Any
2
2
 
3
-from fastapi import APIRouter, Depends, HTTPException
3
+from fastapi import APIRouter, Depends, HTTPException, Query
4
4
 
5
5
 from app.auth import verify_token
6
-from app.ops_runner import run_ops, run_ops_json, run_ops_host, _BACKUP_TIMEOUT
6
+from app.ops_runner import run_ops, run_ops_json, run_ops_host, run_ops_host_json, run_command_host, _BACKUP_TIMEOUT
7
7
 
8
8
 router = APIRouter()
9
9
 
..
..
@@ -33,9 +33,22 @@
33
33
     _: str = Depends(verify_token),
34
34
 ) -> list[dict[str, Any]]:
35
35
     """Returns a list of offsite backup records."""
36
+    # Get project list from registry
37
+    import yaml
38
+    registry_path = "/opt/infrastructure/servers/hetzner-vps/registry.yaml"
39
+    try:
40
+        with open(registry_path) as f:
41
+            registry = yaml.safe_load(f)
42
+        projects = [
43
+            name for name, cfg in registry.get("projects", {}).items()
44
+            if cfg.get("backup_dir") and not cfg.get("infrastructure") and not cfg.get("static")
45
+        ]
46
+    except Exception:
47
+        projects = ["mdf", "seriousletter"]  # Fallback
48
+
36
49
     all_backups = []
37
-    for project in ["mdf", "seriousletter"]:
38
-        result = await run_ops_json(["offsite", "list", project])
50
+    for project in projects:
51
+        result = await run_ops_host_json(["offsite", "list", project])
39
52
         if result["success"] and isinstance(result["data"], list):
40
53
             for b in result["data"]:
41
54
                 b["project"] = project
..
..
@@ -99,3 +112,50 @@
99
112
             detail=f"Retention policy failed: {result['error'] or result['output']}",
100
113
         )
101
114
     return {"success": True, "output": result["output"]}
115
+
116
+
117
+@router.delete("/{project}/{env}/{name}", summary="Delete a backup")
118
+async def delete_backup(
119
+    project: str,
120
+    env: str,
121
+    name: str,
122
+    target: str = Query("local", regex="^(local|offsite|both)$"),
123
+    _: str = Depends(verify_token),
124
+) -> dict[str, Any]:
125
+    """
126
+    Delete a backup from local storage, offsite, or both.
127
+
128
+    Query param `target`: local | offsite | both (default: local).
129
+    """
130
+    if "/" in name or "\\" in name or ".." in name:
131
+        raise HTTPException(status_code=400, detail="Invalid backup name")
132
+
133
+    results = {"local": None, "offsite": None}
134
+
135
+    # Delete local
136
+    if target in ("local", "both"):
137
+        backup_path = f"/opt/data/backups/{project}/{env}/{name}"
138
+        check = await run_command_host(["test", "-f", backup_path])
139
+        if check["success"]:
140
+            result = await run_command_host(["rm", backup_path])
141
+            results["local"] = "ok" if result["success"] else "failed"
142
+        else:
143
+            results["local"] = "not_found"
144
+
145
+    # Delete offsite
146
+    if target in ("offsite", "both"):
147
+        result = await run_command_host([
148
+            "/opt/data/\u03c0/bin/python3", "-c",
149
+            f"import sys; sys.path.insert(0, '/opt/data/scripts'); "
150
+            f"from offsite import delete; "
151
+            f"ok = delete('{name}', '{project}', '{env}', quiet=True); "
152
+            f"sys.exit(0 if ok else 1)"
153
+        ])
154
+        results["offsite"] = "ok" if result["success"] else "failed"
155
+
156
+    # Check if anything succeeded
157
+    any_ok = "ok" in results.values()
158
+    if not any_ok:
159
+        raise HTTPException(status_code=500, detail=f"Delete failed: {results}")
160
+
161
+    return {"success": True, "project": project, "env": env, "name": name, "results": results}
 app/routers/promote.py
..
..
@@ -0,0 +1,72 @@
1
+import json
2
+from datetime import datetime, timezone
3
+from typing import AsyncGenerator, Literal
4
+
5
+from fastapi import APIRouter, Depends, HTTPException, Query
6
+from fastapi.responses import StreamingResponse
7
+
8
+from app.auth import verify_token
9
+from app.ops_runner import _BACKUP_TIMEOUT, stream_ops_host
10
+
11
+router = APIRouter()
12
+
13
+# Only adjacent-environment promote paths are allowed (code flows up)
14
+_VALID_PROMOTE_PAIRS = {("dev", "int"), ("int", "prod")}
15
+
16
+
17
+def _sse_line(payload: dict) -> str:
18
+    return f"data: {json.dumps(payload)}\n\n"
19
+
20
+
21
+def _now() -> str:
22
+    return datetime.now(timezone.utc).isoformat()
23
+
24
+
25
+async def _promote_generator(
26
+    project: str,
27
+    from_env: str,
28
+    to_env: str,
29
+    dry_run: bool,
30
+) -> AsyncGenerator[str, None]:
31
+    """Stream promote output via SSE."""
32
+    args = ["promote", project, from_env, to_env]
33
+    if dry_run:
34
+        args.append("--dry-run")
35
+    args.append("--yes")
36
+
37
+    label = f"Promoting {project}: {from_env} -> {to_env}"
38
+    if dry_run:
39
+        label += "  (dry run)"
40
+    yield _sse_line({"line": label, "timestamp": _now()})
41
+
42
+    success = True
43
+    async for line in stream_ops_host(args, timeout=_BACKUP_TIMEOUT):
44
+        yield _sse_line({"line": line, "timestamp": _now()})
45
+        if line.startswith("[error]") or "failed" in line.lower():
46
+            success = False
47
+
48
+    yield _sse_line({"done": True, "success": success})
49
+
50
+
51
+@router.get("/{project}/{from_env}/{to_env}", summary="Promote code with real-time output")
52
+async def promote_code(
53
+    project: str,
54
+    from_env: str,
55
+    to_env: str,
56
+    dry_run: bool = Query(default=False, alias="dry_run"),
57
+    _: str = Depends(verify_token),
58
+) -> StreamingResponse:
59
+    """Promote code forward (dev->int, int->prod) with SSE streaming."""
60
+    if (from_env, to_env) not in _VALID_PROMOTE_PAIRS:
61
+        raise HTTPException(
62
+            status_code=400,
63
+            detail=f"Invalid promote path '{from_env} -> {to_env}'. Only adjacent pairs are allowed: dev->int, int->prod.",
64
+        )
65
+    return StreamingResponse(
66
+        _promote_generator(project, from_env, to_env, dry_run),
67
+        media_type="text/event-stream",
68
+        headers={
69
+            "Cache-Control": "no-cache",
70
+            "X-Accel-Buffering": "no",
71
+        },
72
+    )
 app/routers/rebuild.py
..
..
@@ -0,0 +1,526 @@
1
+"""
2
+Container lifecycle operations via Coolify API + SSH.
3
+
4
+Three operations:
5
+  restart  – docker restart {containers} via SSH (no Coolify, no image pruning)
6
+  rebuild  – Coolify stop → docker build → Coolify start
7
+  recreate – Coolify stop → wipe data → docker build → Coolify start → show backups banner
8
+"""
9
+import json
10
+import os
11
+import urllib.request
12
+import urllib.error
13
+from datetime import datetime, timezone
14
+from typing import AsyncGenerator
15
+
16
+import yaml
17
+from fastapi import APIRouter, Depends, Query
18
+from fastapi.responses import StreamingResponse
19
+
20
+from app.auth import verify_token
21
+from app.ops_runner import (
22
+    _BACKUP_TIMEOUT,
23
+    run_command,
24
+    run_command_host,
25
+    stream_command_host,
26
+)
27
+
28
+router = APIRouter()
29
+
30
+# ---------------------------------------------------------------------------
31
+# Configuration
32
+# ---------------------------------------------------------------------------
33
+
34
+_REGISTRY_PATH = os.environ.get(
35
+    "REGISTRY_PATH",
36
+    "/opt/infrastructure/servers/hetzner-vps/registry.yaml",
37
+)
38
+
39
+_COOLIFY_BASE = os.environ.get(
40
+    "COOLIFY_BASE_URL",
41
+    "https://cockpit.tekmidian.com/api/v1",
42
+)
43
+
44
+_COOLIFY_TOKEN = os.environ.get(
45
+    "COOLIFY_API_TOKEN",
46
+    "3|f1fa8ee5791440ddd37e6cecafd964c8cd734dd4a8891180c424efad6bfdb7f5",
47
+)
48
+
49
+_COOLIFY_TIMEOUT = 30   # seconds for API calls
50
+_POLL_INTERVAL  = 5    # seconds between container status polls
51
+_POLL_MAX_WAIT  = 180  # max seconds to wait for containers to stop/start
52
+
53
+
54
+# ---------------------------------------------------------------------------
55
+# Registry helpers
56
+# ---------------------------------------------------------------------------
57
+
58
+def _load_registry() -> dict:
59
+    with open(_REGISTRY_PATH) as f:
60
+        return yaml.safe_load(f) or {}
61
+
62
+
63
+def _project_cfg(project: str) -> dict:
64
+    reg = _load_registry()
65
+    projects = reg.get("projects", {})
66
+    if project not in projects:
67
+        raise ValueError(f"Unknown project '{project}'")
68
+    return projects[project]
69
+
70
+
71
+def _coolify_uuid(project: str, env: str) -> str:
72
+    cfg = _project_cfg(project)
73
+    uuids = cfg.get("coolify_uuids", {})
74
+    uuid = uuids.get(env)
75
+    if not uuid:
76
+        raise ValueError(
77
+            f"No coolify_uuid configured for {project}/{env} in registry.yaml"
78
+        )
79
+    return uuid
80
+
81
+
82
+def _data_dir(project: str, env: str) -> str:
83
+    cfg = _project_cfg(project)
84
+    template = cfg.get("data_dir", "")
85
+    if not template:
86
+        raise ValueError(f"No data_dir configured for {project} in registry.yaml")
87
+    return template.replace("{env}", env)
88
+
89
+
90
+def _build_cfg(project: str, env: str) -> dict | None:
91
+    """Return build config or None if the project uses registry-only images."""
92
+    cfg = _project_cfg(project)
93
+    build = cfg.get("build", {})
94
+    if build.get("no_local_image"):
95
+        return None
96
+    ctx_template = build.get("build_context", "")
97
+    if not ctx_template:
98
+        return None
99
+    return {
100
+        "build_context": ctx_template.replace("{env}", env),
101
+        "image_name": build.get("image_name", project),
102
+        "env": env,
103
+    }
104
+
105
+
106
+# ---------------------------------------------------------------------------
107
+# SSE helpers
108
+# ---------------------------------------------------------------------------
109
+
110
+def _sse(payload: dict) -> str:
111
+    return f"data: {json.dumps(payload)}\n\n"
112
+
113
+
114
+def _now() -> str:
115
+    return datetime.now(timezone.utc).isoformat()
116
+
117
+
118
+def _line(text: str) -> str:
119
+    return _sse({"line": text, "timestamp": _now()})
120
+
121
+
122
+def _done(success: bool, project: str, env: str, action: str) -> str:
123
+    return _sse({
124
+        "done": True,
125
+        "success": success,
126
+        "project": project,
127
+        "env": env,
128
+        "action": action,
129
+    })
130
+
131
+
132
+# ---------------------------------------------------------------------------
133
+# Coolify API (synchronous — called from async context via run_in_executor)
134
+# ---------------------------------------------------------------------------
135
+
136
+def _coolify_request(method: str, path: str) -> dict:
137
+    """Make a Coolify API request. Returns parsed JSON body."""
138
+    url = f"{_COOLIFY_BASE}{path}"
139
+    req = urllib.request.Request(
140
+        url,
141
+        method=method,
142
+        headers={
143
+            "Authorization": f"Bearer {_COOLIFY_TOKEN}",
144
+            "Content-Type": "application/json",
145
+            "Accept": "application/json",
146
+        },
147
+    )
148
+    try:
149
+        with urllib.request.urlopen(req, timeout=_COOLIFY_TIMEOUT) as resp:
150
+            body = resp.read()
151
+            return json.loads(body) if body else {}
152
+    except urllib.error.HTTPError as exc:
153
+        body = exc.read()
154
+        raise RuntimeError(
155
+            f"Coolify API {method} {path} returned HTTP {exc.code}: {body.decode(errors='replace')[:500]}"
156
+        ) from exc
157
+    except Exception as exc:
158
+        raise RuntimeError(f"Coolify API call failed: {exc}") from exc
159
+
160
+
161
+async def _coolify_action(action: str, uuid: str) -> dict:
162
+    """Call a Coolify service action endpoint (stop/start/restart)."""
163
+    import asyncio
164
+    loop = asyncio.get_event_loop()
165
+    return await loop.run_in_executor(
166
+        None, _coolify_request, "POST", f"/services/{uuid}/{action}"
167
+    )
168
+
169
+
170
+# ---------------------------------------------------------------------------
171
+# Container polling helpers
172
+# ---------------------------------------------------------------------------
173
+
174
+async def _find_containers_for_service(project: str, env: str) -> list[str]:
175
+    """
176
+    Find all running Docker containers belonging to a project/env.
177
+    Uses the registry name_prefix and matches {env}-{prefix}-* pattern.
178
+    """
179
+    cfg = _project_cfg(project)
180
+    prefix = cfg.get("name_prefix", project)
181
+    name_pattern = f"{env}-{prefix}-"
182
+
183
+    result = await run_command(
184
+        ["docker", "ps", "--filter", f"name={name_pattern}", "--format", "{{.Names}}"],
185
+        timeout=15,
186
+    )
187
+    containers = []
188
+    if result["success"]:
189
+        for name in result["output"].strip().splitlines():
190
+            name = name.strip()
191
+            if name and name.startswith(name_pattern):
192
+                containers.append(name)
193
+    return containers
194
+
195
+
196
+async def _poll_until_stopped(
197
+    project: str,
198
+    env: str,
199
+    max_wait: int = _POLL_MAX_WAIT,
200
+) -> bool:
201
+    """Poll until no containers for project/env are running. Returns True if stopped."""
202
+    import asyncio
203
+    cfg = _project_cfg(project)
204
+    prefix = cfg.get("name_prefix", project)
205
+    name_pattern = f"{env}-{prefix}-"
206
+    waited = 0
207
+    while waited < max_wait:
208
+        result = await run_command(
209
+            ["docker", "ps", "--filter", f"name={name_pattern}", "--format", "{{.Names}}"],
210
+            timeout=15,
211
+        )
212
+        running = [
213
+            n.strip()
214
+            for n in result["output"].strip().splitlines()
215
+            if n.strip().startswith(name_pattern)
216
+        ] if result["success"] else []
217
+        if not running:
218
+            return True
219
+        await asyncio.sleep(_POLL_INTERVAL)
220
+        waited += _POLL_INTERVAL
221
+    return False
222
+
223
+
224
+async def _poll_until_running(
225
+    project: str,
226
+    env: str,
227
+    max_wait: int = _POLL_MAX_WAIT,
228
+) -> bool:
229
+    """Poll until at least one container for project/env is running. Returns True if up."""
230
+    import asyncio
231
+    cfg = _project_cfg(project)
232
+    prefix = cfg.get("name_prefix", project)
233
+    name_pattern = f"{env}-{prefix}-"
234
+    waited = 0
235
+    while waited < max_wait:
236
+        result = await run_command(
237
+            ["docker", "ps", "--filter", f"name={name_pattern}", "--format", "{{.Names}}"],
238
+            timeout=15,
239
+        )
240
+        running = [
241
+            n.strip()
242
+            for n in result["output"].strip().splitlines()
243
+            if n.strip().startswith(name_pattern)
244
+        ] if result["success"] else []
245
+        if running:
246
+            return True
247
+        await asyncio.sleep(_POLL_INTERVAL)
248
+        waited += _POLL_INTERVAL
249
+    return False
250
+
251
+
252
+# ---------------------------------------------------------------------------
253
+# Operation: Restart
254
+# ---------------------------------------------------------------------------
255
+
256
+async def _op_restart(project: str, env: str) -> AsyncGenerator[str, None]:
257
+    """
258
+    Restart: docker restart {containers} via SSH/nsenter.
259
+    No Coolify involvement — avoids the image-pruning stop/start cycle.
260
+    """
261
+    yield _line(f"[restart] Finding containers for {project}/{env}...")
262
+
263
+    try:
264
+        containers = await _find_containers_for_service(project, env)
265
+    except Exception as exc:
266
+        yield _line(f"[error] Registry lookup failed: {exc}")
267
+        yield _done(False, project, env, "restart")
268
+        return
269
+
270
+    if not containers:
271
+        yield _line(f"[error] No running containers found for {project}/{env}")
272
+        yield _done(False, project, env, "restart")
273
+        return
274
+
275
+    yield _line(f"[restart] Restarting {len(containers)} container(s): {', '.join(containers)}")
276
+
277
+    cmd = ["docker", "restart"] + containers
278
+    result = await run_command(cmd, timeout=120)
279
+
280
+    if result["output"].strip():
281
+        for line in result["output"].strip().splitlines():
282
+            yield _line(line)
283
+    if result["error"].strip():
284
+        for line in result["error"].strip().splitlines():
285
+            yield _line(f"[stderr] {line}")
286
+
287
+    if result["success"]:
288
+        yield _line(f"[restart] All containers restarted successfully.")
289
+        yield _done(True, project, env, "restart")
290
+    else:
291
+        yield _line(f"[error] docker restart failed (exit code non-zero)")
292
+        yield _done(False, project, env, "restart")
293
+
294
+
295
+# ---------------------------------------------------------------------------
296
+# Operation: Rebuild
297
+# ---------------------------------------------------------------------------
298
+
299
+async def _op_rebuild(project: str, env: str) -> AsyncGenerator[str, None]:
300
+    """
301
+    Rebuild: Coolify stop → docker build → Coolify start.
302
+    No data loss. For code/Dockerfile changes.
303
+    """
304
+    try:
305
+        uuid = _coolify_uuid(project, env)
306
+        build = _build_cfg(project, env)
307
+    except ValueError as exc:
308
+        yield _line(f"[error] Config error: {exc}")
309
+        yield _done(False, project, env, "rebuild")
310
+        return
311
+
312
+    # Step 1: Stop via Coolify
313
+    yield _line(f"[rebuild] Stopping {project}/{env} via Coolify (uuid={uuid})...")
314
+    try:
315
+        await _coolify_action("stop", uuid)
316
+        yield _line(f"[rebuild] Coolify stop queued. Waiting for containers to stop...")
317
+        # Step 2: Poll until stopped
318
+        stopped = await _poll_until_stopped(project, env)
319
+        if not stopped:
320
+            yield _line(f"[warn] Containers may still be running after {_POLL_MAX_WAIT}s — proceeding anyway")
321
+        else:
322
+            yield _line(f"[rebuild] All containers stopped.")
323
+    except RuntimeError as exc:
324
+        if "already stopped" in str(exc).lower():
325
+            yield _line(f"[rebuild] Service already stopped — continuing with build.")
326
+        else:
327
+            yield _line(f"[error] Coolify stop failed: {exc}")
328
+            yield _done(False, project, env, "rebuild")
329
+            return
330
+
331
+    # Step 3: Build image (if project uses local images)
332
+    if build:
333
+        ctx = build["build_context"]
334
+        image = f"{build['image_name']}:{env}"
335
+        yield _line(f"[rebuild] Building Docker image: {image}")
336
+        yield _line(f"[rebuild] Build context: {ctx}")
337
+
338
+        async for line in stream_command_host(
339
+            ["docker", "build", "-t", image, ctx],
340
+            timeout=_BACKUP_TIMEOUT,
341
+        ):
342
+            yield _line(line)
343
+    else:
344
+        yield _line(f"[rebuild] No local image build needed (registry images only).")
345
+
346
+    # Step 4: Start via Coolify
347
+    yield _line(f"[rebuild] Starting {project}/{env} via Coolify...")
348
+    try:
349
+        await _coolify_action("start", uuid)
350
+        yield _line(f"[rebuild] Coolify start queued. Waiting for containers...")
351
+    except RuntimeError as exc:
352
+        yield _line(f"[error] Coolify start failed: {exc}")
353
+        yield _done(False, project, env, "rebuild")
354
+        return
355
+
356
+    # Step 5: Poll until running
357
+    running = await _poll_until_running(project, env)
358
+    if running:
359
+        yield _line(f"[rebuild] Containers are up.")
360
+        yield _done(True, project, env, "rebuild")
361
+    else:
362
+        yield _line(f"[warn] Containers did not appear healthy within {_POLL_MAX_WAIT}s — check Coolify logs.")
363
+        yield _done(False, project, env, "rebuild")
364
+
365
+
366
+# ---------------------------------------------------------------------------
367
+# Operation: Recreate (Disaster Recovery)
368
+# ---------------------------------------------------------------------------
369
+
370
+async def _op_recreate(project: str, env: str) -> AsyncGenerator[str, None]:
371
+    """
372
+    Recreate: Coolify stop → wipe data → docker build → Coolify start.
373
+    DESTRUCTIVE — wipes all data volumes. Shows "Go to Backups" banner on success.
374
+    """
375
+    try:
376
+        uuid = _coolify_uuid(project, env)
377
+        build = _build_cfg(project, env)
378
+        data_dir = _data_dir(project, env)
379
+        cfg = _project_cfg(project)
380
+    except ValueError as exc:
381
+        yield _line(f"[error] Config error: {exc}")
382
+        yield _done(False, project, env, "recreate")
383
+        return
384
+
385
+    # Step 1: Stop via Coolify
386
+    yield _line(f"[recreate] Stopping {project}/{env} via Coolify (uuid={uuid})...")
387
+    try:
388
+        await _coolify_action("stop", uuid)
389
+        yield _line(f"[recreate] Coolify stop queued. Waiting for containers to stop...")
390
+        # Step 2: Poll until stopped
391
+        stopped = await _poll_until_stopped(project, env)
392
+        if not stopped:
393
+            yield _line(f"[warn] Containers may still be running after {_POLL_MAX_WAIT}s — proceeding anyway")
394
+        else:
395
+            yield _line(f"[recreate] All containers stopped.")
396
+    except RuntimeError as exc:
397
+        if "already stopped" in str(exc).lower():
398
+            yield _line(f"[recreate] Service already stopped — skipping stop step.")
399
+        else:
400
+            yield _line(f"[error] Coolify stop failed: {exc}")
401
+            yield _done(False, project, env, "recreate")
402
+            return
403
+
404
+    # Step 3: Wipe data volumes
405
+    yield _line(f"[recreate] WARNING: Wiping data directory: {data_dir}")
406
+    # Verify THIS env's containers are actually stopped before wiping
407
+    name_prefix = cfg.get("name_prefix", project)
408
+    # Use grep to AND-match both prefix and env (docker --filter uses OR for multiple name filters)
409
+    verify = await run_command_host(
410
+        ["sh", "-c", f"docker ps --format '{{{{.Names}}}}' | grep '^{env}-{name_prefix}\\|^{name_prefix}-{env}' || true"],
411
+        timeout=30,
412
+    )
413
+    running = verify["output"].strip()
414
+    if running:
415
+        yield _line(f"[error] Containers still running for {project}/{env}:")
416
+        for line in running.splitlines():
417
+            yield _line(f"  {line}")
418
+        yield _done(False, project, env, "recreate")
419
+        return
420
+
421
+    wipe_result = await run_command_host(
422
+        ["sh", "-c", f"rm -r {data_dir}/* 2>&1; echo EXIT_CODE=$?"],
423
+        timeout=120,
424
+    )
425
+    for line in (wipe_result["output"].strip() + "\n" + wipe_result["error"].strip()).strip().splitlines():
426
+        if line:
427
+            yield _line(line)
428
+    if "EXIT_CODE=0" in wipe_result["output"]:
429
+        yield _line(f"[recreate] Data directory wiped.")
430
+    else:
431
+        yield _line(f"[error] Wipe may have failed — check output above.")
432
+        yield _done(False, project, env, "recreate")
433
+        return
434
+
435
+    # Step 4: Build image (if project uses local images)
436
+    if build:
437
+        ctx = build["build_context"]
438
+        image = f"{build['image_name']}:{env}"
439
+        yield _line(f"[recreate] Building Docker image: {image}")
440
+        yield _line(f"[recreate] Build context: {ctx}")
441
+
442
+        async for line in stream_command_host(
443
+            ["docker", "build", "-t", image, ctx],
444
+            timeout=_BACKUP_TIMEOUT,
445
+        ):
446
+            yield _line(line)
447
+    else:
448
+        yield _line(f"[recreate] No local image build needed (registry images only).")
449
+
450
+    # Step 5: Start via Coolify
451
+    yield _line(f"[recreate] Starting {project}/{env} via Coolify...")
452
+    try:
453
+        await _coolify_action("start", uuid)
454
+        yield _line(f"[recreate] Coolify start queued. Waiting for containers...")
455
+    except RuntimeError as exc:
456
+        yield _line(f"[error] Coolify start failed: {exc}")
457
+        yield _done(False, project, env, "recreate")
458
+        return
459
+
460
+    # Step 6: Poll until running
461
+    running = await _poll_until_running(project, env)
462
+    if running:
463
+        yield _line(f"[recreate] Containers are up. Restore a backup to complete recovery.")
464
+        yield _done(True, project, env, "recreate")
465
+    else:
466
+        yield _line(f"[warn] Containers did not appear within {_POLL_MAX_WAIT}s — check Coolify logs.")
467
+        # Still return success=True so the "Go to Backups" banner appears
468
+        yield _done(True, project, env, "recreate")
469
+
470
+
471
+# ---------------------------------------------------------------------------
472
+# Dispatch wrapper
473
+# ---------------------------------------------------------------------------
474
+
475
+async def _op_generator(
476
+    project: str,
477
+    env: str,
478
+    action: str,
479
+) -> AsyncGenerator[str, None]:
480
+    """Route to the correct operation generator."""
481
+    if action == "restart":
482
+        async for chunk in _op_restart(project, env):
483
+            yield chunk
484
+    elif action == "rebuild":
485
+        async for chunk in _op_rebuild(project, env):
486
+            yield chunk
487
+    elif action == "recreate":
488
+        async for chunk in _op_recreate(project, env):
489
+            yield chunk
490
+    else:
491
+        yield _line(f"[error] Unknown action '{action}'. Valid: restart, rebuild, recreate")
492
+        yield _done(False, project, env, action)
493
+
494
+
495
+# ---------------------------------------------------------------------------
496
+# Endpoint
497
+# ---------------------------------------------------------------------------
498
+
499
+@router.get(
500
+    "/{project}/{env}",
501
+    summary="Container lifecycle operation with real-time SSE output",
502
+)
503
+async def lifecycle_op(
504
+    project: str,
505
+    env: str,
506
+    action: str = Query(
507
+        default="restart",
508
+        description="Operation: restart | rebuild | recreate",
509
+    ),
510
+    _: str = Depends(verify_token),
511
+) -> StreamingResponse:
512
+    """
513
+    Stream a container lifecycle operation via SSE.
514
+
515
+    - restart:  docker restart containers (safe, fast)
516
+    - rebuild:  stop via Coolify, rebuild image, start via Coolify
517
+    - recreate: stop, wipe data, rebuild image, start (destructive — DR only)
518
+    """
519
+    return StreamingResponse(
520
+        _op_generator(project, env, action),
521
+        media_type="text/event-stream",
522
+        headers={
523
+            "Cache-Control": "no-cache",
524
+            "X-Accel-Buffering": "no",
525
+        },
526
+    )
 app/routers/registry.py
..
..
@@ -0,0 +1,40 @@
1
+import yaml
2
+from pathlib import Path
3
+from typing import Any
4
+
5
+from fastapi import APIRouter, Depends
6
+
7
+from app.auth import verify_token
8
+
9
+router = APIRouter()
10
+
11
+_REGISTRY_PATH = Path("/opt/infrastructure/servers/hetzner-vps/registry.yaml")
12
+
13
+
14
+def _load_registry() -> dict:
15
+    """Load and return the registry YAML."""
16
+    with open(_REGISTRY_PATH) as f:
17
+        return yaml.safe_load(f)
18
+
19
+
20
+@router.get("/", summary="Get project registry")
21
+async def get_registry(
22
+    _: str = Depends(verify_token),
23
+) -> dict[str, Any]:
24
+    """Return project list with environments, promote config, and domains."""
25
+    registry = _load_registry()
26
+    projects = {}
27
+
28
+    for name, cfg in registry.get("projects", {}).items():
29
+        projects[name] = {
30
+            "environments": cfg.get("environments", []),
31
+            "domains": cfg.get("domains", {}),
32
+            "promote": cfg.get("promote"),
33
+            "has_cli": bool(cfg.get("cli")),
34
+            "static": cfg.get("static", False),
35
+            "infrastructure": cfg.get("infrastructure", False),
36
+            "backup_dir": cfg.get("backup_dir"),
37
+            "has_coolify": bool(cfg.get("coolify_uuids")),
38
+        }
39
+
40
+    return {"projects": projects}
 app/routers/restore.py
..
..
@@ -21,6 +21,8 @@
21
21
     env: str,
22
22
     source: str,
23
23
     dry_run: bool,
24
+    name: str | None = None,
25
+    mode: str = "full",
24
26
 ) -> AsyncGenerator[str, None]:
25
27
     """Async generator that drives the restore workflow and yields SSE events.
26
28
 
..
..
@@ -28,8 +30,20 @@
28
30
     that use host Python venvs incompatible with the container's Python.
29
31
     """
30
32
     base_args = ["restore", project, env]
33
+
34
+    # Pass the backup file path to avoid interactive selection prompt
35
+    if name:
36
+        backup_path = f"/opt/data/backups/{project}/{env}/{name}"
37
+        base_args.append(backup_path)
38
+
31
39
     if dry_run:
32
40
         base_args.append("--dry-run")
41
+
42
+    # Granular restore mode
43
+    if mode == "db":
44
+        base_args.append("--db-only")
45
+    elif mode == "wp":
46
+        base_args.append("--wp-only")
33
47
 
34
48
     if source == "offsite":
35
49
         # ops offsite restore <project> <env>
..
..
@@ -67,6 +81,8 @@
67
81
     env: str,
68
82
     source: Literal["local", "offsite"] = Query(default="local"),
69
83
     dry_run: bool = Query(default=False, alias="dry_run"),
84
+    name: str | None = Query(default=None),
85
+    mode: Literal["full", "db", "wp"] = Query(default="full"),
70
86
     _: str = Depends(verify_token),
71
87
 ) -> StreamingResponse:
72
88
     """
..
..
@@ -74,9 +90,11 @@
74
90
 
75
91
     Uses Server-Sent Events (SSE) to stream real-time progress.
76
92
     Runs on the host via nsenter for Python venv compatibility.
93
+
94
+    Modes: full (default), db (database only), wp (wp-content only).
77
95
     """
78
96
     return StreamingResponse(
79
-        _restore_generator(project, env, source, dry_run),
97
+        _restore_generator(project, env, source, dry_run, name, mode),
80
98
         media_type="text/event-stream",
81
99
         headers={
82
100
             "Cache-Control": "no-cache",
 app/routers/sync_data.py
..
..
@@ -0,0 +1,76 @@
1
+import json
2
+from datetime import datetime, timezone
3
+from typing import AsyncGenerator, Literal
4
+
5
+from fastapi import APIRouter, Depends, HTTPException, Query
6
+from fastapi.responses import StreamingResponse
7
+
8
+from app.auth import verify_token
9
+from app.ops_runner import _BACKUP_TIMEOUT, stream_ops_host
10
+
11
+router = APIRouter()
12
+
13
+# Only adjacent-environment sync paths are allowed (data flows down)
14
+_VALID_SYNC_PAIRS = {("prod", "int"), ("int", "dev")}
15
+
16
+
17
+def _sse_line(payload: dict) -> str:
18
+    return f"data: {json.dumps(payload)}\n\n"
19
+
20
+
21
+def _now() -> str:
22
+    return datetime.now(timezone.utc).isoformat()
23
+
24
+
25
+async def _sync_generator(
26
+    project: str,
27
+    from_env: str,
28
+    to_env: str,
29
+    db_only: bool,
30
+    uploads_only: bool,
31
+) -> AsyncGenerator[str, None]:
32
+    """Stream sync output via SSE."""
33
+    args = ["sync", project, "--from", from_env, "--to", to_env, "--yes"]
34
+    if db_only:
35
+        args.append("--db-only")
36
+    if uploads_only:
37
+        args.append("--uploads-only")
38
+
39
+    mode = "db-only" if db_only else ("uploads-only" if uploads_only else "full")
40
+    yield _sse_line({
41
+        "line": f"Syncing {project}: {from_env} -> {to_env} ({mode})...",
42
+        "timestamp": _now(),
43
+    })
44
+
45
+    success = True
46
+    async for line in stream_ops_host(args, timeout=_BACKUP_TIMEOUT):
47
+        yield _sse_line({"line": line, "timestamp": _now()})
48
+        if line.startswith("[error]") or "failed" in line.lower():
49
+            success = False
50
+
51
+    yield _sse_line({"done": True, "success": success})
52
+
53
+
54
+@router.get("/{project}", summary="Sync data with real-time output")
55
+async def sync_data(
56
+    project: str,
57
+    from_env: str = Query(default="prod", alias="from"),
58
+    to_env: str = Query(default="int", alias="to"),
59
+    db_only: bool = Query(default=False),
60
+    uploads_only: bool = Query(default=False),
61
+    _: str = Depends(verify_token),
62
+) -> StreamingResponse:
63
+    """Sync data backward (prod->int, int->dev) with SSE streaming."""
64
+    if (from_env, to_env) not in _VALID_SYNC_PAIRS:
65
+        raise HTTPException(
66
+            status_code=400,
67
+            detail=f"Invalid sync path '{from_env} -> {to_env}'. Only adjacent pairs are allowed: prod->int, int->dev.",
68
+        )
69
+    return StreamingResponse(
70
+        _sync_generator(project, from_env, to_env, db_only, uploads_only),
71
+        media_type="text/event-stream",
72
+        headers={
73
+            "Cache-Control": "no-cache",
74
+            "X-Accel-Buffering": "no",
75
+        },
76
+    )
 app/routers/system.py
..
..
@@ -6,7 +6,7 @@
6
6
 from fastapi import APIRouter, Depends, HTTPException
7
7
 
8
8
 from app.auth import verify_token
9
-from app.ops_runner import run_command, run_ops
9
+from app.ops_runner import run_command, run_command_host, run_ops, run_ops_host
10
10
 
11
11
 router = APIRouter()
12
12
 
..
..
@@ -54,31 +54,19 @@
54
54
 
55
55
 
56
56
 def _parse_timers_output(raw: str) -> list[dict[str, str]]:
57
-    """Parse `systemctl list-timers` output into timer dicts."""
57
+    """Parse `systemctl list-timers` by anchoring on timestamp and .timer patterns."""
58
58
     timers: list[dict[str, str]] = []
59
-    lines = raw.strip().splitlines()
60
-    if not lines:
61
-        return timers
62
-
63
-    header_idx = 0
64
-    for i, line in enumerate(lines):
65
-        if re.match(r"(?i)next\s+left", line):
66
-            header_idx = i
67
-            break
68
-
69
-    for line in lines[header_idx + 1:]:
70
-        line = line.strip()
71
-        if not line or line.startswith("timers listed") or line.startswith("To show"):
72
-            continue
73
-        parts = re.split(r"\s{2,}", line)
74
-        if len(parts) >= 5:
75
-            timers.append({
76
-                "next": parts[0], "left": parts[1], "last": parts[2],
77
-                "passed": parts[3], "unit": parts[4],
78
-                "activates": parts[5] if len(parts) > 5 else "",
79
-            })
80
-        elif parts:
81
-            timers.append({"unit": parts[0], "next": "", "left": "", "last": "", "passed": "", "activates": ""})
59
+    # Timestamp pattern: "Day YYYY-MM-DD HH:MM:SS TZ" or "-"
60
+    ts = r"(?:\w{3} \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} \w+|-)"
61
+    timer_re = re.compile(
62
+        rf"^(?P<next>{ts})\s+(?P<left>.+?)\s+"
63
+        rf"(?P<last>{ts})\s+(?P<passed>.+?)\s+"
64
+        r"(?P<unit>\S+\.timer)\s+(?P<activates>\S+)"
65
+    )
66
+    for line in raw.strip().splitlines():
67
+        m = timer_re.match(line)
68
+        if m:
69
+            timers.append({k: v.strip() for k, v in m.groupdict().items()})
82
70
     return timers
83
71
 
84
72
 
..
..
@@ -156,8 +144,8 @@
156
144
 async def health_check(
157
145
     _: str = Depends(verify_token),
158
146
 ) -> dict[str, Any]:
159
-    """Returns health check results via `ops health`."""
160
-    result = await run_ops(["health"])
147
+    """Returns health check results via `ops health` on the host."""
148
+    result = await run_ops_host(["health"])
161
149
     if not result["success"] and not result["output"].strip():
162
150
         raise HTTPException(status_code=500, detail=f"Failed to run health checks: {result['error']}")
163
151
     return {
..
..
@@ -170,8 +158,8 @@
170
158
 async def list_timers(
171
159
     _: str = Depends(verify_token),
172
160
 ) -> dict[str, Any]:
173
-    """Lists systemd timers."""
174
-    result = await run_command(["systemctl", "list-timers", "--no-pager"])
161
+    """Lists systemd timers via nsenter on the host."""
162
+    result = await run_command_host(["systemctl", "list-timers", "--no-pager"])
175
163
     if not result["success"] and not result["output"].strip():
176
164
         raise HTTPException(status_code=500, detail=f"Failed to list timers: {result['error']}")
177
165
     return {
..
..
@@ -185,13 +173,12 @@
185
173
     _: str = Depends(verify_token),
186
174
 ) -> dict[str, Any]:
187
175
     """
188
-    Returns system uptime, load average, CPU usage, memory, and swap.
176
+    Returns system uptime, CPU usage, memory, and swap.
189
177
 
190
178
     CPU usage is measured over a 0.5s window from /proc/stat.
191
179
     Memory/swap are read from /proc/meminfo.
192
180
     """
193
181
     uptime_str = ""
194
-    load_str = ""
195
182
 
196
183
     # Uptime
197
184
     try:
..
..
@@ -201,14 +188,6 @@
201
188
             hours = int((seconds_up % 86400) // 3600)
202
189
             minutes = int((seconds_up % 3600) // 60)
203
190
             uptime_str = f"{days}d {hours}h {minutes}m"
204
-    except Exception:
205
-        pass
206
-
207
-    # Load average
208
-    try:
209
-        with open("/proc/loadavg") as f:
210
-            parts = f.read().split()
211
-            load_str = f"{parts[0]}, {parts[1]}, {parts[2]}"
212
191
     except Exception:
213
192
         pass
214
193
 
..
..
@@ -233,21 +212,41 @@
233
212
     # Memory + Swap
234
213
     mem_info = _read_memory()
235
214
 
236
-    # Fallback for uptime/load if /proc wasn't available
237
-    if not uptime_str or not load_str:
215
+    # Container count
216
+    containers_str = ""
217
+    try:
218
+        result = await run_command(["docker", "ps", "--format", "{{.State}}"])
219
+        if result["success"]:
220
+            states = [s for s in result["output"].strip().splitlines() if s]
221
+            running = sum(1 for s in states if s == "running")
222
+            containers_str = f"{running}/{len(states)}"
223
+    except Exception:
224
+        pass
225
+
226
+    # Process count
227
+    processes = 0
228
+    try:
229
+        with open("/proc/loadavg") as f:
230
+            parts = f.read().split()
231
+            if len(parts) >= 4:
232
+                # /proc/loadavg field 4 is "running/total" processes
233
+                processes = int(parts[3].split("/")[1])
234
+    except Exception:
235
+        pass
236
+
237
+    # Fallback for uptime if /proc wasn't available
238
+    if not uptime_str:
238
239
         result = await run_command(["uptime"])
239
240
         if result["success"]:
240
241
             raw = result["output"].strip()
241
242
             up_match = re.search(r"up\s+(.+?),\s+\d+\s+user", raw)
242
243
             if up_match:
243
-                uptime_str = uptime_str or up_match.group(1).strip()
244
-            load_match = re.search(r"load average[s]?:\s*(.+)$", raw, re.IGNORECASE)
245
-            if load_match:
246
-                load_str = load_str or load_match.group(1).strip()
244
+                uptime_str = up_match.group(1).strip()
247
245
 
248
246
     return {
249
247
         "uptime": uptime_str or "unavailable",
250
-        "load": load_str or "unavailable",
251
248
         "cpu": cpu_info or None,
249
+        "containers": containers_str or "n/a",
250
+        "processes": processes or 0,
252
251
         **mem_info,
253
252
     }
 docker-compose.yml
..
..
@@ -8,6 +8,7 @@
8
8
       - /opt/infrastructure:/opt/infrastructure
9
9
       - /opt/data:/opt/data
10
10
       - /var/run/docker.sock:/var/run/docker.sock
11
+      - ./static:/app/static
11
12
     labels:
12
13
       - "traefik.enable=true"
13
14
       - "traefik.http.routers.ops-dashboard.rule=Host(`cockpit.tekmidian.com`)"
 static/css/style.css
..
..
@@ -401,3 +401,127 @@
401
401
   overflow: hidden;
402
402
 }
403
403
 .mono { font-family: 'SF Mono', 'Fira Code', 'Cascadia Code', monospace; }
404
+
405
+/* ---------- Backup Selection Bar ---------- */
406
+.selection-bar {
407
+  display: flex;
408
+  align-items: center;
409
+  gap: 1rem;
410
+  padding: 0.75rem 1.25rem;
411
+  margin-bottom: 0.75rem;
412
+  background: rgba(59,130,246,0.1);
413
+  border: 1px solid rgba(59,130,246,0.3);
414
+  border-radius: 0.5rem;
415
+  font-size: 0.875rem;
416
+  color: #93c5fd;
417
+  animation: fadeIn 0.15s ease-out;
418
+}
419
+.selection-bar span {
420
+  font-weight: 600;
421
+  margin-right: 0.5rem;
422
+}
423
+.selection-bar .btn {
424
+  margin-left: 0.5rem;
425
+}
426
+
427
+/* ---------- Backup Date Groups ---------- */
428
+.date-group {
429
+  margin-bottom: 0.75rem;
430
+}
431
+
432
+.date-group-header {
433
+  display: flex;
434
+  align-items: center;
435
+  gap: 0.75rem;
436
+  padding: 0.75rem 1rem;
437
+  background: #1f2937;
438
+  border: 1px solid #374151;
439
+  border-radius: 0.5rem;
440
+  cursor: pointer;
441
+  user-select: none;
442
+  transition: background 0.15s, border-color 0.15s;
443
+}
444
+.date-group-header:hover {
445
+  background: #263244;
446
+  border-color: #4b5563;
447
+}
448
+
449
+.date-group-header .chevron {
450
+  color: #6b7280;
451
+  font-size: 0.625rem;
452
+  transition: transform 0.2s ease;
453
+  flex-shrink: 0;
454
+  display: inline-block;
455
+}
456
+.date-group-header .chevron.open {
457
+  transform: rotate(90deg);
458
+  color: #60a5fa;
459
+}
460
+
461
+.date-group-title {
462
+  font-weight: 600;
463
+  color: #f3f4f6;
464
+  font-size: 0.9375rem;
465
+  flex: 1;
466
+}
467
+
468
+.date-group-meta {
469
+  font-size: 0.8125rem;
470
+  color: #9ca3af;
471
+  white-space: nowrap;
472
+}
473
+
474
+.date-group-size {
475
+  font-size: 0.8125rem;
476
+  color: #6b7280;
477
+  white-space: nowrap;
478
+}
479
+
480
+.date-group-body {
481
+  display: none;
482
+  margin-top: 0.25rem;
483
+  border-radius: 0 0 0.5rem 0.5rem;
484
+  overflow: hidden;
485
+}
486
+.date-group-body.open {
487
+  display: block;
488
+}
489
+
490
+/* ---------- Backup Location Badges ---------- */
491
+.badge-local {
492
+  background: rgba(16,185,129,0.12);
493
+  color: #34d399;
494
+  border: 1px solid rgba(52,211,153,0.25);
495
+}
496
+.badge-offsite {
497
+  background: rgba(139,92,246,0.12);
498
+  color: #a78bfa;
499
+  border: 1px solid rgba(167,139,250,0.25);
500
+}
501
+.badge-synced {
502
+  background: linear-gradient(90deg, rgba(16,185,129,0.15) 0%, rgba(139,92,246,0.15) 100%);
503
+  color: #a3e8d0;
504
+  border: 1px solid rgba(100,200,180,0.3);
505
+  text-transform: uppercase;
506
+  font-size: 0.7rem;
507
+  letter-spacing: 0.04em;
508
+}
509
+
510
+/* ---------- Restore modal info rows ---------- */
511
+.restore-info-row {
512
+  display: flex;
513
+  align-items: baseline;
514
+  gap: 0.75rem;
515
+  margin-bottom: 0.625rem;
516
+  font-size: 0.875rem;
517
+}
518
+.restore-info-label {
519
+  color: #9ca3af;
520
+  font-size: 0.8125rem;
521
+  font-weight: 500;
522
+  min-width: 5rem;
523
+}
524
+.restore-info-value {
525
+  color: #f3f4f6;
526
+  font-weight: 600;
527
+}
 static/index.html
..
..
@@ -5,7 +5,7 @@
5
5
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
6
6
   <title>OPS Dashboard</title>
7
7
   <script src="https://cdn.tailwindcss.com"></script>
8
-  <link rel="stylesheet" href="/static/css/style.css">
8
+  <link rel="stylesheet" href="/static/css/style.css?v=10">
9
9
   <style>
10
10
     body { background: #0f172a; color: #e2e8f0; margin: 0; }
11
11
     #app { display: flex; min-height: 100vh; }
..
..
@@ -79,13 +79,13 @@
79
79
         <svg width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"><path d="M21 15v4a2 2 0 01-2 2H5a2 2 0 01-2-2v-4"/><polyline points="7 10 12 15 17 10"/><line x1="12" y1="15" x2="12" y2="3"/></svg>
80
80
         Backups
81
81
       </a>
82
+      <a class="sidebar-link" data-page="operations" onclick="showPage('operations')">
83
+        <svg width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"><path d="M12 2L2 7l10 5 10-5-10-5z"/><path d="M2 17l10 5 10-5"/><path d="M2 12l10 5 10-5"/></svg>
84
+        Operations
85
+      </a>
82
86
       <a class="sidebar-link" data-page="system" onclick="showPage('system')">
83
87
         <svg width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"><circle cx="12" cy="12" r="3"/><path d="M19.4 15a1.65 1.65 0 00.33 1.82l.06.06a2 2 0 010 2.83 2 2 0 01-2.83 0l-.06-.06a1.65 1.65 0 00-1.82-.33 1.65 1.65 0 00-1 1.51V21a2 2 0 01-4 0v-.09A1.65 1.65 0 009 19.4a1.65 1.65 0 00-1.82.33l-.06.06a2 2 0 01-2.83-2.83l.06-.06A1.65 1.65 0 004.68 15a1.65 1.65 0 00-1.51-1H3a2 2 0 010-4h.09A1.65 1.65 0 004.6 9a1.65 1.65 0 00-.33-1.82l-.06-.06a2 2 0 012.83-2.83l.06.06A1.65 1.65 0 009 4.68a1.65 1.65 0 001-1.51V3a2 2 0 014 0v.09a1.65 1.65 0 001 1.51 1.65 1.65 0 001.82-.33l.06-.06a2 2 0 012.83 2.83l-.06.06A1.65 1.65 0 0019.4 9a1.65 1.65 0 001.51 1H21a2 2 0 010 4h-.09a1.65 1.65 0 00-1.51 1z"/></svg>
84
88
         System
85
-      </a>
86
-      <a class="sidebar-link" data-page="restore" onclick="showPage('restore')">
87
-        <svg width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"><polyline points="1 4 1 10 7 10"/><path d="M3.51 15a9 9 0 102.13-9.36L1 10"/></svg>
88
-        Restore
89
89
       </a>
90
90
     </nav>
91
91
     <div class="sidebar-footer" id="sidebar-footer">
..
..
@@ -139,6 +139,83 @@
139
139
   </div>
140
140
 </div>
141
141
 
142
-<script src="/static/js/app.js?v=4"></script>
142
+<!-- Restore Modal -->
143
+<div id="restore-modal" class="modal-overlay" style="display:none;" onclick="if(event.target===this)closeRestoreModal()">
144
+  <div class="modal-box" style="max-width:640px;">
145
+    <div class="modal-header">
146
+      <span style="font-weight:600;color:#f3f4f6;">Restore Backup</span>
147
+      <button onclick="closeRestoreModal()" style="background:none;border:none;color:#9ca3af;font-size:1.25rem;cursor:pointer;">&times;</button>
148
+    </div>
149
+    <div class="modal-body">
150
+      <!-- Info rows -->
151
+      <div class="restore-info-row">
152
+        <span class="restore-info-label">Target</span>
153
+        <span class="restore-info-value" id="restore-modal-project"></span>
154
+      </div>
155
+      <div class="restore-info-row" id="restore-source-row">
156
+        <span class="restore-info-label">Source</span>
157
+        <span class="restore-info-value" id="restore-modal-source"></span>
158
+      </div>
159
+      <div id="restore-source-selector" style="display:none;margin-bottom:0.625rem;">
160
+        <div style="display:flex;align-items:baseline;gap:0.75rem;font-size:0.875rem;">
161
+          <span class="restore-info-label">Source</span>
162
+          <label style="display:flex;align-items:center;gap:0.375rem;color:#d1d5db;cursor:pointer;">
163
+            <input type="radio" name="restore-source" value="local" checked style="accent-color:#3b82f6;"> Local
164
+          </label>
165
+          <label style="display:flex;align-items:center;gap:0.375rem;color:#d1d5db;cursor:pointer;">
166
+            <input type="radio" name="restore-source" value="offsite" style="accent-color:#3b82f6;"> Offsite
167
+          </label>
168
+        </div>
169
+      </div>
170
+      <div class="restore-info-row" style="margin-bottom:1rem;">
171
+        <span class="restore-info-label">Backup</span>
172
+        <span class="restore-info-value mono" id="restore-modal-name" style="font-size:0.8125rem;word-break:break-all;"></span>
173
+      </div>
174
+
175
+<!-- Restore mode selector -->      <div style="margin-bottom:1rem;">        <div style="font-size:0.8125rem;font-weight:500;color:#9ca3af;margin-bottom:0.5rem;">Restore Mode</div>        <div style="display:flex;gap:1rem;">          <label style="display:flex;align-items:center;gap:0.375rem;font-size:0.875rem;color:#d1d5db;cursor:pointer;">            <input type="radio" name="restore-mode" value="full" checked style="accent-color:#3b82f6;"> Full          </label>          <label style="display:flex;align-items:center;gap:0.375rem;font-size:0.875rem;color:#d1d5db;cursor:pointer;">            <input type="radio" name="restore-mode" value="db" style="accent-color:#3b82f6;"> Database only          </label>          <label style="display:flex;align-items:center;gap:0.375rem;font-size:0.875rem;color:#d1d5db;cursor:pointer;">            <input type="radio" name="restore-mode" value="wp" style="accent-color:#3b82f6;"> WP-Content only          </label>        </div>      </div>      <!-- Dry run checkbox -->      <label style="display:flex;align-items:center;gap:0.5rem;font-size:0.875rem;color:#d1d5db;cursor:pointer;margin-bottom:1rem;">        <input type="checkbox" id="restore-dry-run" checked style="width:1rem;height:1rem;accent-color:#3b82f6;">        Dry run (preview only — no changes made)      </label>      <!-- Warning -->
176
+      <div style="background:rgba(220,38,38,0.1);border:1px solid rgba(220,38,38,0.3);border-radius:0.5rem;padding:0.75rem 1rem;font-size:0.8125rem;color:#fca5a5;margin-bottom:1rem;">
177
+        Warning: a real restore will stop services, replace data, and restart containers.
178
+        Always run a dry run first.
179
+      </div>
180
+
181
+      <!-- SSE output (shown after start) -->
182
+      <div id="restore-modal-output" style="display:none;">
183
+        <div style="font-size:0.8125rem;font-weight:500;color:#9ca3af;margin-bottom:0.375rem;">Output</div>
184
+        <div id="restore-modal-terminal" class="terminal" style="max-height:300px;"></div>
185
+      </div>
186
+    </div>
187
+    <div class="modal-footer">
188
+      <button class="btn btn-ghost btn-sm" onclick="closeRestoreModal()">Cancel</button>
189
+      <button id="restore-start-btn" class="btn btn-danger btn-sm" onclick="startRestore()">Start Restore</button>
190
+    </div>
191
+  </div>
192
+</div>
193
+
194
+<!-- Operations Modal -->
195
+<div id="ops-modal" class="modal-overlay" style="display:none;" onclick="if(event.target===this)closeOpsModal()">
196
+  <div class="modal-box" style="max-width:700px;">
197
+    <div class="modal-header">
198
+      <span id="ops-modal-title" style="font-weight:600;color:#f3f4f6;">Operation</span>
199
+      <button onclick="closeOpsModal()" style="background:none;border:none;color:#9ca3af;font-size:1.25rem;cursor:pointer;">&times;</button>
200
+    </div>
201
+    <div class="modal-body">
202
+      <div id="ops-modal-info" style="margin-bottom:1rem;"></div>
203
+      <label id="ops-dry-run-row" style="display:flex;align-items:center;gap:0.5rem;font-size:0.875rem;color:#d1d5db;cursor:pointer;margin-bottom:1rem;">
204
+        <input type="checkbox" id="ops-dry-run" checked style="width:1rem;height:1rem;accent-color:#3b82f6;">
205
+        Dry run (preview only)
206
+      </label>
207
+      <div id="ops-modal-output" style="display:none;">
208
+        <div style="font-size:0.8125rem;font-weight:500;color:#9ca3af;margin-bottom:0.375rem;">Output</div>
209
+        <div id="ops-modal-terminal" class="terminal" style="max-height:350px;"></div>
210
+      </div>
211
+    </div>
212
+    <div class="modal-footer">
213
+      <button class="btn btn-ghost btn-sm" onclick="closeOpsModal()">Cancel</button>
214
+      <button id="ops-start-btn" class="btn btn-primary btn-sm" onclick="startOperation()">Start</button>
215
+    </div>
216
+  </div>
217
+</div>
218
+
219
+<script src="/static/js/app.js?v=12"></script>
143
220
 </body>
144
221
 </html>
 static/js/app.js
..
..
@@ -1,8 +1,8 @@
1
1
 'use strict';
2
-const APP_VERSION = 'v4-20260222';
2
+const APP_VERSION = 'v13-20260222';
3
3
 
4
4
 // ============================================================
5
-// OPS Dashboard — Vanilla JS Application (v4)
5
+// OPS Dashboard — Vanilla JS Application (v6)
6
6
 // ============================================================
7
7
 
8
8
 // ---------------------------------------------------------------------------
..
..
@@ -19,12 +19,25 @@
19
19
 let refreshTimer = null;
20
20
 const REFRESH_INTERVAL = 30000;
21
21
 
22
-// Backup filter state
23
-let backupFilterProject = null;  // null = all
24
-let backupFilterEnv = null;      // null = all
22
+// Backup drill-down state
23
+let backupDrillLevel = 0;      // 0=projects, 1=environments, 2=backup list
24
+let backupDrillProject = null;
25
+let backupDrillEnv = null;
26
+let cachedBackups = null;       // merged array, fetched once per page visit
25
27
 
26
28
 // Log modal state
27
29
 let logCtx = { project: null, env: null, service: null };
30
+
31
+// Restore modal state
32
+let restoreCtx = { project: null, env: null, source: null };
33
+let restoreEventSource = null;
34
+
35
+// Backup multi-select state
36
+let selectedBackups = new Set();
37
+// Operations state
38
+let opsEventSource = null;
39
+let opsCtx = { type: null, project: null, fromEnv: null, toEnv: null };
40
+let cachedRegistry = null;
28
41
 
29
42
 // ---------------------------------------------------------------------------
30
43
 // Helpers
..
..
@@ -109,7 +122,7 @@
109
122
       document.getElementById('login-overlay').style.display = 'none';
110
123
       document.getElementById('app').style.display = 'flex';
111
124
       const vEl = document.getElementById('app-version'); if (vEl && typeof APP_VERSION !== 'undefined') vEl.textContent = APP_VERSION;
112
-      showPage('dashboard');
125
+      navigateToHash();
113
126
       startAutoRefresh();
114
127
     })
115
128
     .catch(() => { err.textContent = 'Invalid token.'; err.style.display = 'block'; });
..
..
@@ -161,6 +174,8 @@
161
174
 function showPage(page) {
162
175
   currentPage = page;
163
176
   drillLevel = 0; drillProject = null; drillEnv = null;
177
+  backupDrillLevel = 0; backupDrillProject = null; backupDrillEnv = null;
178
+  cachedBackups = null;
164
179
   if (page !== 'dashboard') { viewMode = 'cards'; tableFilter = null; tableFilterLabel = ''; }
165
180
 
166
181
   document.querySelectorAll('#sidebar-nav .sidebar-link').forEach(el =>
..
..
@@ -169,6 +184,7 @@
169
184
   document.getElementById('mobile-overlay').classList.remove('open');
170
185
 
171
186
   renderPage();
187
+  pushHash();
172
188
 }
173
189
 
174
190
 function renderPage() {
..
..
@@ -180,13 +196,14 @@
180
196
     case 'dashboard': renderDashboard(); break;
181
197
     case 'backups':   renderBackups(); break;
182
198
     case 'system':    renderSystem(); break;
183
-    case 'restore':   renderRestore(); break;
199
+    case 'operations': renderOperations(); break;
184
200
     default:          renderDashboard();
185
201
   }
186
202
 }
187
203
 
188
204
 function refreshCurrentPage() {
189
205
   showSpin();
206
+  cachedBackups = null;
190
207
   fetchStatus().then(() => renderPage()).catch(e => toast('Refresh failed: ' + e.message, 'error')).finally(hideSpin);
191
208
 }
192
209
 
..
..
@@ -198,6 +215,7 @@
198
215
   if (mode === 'cards') { tableFilter = null; tableFilterLabel = ''; }
199
216
   updateViewToggle();
200
217
   renderDashboard();
218
+  pushHash();
201
219
 }
202
220
 
203
221
 function setTableFilter(filter, label) {
..
..
@@ -206,6 +224,7 @@
206
224
   viewMode = 'table';
207
225
   updateViewToggle();
208
226
   renderDashboard();
227
+  pushHash();
209
228
 }
210
229
 
211
230
 function clearFilter() {
..
..
@@ -261,9 +280,18 @@
261
280
     } else if (drillLevel === 2) {
262
281
       h = '<a onclick="drillBack(0)">Dashboard</a><span class="sep">/</span><a onclick="drillBack(1)">' + esc(drillProject) + '</a><span class="sep">/</span><span class="current">' + esc(drillEnv) + '</span>';
263
282
     }
264
-  } else {
265
-    const names = { backups: 'Backups', system: 'System', restore: 'Restore' };
266
-    h = '<span class="current">' + (names[currentPage] || currentPage) + '</span>';
283
+  } else if (currentPage === 'backups') {
284
+    if (backupDrillLevel === 0) {
285
+      h = '<span class="current">Backups</span>';
286
+    } else if (backupDrillLevel === 1) {
287
+      h = '<a onclick="backupDrillBack(0)">Backups</a><span class="sep">/</span><span class="current">' + esc(backupDrillProject) + '</span>';
288
+    } else if (backupDrillLevel === 2) {
289
+      h = '<a onclick="backupDrillBack(0)">Backups</a><span class="sep">/</span><a onclick="backupDrillBack(1)">' + esc(backupDrillProject) + '</a><span class="sep">/</span><span class="current">' + esc(backupDrillEnv) + '</span>';
290
+    }
291
+  } else if (currentPage === 'system') {
292
+    h = '<span class="current">System</span>';
293
+  } else if (currentPage === 'operations') {
294
+    h = '<span class="current">Operations</span>';
267
295
   }
268
296
   bc.innerHTML = h;
269
297
 }
..
..
@@ -272,6 +300,7 @@
272
300
   if (level === 0) { drillLevel = 0; drillProject = null; drillEnv = null; }
273
301
   else if (level === 1) { drillLevel = 1; drillEnv = null; }
274
302
   renderDashboard();
303
+  pushHash();
275
304
 }
276
305
 
277
306
 // ---------------------------------------------------------------------------
..
..
@@ -357,8 +386,8 @@
357
386
   c.innerHTML = h;
358
387
 }
359
388
 
360
-function drillToProject(name) { drillProject = name; drillLevel = 1; renderDashboard(); }
361
-function drillToEnv(name) { drillEnv = name; drillLevel = 2; renderDashboard(); }
389
+function drillToProject(name) { drillProject = name; drillLevel = 1; renderDashboard(); pushHash(); }
390
+function drillToEnv(name) { drillEnv = name; drillLevel = 2; renderDashboard(); pushHash(); }
362
391
 
363
392
 // ---------------------------------------------------------------------------
364
393
 // Dashboard — Table View
..
..
@@ -463,7 +492,7 @@
463
492
 }
464
493
 
465
494
 // ---------------------------------------------------------------------------
466
-// Backups
495
+// Backups — helpers
467
496
 // ---------------------------------------------------------------------------
468
497
 function fmtBackupDate(raw) {
469
498
   if (!raw) return '\u2014';
..
..
@@ -474,96 +503,510 @@
474
503
   return raw;
475
504
 }
476
505
 
506
+// Parse YYYYMMDD_HHMMSS -> { dateKey: 'YYYY-MM-DD', timeStr: 'HH:MM' }
507
+function parseBackupDate(raw) {
508
+  if (!raw) return { dateKey: '', timeStr: '' };
509
+  const m = String(raw).match(/^(\d{4})(\d{2})(\d{2})[_T](\d{2})(\d{2})/);
510
+  if (m) return { dateKey: `${m[1]}-${m[2]}-${m[3]}`, timeStr: `${m[4]}:${m[5]}` };
511
+  return { dateKey: raw, timeStr: '' };
512
+}
513
+
514
+// Format a YYYY-MM-DD key into a friendly group header label
515
+function fmtGroupHeader(dateKey) {
516
+  if (!dateKey) return 'Unknown Date';
517
+  const d = new Date(dateKey + 'T00:00:00');
518
+  const today = new Date(); today.setHours(0, 0, 0, 0);
519
+  const yesterday = new Date(today); yesterday.setDate(today.getDate() - 1);
520
+  const targetDay = new Date(dateKey + 'T00:00:00'); targetDay.setHours(0, 0, 0, 0);
521
+
522
+  const longFmt = d.toLocaleDateString('en-US', { weekday: 'long', year: 'numeric', month: 'short', day: 'numeric' });
523
+
524
+  if (targetDay.getTime() === today.getTime()) return 'Today \u2014 ' + longFmt;
525
+  if (targetDay.getTime() === yesterday.getTime()) return 'Yesterday \u2014 ' + longFmt;
526
+  return longFmt;
527
+}
528
+
529
+// Toggle a date group open/closed
530
+function toggleDateGroup(dateKey) {
531
+  const body = document.getElementById('dg-body-' + dateKey);
532
+  const chevron = document.getElementById('dg-chevron-' + dateKey);
533
+  if (!body) return;
534
+  const isOpen = body.classList.contains('open');
535
+  body.classList.toggle('open', !isOpen);
536
+  if (chevron) chevron.classList.toggle('open', !isOpen);
537
+}
538
+
539
+// ---------------------------------------------------------------------------
540
+// Backups — merge helper (dedup local+offsite by filename)
541
+// ---------------------------------------------------------------------------
542
+function mergeBackups(local, offsite) {
543
+  const byName = new Map();
544
+
545
+  for (const b of local) {
546
+    const name = b.name || b.file || '';
547
+    const key = name || (b.project + '/' + b.env + '/' + (b.date || b.timestamp));
548
+    byName.set(key, {
549
+      project: b.project || '',
550
+      env: b.env || b.environment || '',
551
+      name: name,
552
+      date: b.date || b.timestamp || '',
553
+      size_human: b.size_human || b.size || '',
554
+      size_bytes: Number(b.size || 0),
555
+      hasLocal: true,
556
+      hasOffsite: false,
557
+    });
558
+  }
559
+
560
+  for (const b of offsite) {
561
+    const name = b.name || '';
562
+    const key = name || (b.project + '/' + b.env + '/' + (b.date || ''));
563
+    if (byName.has(key)) {
564
+      byName.get(key).hasOffsite = true;
565
+    } else {
566
+      byName.set(key, {
567
+        project: b.project || '',
568
+        env: b.env || b.environment || '',
569
+        name: name,
570
+        date: b.date || '',
571
+        size_human: b.size || '',
572
+        size_bytes: Number(b.size_bytes || 0),
573
+        hasLocal: false,
574
+        hasOffsite: true,
575
+      });
576
+    }
577
+  }
578
+
579
+  return Array.from(byName.values());
580
+}
581
+
582
+// ---------------------------------------------------------------------------
583
+// Backups — main render (v7: drill-down)
584
+// ---------------------------------------------------------------------------
477
585
 async function renderBackups() {
478
586
   updateBreadcrumbs();
479
587
   const c = document.getElementById('page-content');
480
588
   try {
481
-    const [local, offsite] = await Promise.all([
482
-      api('/api/backups/'),
483
-      api('/api/backups/offsite').catch(() => []),
484
-    ]);
485
-
486
-    // Apply filters
487
-    const filteredLocal = local.filter(b => {
488
-      if (backupFilterProject && b.project !== backupFilterProject) return false;
489
-      if (backupFilterEnv && (b.env || b.environment || '') !== backupFilterEnv) return false;
490
-      return true;
491
-    });
492
-    const filteredOffsite = offsite.filter(b => {
493
-      if (backupFilterProject && b.project !== backupFilterProject) return false;
494
-      if (backupFilterEnv && (b.env || b.environment || '') !== backupFilterEnv) return false;
495
-      return true;
496
-    });
497
-
498
-    let h = '<div class="page-enter">';
499
-
500
-    // Quick backup buttons
501
-    h += '<div style="margin-bottom:1.5rem;">';
502
-    h += '<h2 style="font-size:1.125rem;font-weight:600;color:#f3f4f6;margin-bottom:0.75rem;">Create Backup</h2>';
503
-    h += '<div style="display:flex;flex-wrap:wrap;gap:0.5rem;">';
504
-    for (const p of ['mdf', 'seriousletter']) {
505
-      for (const e of ['dev', 'int', 'prod']) {
506
-        h += `<button class="btn btn-ghost btn-sm" onclick="createBackup('${p}','${e}')">${p}/${e}</button>`;
507
-      }
508
-    }
509
-    h += '</div></div>';
510
-
511
-    // Filter bar
512
-    const activeStyle = 'background:rgba(59,130,246,0.2);color:#60a5fa;';
513
-    h += '<div style="display:flex;flex-wrap:wrap;gap:0.5rem;align-items:center;margin-bottom:1.5rem;padding:0.75rem 1rem;background:#1f2937;border-radius:0.5rem;">';
514
-    h += '<span style="color:#9ca3af;font-size:0.875rem;margin-right:0.25rem;">Project:</span>';
515
-    h += `<button class="btn btn-ghost btn-xs" style="${backupFilterProject === null ? activeStyle : ''}" onclick="setBackupFilter('project',null)">All</button>`;
516
-    h += `<button class="btn btn-ghost btn-xs" style="${backupFilterProject === 'mdf' ? activeStyle : ''}" onclick="setBackupFilter('project','mdf')">mdf</button>`;
517
-    h += `<button class="btn btn-ghost btn-xs" style="${backupFilterProject === 'seriousletter' ? activeStyle : ''}" onclick="setBackupFilter('project','seriousletter')">seriousletter</button>`;
518
-    h += '<span style="color:#374151;margin:0 0.25rem;">|</span>';
519
-    h += '<span style="color:#9ca3af;font-size:0.875rem;margin-right:0.25rem;">Env:</span>';
520
-    h += `<button class="btn btn-ghost btn-xs" style="${backupFilterEnv === null ? activeStyle : ''}" onclick="setBackupFilter('env',null)">All</button>`;
521
-    h += `<button class="btn btn-ghost btn-xs" style="${backupFilterEnv === 'dev' ? activeStyle : ''}" onclick="setBackupFilter('env','dev')">dev</button>`;
522
-    h += `<button class="btn btn-ghost btn-xs" style="${backupFilterEnv === 'int' ? activeStyle : ''}" onclick="setBackupFilter('env','int')">int</button>`;
523
-    h += `<button class="btn btn-ghost btn-xs" style="${backupFilterEnv === 'prod' ? activeStyle : ''}" onclick="setBackupFilter('env','prod')">prod</button>`;
524
-    h += '</div>';
525
-
526
-    // Local
527
-    h += '<h2 style="font-size:1.125rem;font-weight:600;color:#f3f4f6;margin-bottom:0.75rem;">Local Backups</h2>';
528
-    if (filteredLocal.length === 0) {
529
-      h += '<div class="card" style="color:#6b7280;">No local backups match the current filter.</div>';
530
-    } else {
531
-      h += '<div class="table-wrapper"><table class="ops-table"><thead><tr><th>Project</th><th>Env</th><th>File</th><th>Date</th><th>Size</th></tr></thead><tbody>';
532
-      for (const b of filteredLocal) {
533
-        h += `<tr>
534
-          <td>${esc(b.project||'')}</td>
535
-          <td><span class="badge badge-blue">${esc(b.env||b.environment||'')}</span></td>
536
-          <td class="mono" style="font-size:0.8125rem;">${esc(b.name||b.file||'')}</td>
537
-          <td>${esc(fmtBackupDate(b.date||b.timestamp||''))}</td>
538
-          <td>${esc(b.size_human||b.size||'')}</td>
539
-        </tr>`;
540
-      }
541
-      h += '</tbody></table></div>';
589
+    if (!cachedBackups) {
590
+      const [local, offsite] = await Promise.all([
591
+        api('/api/backups/'),
592
+        api('/api/backups/offsite').catch(() => []),
593
+      ]);
594
+      cachedBackups = mergeBackups(local, offsite);
542
595
     }
543
596
 
544
-    // Offsite
545
-    h += '<h2 style="font-size:1.125rem;font-weight:600;color:#f3f4f6;margin:1.5rem 0 0.75rem;">Offsite Backups</h2>';
546
-    if (filteredOffsite.length === 0) {
547
-      h += '<div class="card" style="color:#6b7280;">No offsite backups match the current filter.</div>';
548
-    } else {
549
-      h += '<div class="table-wrapper"><table class="ops-table"><thead><tr><th>Project</th><th>Env</th><th>File</th><th>Date</th><th>Size</th></tr></thead><tbody>';
550
-      for (const b of filteredOffsite) {
551
-        h += `<tr>
552
-          <td>${esc(b.project||'')}</td>
553
-          <td><span class="badge badge-blue">${esc(b.env||b.environment||'')}</span></td>
554
-          <td class="mono" style="font-size:0.8125rem;">${esc(b.name||'')}</td>
555
-          <td>${esc(fmtBackupDate(b.date||''))}</td>
556
-          <td>${esc(b.size||'')}</td>
557
-        </tr>`;
558
-      }
559
-      h += '</tbody></table></div>';
560
-    }
561
-
562
-    h += '</div>';
563
-    c.innerHTML = h;
597
+    if (backupDrillLevel === 0) renderBackupProjects(c);
598
+    else if (backupDrillLevel === 1) renderBackupEnvironments(c);
599
+    else renderBackupList(c);
564
600
   } catch (e) {
565
601
     c.innerHTML = '<div class="card" style="color:#f87171;">Failed to load backups: ' + esc(e.message) + '</div>';
566
602
   }
603
+}
604
+
605
+// ---------------------------------------------------------------------------
606
+// Backups — Level 0: Project cards
607
+// ---------------------------------------------------------------------------
608
+function renderBackupProjects(c) {
609
+  const all = cachedBackups;
610
+  const localCount = all.filter(b => b.hasLocal).length;
611
+  const offsiteCount = all.filter(b => b.hasOffsite).length;
612
+  const syncedCount = all.filter(b => b.hasLocal && b.hasOffsite).length;
613
+  let latestTs = '';
614
+  for (const b of all) { if (b.date > latestTs) latestTs = b.date; }
615
+  const latestDisplay = latestTs ? fmtBackupDate(latestTs) : '\u2014';
616
+
617
+  let h = '<div class="page-enter">';
618
+
619
+  // Create Backup buttons
620
+  h += '<div style="margin-bottom:1.5rem;">';
621
+  h += '<h2 style="font-size:1.125rem;font-weight:600;color:#f3f4f6;margin-bottom:0.75rem;">Create Backup</h2>';
622
+  h += '<div style="display:flex;flex-wrap:wrap;gap:0.5rem;">';
623
+  for (const p of ['mdf', 'seriousletter']) {
624
+    for (const e of ['dev', 'int', 'prod']) {
625
+      h += `<button class="btn btn-ghost btn-sm" onclick="createBackup('${p}','${e}')">${p}/${e}</button>`;
626
+    }
627
+  }
628
+  h += '</div></div>';
629
+
630
+  // Global stat tiles
631
+  h += '<div class="grid-stats" style="margin-bottom:1.5rem;">';
632
+  h += statTile('Local', localCount, '#3b82f6');
633
+  h += statTile('Offsite', offsiteCount, '#8b5cf6');
634
+  h += statTile('Synced', syncedCount, '#10b981');
635
+  h += statTile('Latest', latestDisplay, '#f59e0b');
636
+  h += '</div>';
637
+
638
+  // Project cards
639
+  const projects = groupBy(all, 'project');
640
+  h += '<div class="grid-auto">';
641
+  for (const [name, backups] of Object.entries(projects)) {
642
+    const envs = [...new Set(backups.map(b => b.env))].sort();
643
+    let projLatest = '';
644
+    for (const b of backups) { if (b.date > projLatest) projLatest = b.date; }
645
+    const projSize = backups.reduce((acc, b) => acc + (b.size_bytes || 0), 0);
646
+
647
+    h += `<div class="card card-clickable" onclick="backupDrillToProject('${esc(name)}')">
648
+      <div style="display:flex;align-items:center;gap:0.5rem;margin-bottom:0.75rem;">
649
+        <span style="font-weight:600;font-size:1.0625rem;color:#f3f4f6;">${esc(name)}</span>
650
+        <span style="margin-left:auto;font-size:0.8125rem;color:#6b7280;">${backups.length} backup${backups.length !== 1 ? 's' : ''}</span>
651
+      </div>
652
+      <div style="display:flex;flex-wrap:wrap;gap:0.375rem;margin-bottom:0.5rem;">
653
+        ${envs.map(e => `<span class="badge badge-blue">${esc(e)}</span>`).join('')}
654
+      </div>
655
+      <div style="font-size:0.8125rem;color:#9ca3af;">
656
+        Latest: ${projLatest ? fmtBackupDate(projLatest) : '\u2014'}
657
+        ${projSize > 0 ? ' &middot; ' + fmtBytes(projSize) : ''}
658
+      </div>
659
+    </div>`;
660
+  }
661
+  h += '</div></div>';
662
+  c.innerHTML = h;
663
+}
664
+
665
+// ---------------------------------------------------------------------------
666
+// Backups — Level 1: Environment cards for a project
667
+// ---------------------------------------------------------------------------
668
+function renderBackupEnvironments(c) {
669
+  const projBackups = cachedBackups.filter(b => b.project === backupDrillProject);
670
+  const envGroups = groupBy(projBackups, 'env');
671
+  const envOrder = ['dev', 'int', 'prod'];
672
+  const sortedEnvs = Object.keys(envGroups).sort((a, b) => {
673
+    const ai = envOrder.indexOf(a), bi = envOrder.indexOf(b);
674
+    return (ai === -1 ? 99 : ai) - (bi === -1 ? 99 : bi);
675
+  });
676
+
677
+  let h = '<div class="page-enter"><div class="grid-auto">';
678
+  for (const envName of sortedEnvs) {
679
+    const backups = envGroups[envName];
680
+    const count = backups.length;
681
+    let envLatest = '';
682
+    for (const b of backups) { if (b.date > envLatest) envLatest = b.date; }
683
+    const envSize = backups.reduce((acc, b) => acc + (b.size_bytes || 0), 0);
684
+    const ep = esc(backupDrillProject), ee = esc(envName);
685
+
686
+    // Restore button logic
687
+    let restoreBtn = '';
688
+    if (count === 0) {
689
+      restoreBtn = `<button class="btn btn-danger btn-xs" disabled>Restore</button>`;
690
+    } else if (count === 1) {
691
+      const b = backups[0];
692
+      const src = b.hasLocal ? 'local' : 'offsite';
693
+      restoreBtn = `<button class="btn btn-danger btn-xs" onclick="event.stopPropagation();openRestoreModal('${ep}','${ee}','${src}','${esc(b.name)}')">Restore</button>`;
694
+    } else {
695
+      restoreBtn = `<button class="btn btn-danger btn-xs" onclick="event.stopPropagation();backupDrillToEnv('${ee}')">Restore (${count})</button>`;
696
+    }
697
+
698
+    h += `<div class="card card-clickable" onclick="backupDrillToEnv('${ee}')">
699
+      <div style="display:flex;align-items:center;gap:0.5rem;margin-bottom:0.75rem;">
700
+        <span style="font-weight:600;font-size:1.0625rem;color:#f3f4f6;">${ee.toUpperCase()}</span>
701
+        <span style="margin-left:auto;font-size:0.8125rem;color:#6b7280;">${count} backup${count !== 1 ? 's' : ''}</span>
702
+      </div>
703
+      <div style="font-size:0.8125rem;color:#9ca3af;margin-bottom:0.75rem;">
704
+        Latest: ${envLatest ? fmtBackupDate(envLatest) : '\u2014'}
705
+        ${envSize > 0 ? ' &middot; ' + fmtBytes(envSize) : ''}
706
+      </div>
707
+      <div style="display:flex;gap:0.5rem;">
708
+        <button class="btn btn-ghost btn-xs" onclick="event.stopPropagation();createBackup('${ep}','${ee}')">Create Backup</button>
709
+        ${restoreBtn}
710
+      </div>
711
+    </div>`;
712
+  }
713
+  h += '</div></div>';
714
+  c.innerHTML = h;
715
+}
716
+
717
+// ---------------------------------------------------------------------------
718
+// Backups — Level 2: Individual backups for project/env
719
+// ---------------------------------------------------------------------------
720
+function renderBackupList(c) {
721
+  const filtered = cachedBackups.filter(b => b.project === backupDrillProject && b.env === backupDrillEnv);
722
+
723
+  let h = '<div class="page-enter">';
724
+
725
+  // Selection action bar
726
+  h += `<div id="backup-selection-bar" class="selection-bar" style="display:${selectedBackups.size > 0 ? 'flex' : 'none'};">`;
727
+  h += `<span id="selection-count">${selectedBackups.size} selected</span>`;
728
+  h += `<button class="btn btn-danger btn-xs" onclick="deleteSelected()">Delete selected</button>`;
729
+  h += `<button class="btn btn-ghost btn-xs" onclick="clearSelection()">Clear</button>`;
730
+  h += `</div>`;
731
+
732
+  if (filtered.length === 0) {
733
+    h += '<div class="card" style="color:#6b7280;">No backups for ' + esc(backupDrillProject) + '/' + esc(backupDrillEnv) + '.</div>';
734
+  } else {
735
+    // Group by date key (YYYY-MM-DD), sort descending
736
+    const groups = {};
737
+    for (const b of filtered) {
738
+      const { dateKey, timeStr } = parseBackupDate(b.date);
739
+      b._dateKey = dateKey;
740
+      b._timeStr = timeStr;
741
+      if (!groups[dateKey]) groups[dateKey] = [];
742
+      groups[dateKey].push(b);
743
+    }
744
+
745
+    const sortedKeys = Object.keys(groups).sort().reverse();
746
+    const today = new Date(); today.setHours(0, 0, 0, 0);
747
+    const yesterday = new Date(today); yesterday.setDate(today.getDate() - 1);
748
+
749
+    for (const dateKey of sortedKeys) {
750
+      const items = groups[dateKey].sort((a, b) => b.date.localeCompare(a.date));
751
+      const groupSizeBytes = items.reduce((acc, b) => acc + (b.size_bytes || 0), 0);
752
+      const headerLabel = fmtGroupHeader(dateKey);
753
+      const safeKey = backupDrillProject + backupDrillEnv + dateKey.replace(/-/g, '');
754
+
755
+      const targetDay = new Date(dateKey + 'T00:00:00'); targetDay.setHours(0, 0, 0, 0);
756
+      const isRecent = targetDay.getTime() >= yesterday.getTime();
757
+
758
+      h += `<div class="date-group">`;
759
+      h += `<div class="date-group-header" onclick="toggleDateGroup('${safeKey}')">`;
760
+      h += `<span class="chevron${isRecent ? ' open' : ''}" id="dg-chevron-${safeKey}">&#9654;</span>`;
761
+      h += `<span class="date-group-title">${esc(headerLabel)}</span>`;
762
+      h += `<span class="date-group-meta">${items.length} backup${items.length !== 1 ? 's' : ''}</span>`;
763
+      if (groupSizeBytes > 0) {
764
+        h += `<span class="date-group-size">${fmtBytes(groupSizeBytes)}</span>`;
765
+      }
766
+      h += `</div>`;
767
+
768
+      h += `<div class="date-group-body${isRecent ? ' open' : ''}" id="dg-body-${safeKey}">`;
769
+      h += `<div class="table-wrapper"><table class="ops-table">`;
770
+      h += `<thead><tr><th style="width:2rem;padding-left:0.75rem;"><input type="checkbox" onclick="toggleSelectAll(this)" style="accent-color:#3b82f6;cursor:pointer;"></th><th>Location</th><th>Time</th><th>Size</th><th>Actions</th></tr></thead><tbody>`;
771
+      for (const b of items) {
772
+        let locationBadge;
773
+        if (b.hasLocal && b.hasOffsite) {
774
+          locationBadge = '<span class="badge badge-synced">local + offsite</span>';
775
+        } else if (b.hasLocal) {
776
+          locationBadge = '<span class="badge badge-local">local</span>';
777
+        } else {
778
+          locationBadge = '<span class="badge badge-offsite">offsite</span>';
779
+        }
780
+
781
+        const restoreSource = b.hasLocal ? 'local' : 'offsite';
782
+        const checked = selectedBackups.has(b.name) ? ' checked' : '';
783
+        const deleteBtn = `<button class="btn btn-ghost btn-xs" style="color:#f87171;border-color:#7f1d1d;" onclick="deleteBackup('${esc(b.project)}','${esc(b.env)}','${esc(b.name)}',${b.hasLocal},${b.hasOffsite})">Delete</button>`;
784
+        const uploadBtn = (b.hasLocal && !b.hasOffsite)
785
+          ? `<button class="btn btn-ghost btn-xs" style="color:#a78bfa;border-color:rgba(167,139,250,0.25);" onclick="uploadOffsiteBackup('${esc(b.project)}','${esc(b.env)}')">Upload</button>`
786
+          : '';
787
+        h += `<tr>
788
+          <td style="padding-left:0.75rem;"><input type="checkbox" class="backup-cb" value="${esc(b.name)}"${checked} onclick="toggleBackupSelect('${esc(b.name)}')" style="accent-color:#3b82f6;cursor:pointer;"></td>
789
+          <td>${locationBadge}</td>
790
+          <td class="mono">${esc(b._timeStr || '\u2014')}</td>
791
+          <td>${esc(b.size_human || '\u2014')}</td>
792
+          <td style="white-space:nowrap;">
793
+            <button class="btn btn-danger btn-xs" onclick="openRestoreModal('${esc(b.project)}','${esc(b.env)}','${restoreSource}','${esc(b.name)}',${b.hasLocal},${b.hasOffsite})">Restore</button>
794
+            ${uploadBtn}
795
+            ${deleteBtn}
796
+          </td>
797
+        </tr>`;
798
+      }
799
+      h += `</tbody></table></div>`;
800
+      h += `</div>`;
801
+      h += `</div>`;
802
+    }
803
+  }
804
+
805
+  h += '</div>';
806
+  c.innerHTML = h;
807
+}
808
+
809
+// ---------------------------------------------------------------------------
810
+// Backups — Drill-down navigation
811
+// ---------------------------------------------------------------------------
812
+function backupDrillToProject(name) { backupDrillProject = name; backupDrillLevel = 1; selectedBackups.clear(); renderBackups(); pushHash(); }
813
+function backupDrillToEnv(name) { backupDrillEnv = name; backupDrillLevel = 2; selectedBackups.clear(); renderBackups(); pushHash(); }
814
+function backupDrillBack(level) {
815
+  if (level === 0) { backupDrillLevel = 0; backupDrillProject = null; backupDrillEnv = null; }
816
+  else if (level === 1) { backupDrillLevel = 1; backupDrillEnv = null; }
817
+  selectedBackups.clear();
818
+  renderBackups();
819
+  pushHash();
820
+}
821
+
822
+// ---------------------------------------------------------------------------
823
+// Backup Multi-Select
824
+// ---------------------------------------------------------------------------
825
+function toggleBackupSelect(name) {
826
+  if (selectedBackups.has(name)) selectedBackups.delete(name);
827
+  else selectedBackups.add(name);
828
+  updateSelectionBar();
829
+}
830
+
831
+function toggleSelectAll(masterCb) {
832
+  const table = masterCb.closest('table');
833
+  const cbs = table.querySelectorAll('.backup-cb');
834
+  if (masterCb.checked) {
835
+    cbs.forEach(cb => { cb.checked = true; selectedBackups.add(cb.value); });
836
+  } else {
837
+    cbs.forEach(cb => { cb.checked = false; selectedBackups.delete(cb.value); });
838
+  }
839
+  updateSelectionBar();
840
+}
841
+
842
+function clearSelection() {
843
+  selectedBackups.clear();
844
+  document.querySelectorAll('.backup-cb').forEach(cb => { cb.checked = false; });
845
+  document.querySelectorAll('th input[type="checkbox"]').forEach(cb => { cb.checked = false; });
846
+  updateSelectionBar();
847
+}
848
+
849
+function updateSelectionBar() {
850
+  const bar = document.getElementById('backup-selection-bar');
851
+  const count = document.getElementById('selection-count');
852
+  if (bar) {
853
+    bar.style.display = selectedBackups.size > 0 ? 'flex' : 'none';
854
+    if (count) count.textContent = selectedBackups.size + ' selected';
855
+  }
856
+}
857
+
858
+async function deleteSelected() {
859
+  const names = [...selectedBackups];
860
+  if (names.length === 0) return;
861
+  // Check if any selected backups have both locations
862
+  const anyBoth = cachedBackups && cachedBackups.some(b => names.includes(b.name) && b.hasLocal && b.hasOffsite);
863
+  let target = 'local';
864
+  if (anyBoth) {
865
+    target = await showDeleteTargetDialog(names.length + ' selected backup(s)');
866
+    if (!target) return;
867
+  } else {
868
+    // Determine if all are offsite-only
869
+    const allOffsite = cachedBackups && names.every(n => { const b = cachedBackups.find(x => x.name === n); return b && !b.hasLocal && b.hasOffsite; });
870
+    if (allOffsite) target = 'offsite';
871
+  }
872
+  const label = target === 'both' ? 'local + offsite' : target;
873
+  if (!confirm(`Delete ${names.length} backup${names.length > 1 ? 's' : ''} (${label})?\n\nThis cannot be undone.`)) return;
874
+  toast(`Deleting ${names.length} backups (${label})...`, 'info');
875
+  let ok = 0, fail = 0;
876
+  for (const name of names) {
877
+    try {
878
+      await api(`/api/backups/${encodeURIComponent(backupDrillProject)}/${encodeURIComponent(backupDrillEnv)}/${encodeURIComponent(name)}?target=${target}`, { method: 'DELETE' });
879
+      ok++;
880
+    } catch (_) { fail++; }
881
+  }
882
+  selectedBackups.clear();
883
+  cachedBackups = null;
884
+  toast(`Deleted ${ok}${fail > 0 ? ', ' + fail + ' failed' : ''}`, fail > 0 ? 'warning' : 'success');
885
+  if (currentPage === 'backups') renderBackups();
886
+}
887
+
888
+async function uploadOffsiteBackup(project, env) {
889
+  if (!confirm(`Upload latest ${project}/${env} backup to offsite storage?`)) return;
890
+  toast('Uploading to offsite...', 'info');
891
+  try {
892
+    await api(`/api/backups/offsite/upload/${encodeURIComponent(project)}/${encodeURIComponent(env)}`, { method: 'POST' });
893
+    toast('Offsite upload complete for ' + project + '/' + env, 'success');
894
+    cachedBackups = null;
895
+    if (currentPage === 'backups') renderBackups();
896
+  } catch (e) { toast('Upload failed: ' + e.message, 'error'); }
897
+}
898
+
899
+// ---------------------------------------------------------------------------
900
+// Restore Modal
901
+// ---------------------------------------------------------------------------
902
+function openRestoreModal(project, env, source, name, hasLocal, hasOffsite) {
903
+  restoreCtx = { project, env, source, name, hasLocal: !!hasLocal, hasOffsite: !!hasOffsite };
904
+
905
+  // Close any running event source
906
+  if (restoreEventSource) { restoreEventSource.close(); restoreEventSource = null; }
907
+
908
+  // Populate modal info
909
+  document.getElementById('restore-modal-project').textContent = project + '/' + env;
910
+  document.getElementById('restore-modal-name').textContent = name || '(latest)';
911
+  document.getElementById('restore-dry-run').checked = false;
912
+
913
+  // Source selector: show radios when both local+offsite, static text otherwise
914
+  const sourceRow = document.getElementById('restore-source-row');
915
+  const sourceSelector = document.getElementById('restore-source-selector');
916
+  if (hasLocal && hasOffsite) {
917
+    sourceRow.style.display = 'none';
918
+    sourceSelector.style.display = 'block';
919
+    document.querySelectorAll('input[name="restore-source"]').forEach(r => {
920
+      r.checked = r.value === source;
921
+    });
922
+  } else {
923
+    sourceRow.style.display = 'flex';
924
+    sourceSelector.style.display = 'none';
925
+    document.getElementById('restore-modal-source').textContent = source;
926
+  }
927
+
928
+  // Reset mode to "full"
929
+  const modeRadios = document.querySelectorAll('input[name="restore-mode"]');
930
+  modeRadios.forEach(r => { r.checked = r.value === 'full'; });
931
+
932
+  // Reset terminal
933
+  const term = document.getElementById('restore-modal-terminal');
934
+  term.textContent = '';
935
+  document.getElementById('restore-modal-output').style.display = 'none';
936
+
937
+  // Enable start button
938
+  const startBtn = document.getElementById('restore-start-btn');
939
+  startBtn.disabled = false;
940
+  startBtn.textContent = 'Start Restore';
941
+
942
+  document.getElementById('restore-modal').style.display = 'flex';
943
+}
944
+
945
+function closeRestoreModal() {
946
+  if (restoreEventSource) { restoreEventSource.close(); restoreEventSource = null; }
947
+  document.getElementById('restore-modal').style.display = 'none';
948
+  restoreCtx = { project: null, env: null, source: null, name: null };
949
+}
950
+
951
+function startRestore() {
952
+  const { project, env, hasLocal, hasOffsite } = restoreCtx;
953
+  if (!project || !env) return;
954
+
955
+  // Determine source: from radio if both available, otherwise from context
956
+  let source = restoreCtx.source;
957
+  if (hasLocal && hasOffsite) {
958
+    const srcEl = document.querySelector('input[name="restore-source"]:checked');
959
+    if (srcEl) source = srcEl.value;
960
+  }
961
+
962
+  const dryRun = document.getElementById('restore-dry-run').checked;
963
+  const startBtn = document.getElementById('restore-start-btn');
964
+
965
+  // Show terminal
966
+  const outputDiv = document.getElementById('restore-modal-output');
967
+  const term = document.getElementById('restore-modal-terminal');
968
+  outputDiv.style.display = 'block';
969
+  term.textContent = 'Starting restore...\n';
970
+
971
+  startBtn.disabled = true;
972
+  startBtn.textContent = dryRun ? 'Running preview...' : 'Restoring...';
973
+
974
+  const name = restoreCtx.name || '';
975
+  const modeEl = document.querySelector('input[name="restore-mode"]:checked');
976
+  const mode = modeEl ? modeEl.value : 'full';
977
+  const url = `/api/restore/${encodeURIComponent(project)}/${encodeURIComponent(env)}?source=${encodeURIComponent(source)}${dryRun ? '&dry_run=true' : ''}&token=${encodeURIComponent(getToken())}${name ? '&name=' + encodeURIComponent(name) : ''}&mode=${encodeURIComponent(mode)}`;
978
+  const es = new EventSource(url);
979
+  restoreEventSource = es;
980
+
981
+  es.onmessage = function(e) {
982
+    try {
983
+      const d = JSON.parse(e.data);
984
+      if (d.done) {
985
+        es.close();
986
+        restoreEventSource = null;
987
+        const msg = d.success ? '\n--- Restore complete ---\n' : '\n--- Restore FAILED ---\n';
988
+        term.textContent += msg;
989
+        term.scrollTop = term.scrollHeight;
990
+        toast(d.success ? 'Restore completed' : 'Restore failed', d.success ? 'success' : 'error');
991
+        startBtn.disabled = false;
992
+        startBtn.textContent = 'Start Restore';
993
+        return;
994
+      }
995
+      if (d.line) {
996
+        term.textContent += d.line + '\n';
997
+        term.scrollTop = term.scrollHeight;
998
+      }
999
+    } catch (_) {}
1000
+  };
1001
+
1002
+  es.onerror = function() {
1003
+    es.close();
1004
+    restoreEventSource = null;
1005
+    term.textContent += '\n--- Connection lost ---\n';
1006
+    toast('Connection lost', 'error');
1007
+    startBtn.disabled = false;
1008
+    startBtn.textContent = 'Start Restore';
1009
+  };
567
1010
 }
568
1011
 
569
1012
 // ---------------------------------------------------------------------------
..
..
@@ -577,7 +1020,7 @@
577
1020
       api('/api/system/disk').catch(e => ({ filesystems: [], raw: e.message })),
578
1021
       api('/api/system/health').catch(e => ({ checks: [], raw: e.message })),
579
1022
       api('/api/system/timers').catch(e => ({ timers: [], raw: e.message })),
580
-      api('/api/system/info').catch(e => ({ uptime: 'error', load: 'error' })),
1023
+      api('/api/system/info').catch(e => ({ uptime: 'error' })),
581
1024
     ]);
582
1025
 
583
1026
     let h = '<div class="page-enter">';
..
..
@@ -614,7 +1057,8 @@
614
1057
     // Quick stats row
615
1058
     h += '<div class="grid-stats" style="margin-bottom:1.5rem;">';
616
1059
     h += statTile('Uptime', info.uptime || 'n/a', '#3b82f6');
617
-    h += statTile('Load', info.load || 'n/a', '#8b5cf6');
1060
+    h += statTile('Containers', info.containers || 'n/a', '#8b5cf6');
1061
+    h += statTile('Processes', info.processes || '0', '#f59e0b');
618
1062
     h += '</div>';
619
1063
 
620
1064
     // Disk usage — only real filesystems
..
..
@@ -676,50 +1120,412 @@
676
1120
 }
677
1121
 
678
1122
 // ---------------------------------------------------------------------------
679
-// Restore
1123
+// Operations Page
680
1124
 // ---------------------------------------------------------------------------
681
-function renderRestore() {
1125
+async function renderOperations() {
682
1126
   updateBreadcrumbs();
683
1127
   const c = document.getElementById('page-content');
684
-  let h = '<div class="page-enter">';
685
-  h += '<h2 style="font-size:1.125rem;font-weight:600;color:#f3f4f6;margin-bottom:0.75rem;">Restore Backup</h2>';
686
-  h += '<div class="card" style="max-width:480px;">';
687
-  h += '<div style="margin-bottom:1rem;"><label class="form-label">Project</label><select id="restore-project" class="form-select"><option value="mdf">mdf</option><option value="seriousletter">seriousletter</option></select></div>';
688
-  h += '<div style="margin-bottom:1rem;"><label class="form-label">Environment</label><select id="restore-env" class="form-select"><option value="dev">dev</option><option value="int">int</option><option value="prod">prod</option></select></div>';
689
-  h += '<div style="margin-bottom:1rem;"><label class="form-label">Source</label><select id="restore-source" class="form-select"><option value="local">Local</option><option value="offsite">Offsite</option></select></div>';
690
-  h += '<div style="margin-bottom:1rem;"><label style="display:flex;align-items:center;gap:0.5rem;font-size:0.875rem;color:#9ca3af;"><input type="checkbox" id="restore-dry" checked> Dry run (preview only)</label></div>';
691
-  h += '<button class="btn btn-danger" onclick="startRestore()">Start Restore</button>';
1128
+
1129
+  // Fetch registry if not cached
1130
+  if (!cachedRegistry) {
1131
+    try {
1132
+      cachedRegistry = await api('/api/registry/');
1133
+    } catch (e) {
1134
+      c.innerHTML = '<div class="card" style="color:#f87171;">Failed to load registry: ' + esc(e.message) + '</div>';
1135
+      return;
1136
+    }
1137
+  }
1138
+
1139
+  const projects = cachedRegistry.projects || {};
1140
+
1141
+  let h = '<div style="max-width:900px;">';
1142
+
1143
+  // Section: Promote Code (Forward)
1144
+  h += '<h2 style="font-size:1.125rem;font-weight:600;color:#f3f4f6;margin-bottom:0.75rem;">Promote Code</h2>';
1145
+  h += '<p style="font-size:0.8125rem;color:#9ca3af;margin-bottom:1rem;">Push code forward: dev &rarr; int &rarr; prod. Each project defines its own promotion type (git pull or rsync).</p>';
1146
+  h += '<div class="grid-auto" style="margin-bottom:2rem;">';
1147
+
1148
+  for (const [name, cfg] of Object.entries(projects)) {
1149
+    if (!cfg.promote || cfg.static || cfg.infrastructure) continue;
1150
+    const pType = cfg.promote.type || 'unknown';
1151
+    const envs = cfg.environments || [];
1152
+    const typeBadge = pType === 'git'
1153
+      ? '<span class="badge badge-blue" style="font-size:0.6875rem;">git</span>'
1154
+      : '<span class="badge badge-purple" style="font-size:0.6875rem;">rsync</span>';
1155
+
1156
+    h += '<div class="card">';
1157
+    h += '<div style="display:flex;justify-content:space-between;align-items:center;margin-bottom:0.75rem;">';
1158
+    h += '<span style="font-weight:600;color:#f3f4f6;">' + esc(name) + '</span>';
1159
+    h += typeBadge;
1160
+    h += '</div>';
1161
+
1162
+    const promotions = [];
1163
+    if (envs.includes('dev') && envs.includes('int')) promotions.push(['dev', 'int']);
1164
+    if (envs.includes('int') && envs.includes('prod')) promotions.push(['int', 'prod']);
1165
+
1166
+    if (promotions.length === 0) {
1167
+      h += '<div style="font-size:0.8125rem;color:#6b7280;">No promotion paths available</div>';
1168
+    } else {
1169
+      h += '<div style="display:flex;flex-direction:column;gap:0.5rem;">';
1170
+      for (const [from, to] of promotions) {
1171
+        h += '<button class="btn btn-ghost btn-sm" style="justify-content:flex-start;" onclick="openOpsModal(&apos;promote&apos;,&apos;' + esc(name) + '&apos;,&apos;' + esc(from) + '&apos;,&apos;' + esc(to) + '&apos;)">';
1172
+        h += '<span style="color:#60a5fa;">' + esc(from) + '</span>';
1173
+        h += ' <span style="color:#6b7280;">&rarr;</span> ';
1174
+        h += '<span style="color:#fbbf24;">' + esc(to) + '</span>';
1175
+        h += '</button>';
1176
+      }
1177
+      h += '</div>';
1178
+    }
1179
+    h += '</div>';
1180
+  }
692
1181
   h += '</div>';
693
-  h += '<div id="restore-output" style="display:none;margin-top:1rem;"><h3 style="font-size:1rem;font-weight:600;color:#f3f4f6;margin-bottom:0.5rem;">Output</h3><div id="restore-terminal" class="terminal" style="max-height:400px;"></div></div>';
1182
+
1183
+  // Section: Sync Data (Backward)
1184
+  h += '<h2 style="font-size:1.125rem;font-weight:600;color:#f3f4f6;margin-bottom:0.75rem;">Sync Data</h2>';
1185
+  h += '<p style="font-size:0.8125rem;color:#9ca3af;margin-bottom:1rem;">Sync content between environments. Choose the direction when syncing.</p>';
1186
+  h += '<div class="grid-auto" style="margin-bottom:2rem;">';
1187
+
1188
+  for (const [name, cfg] of Object.entries(projects)) {
1189
+    if (!cfg.has_cli || cfg.static || cfg.infrastructure) continue;
1190
+    const envs = cfg.environments || [];
1191
+
1192
+    h += '<div class="card">';
1193
+    h += '<div style="margin-bottom:0.75rem;font-weight:600;color:#f3f4f6;">' + esc(name) + '</div>';
1194
+
1195
+    const syncPairs = [];
1196
+    if (envs.includes('prod') && envs.includes('int')) syncPairs.push(['prod', 'int']);
1197
+    if (envs.includes('int') && envs.includes('dev')) syncPairs.push(['int', 'dev']);
1198
+
1199
+    if (syncPairs.length === 0) {
1200
+      h += '<div style="font-size:0.8125rem;color:#6b7280;">No sync paths available</div>';
1201
+    } else {
1202
+      h += '<div style="display:flex;flex-direction:column;gap:0.5rem;">';
1203
+      for (const [a, b] of syncPairs) {
1204
+        h += '<button class="btn btn-ghost btn-sm" style="justify-content:flex-start;" onclick="openSyncModal(&apos;' + esc(name) + '&apos;,&apos;' + esc(a) + '&apos;,&apos;' + esc(b) + '&apos;)">';
1205
+        h += '<span style="color:#60a5fa;">' + esc(a) + '</span>';
1206
+        h += ' <span style="color:#6b7280;">&harr;</span> ';
1207
+        h += '<span style="color:#fbbf24;">' + esc(b) + '</span>';
1208
+        h += '</button>';
1209
+      }
1210
+      h += '</div>';
1211
+    }
1212
+    h += '</div>';
1213
+  }
694
1214
   h += '</div>';
1215
+
1216
+  // Section: Container Lifecycle
1217
+  h += '<h2 style="font-size:1.125rem;font-weight:600;color:#f3f4f6;margin-bottom:0.375rem;">Container Lifecycle</h2>';
1218
+  h += '<p style="font-size:0.8125rem;color:#9ca3af;margin-bottom:1rem;">Manage container state via Coolify API. '
1219
+    + '<span style="color:#6ee7b7;">Restart</span> is safe. '
1220
+    + '<span style="color:#fbbf24;">Rebuild</span> refreshes the image. '
1221
+    + '<span style="color:#f87171;">Recreate</span> wipes data (disaster recovery only).</p>';
1222
+  h += '<div class="grid-auto" style="margin-bottom:2rem;">';
1223
+
1224
+  for (const [name, cfg] of Object.entries(projects)) {
1225
+    if (cfg.static || cfg.infrastructure || !cfg.has_coolify) continue;
1226
+    const envs = (cfg.environments || []).filter(e => e !== 'infra');
1227
+    if (!envs.length) continue;
1228
+
1229
+    h += '<div class="card">';
1230
+    h += '<div style="margin-bottom:0.75rem;font-weight:600;color:#f3f4f6;">' + esc(name) + '</div>';
1231
+    h += '<div style="display:flex;flex-direction:column;gap:0.625rem;">';
1232
+
1233
+    for (const env of envs) {
1234
+      h += '<div style="display:flex;align-items:center;gap:0.5rem;">';
1235
+      // Environment label
1236
+      h += '<span style="min-width:2.5rem;font-size:0.75rem;color:#9ca3af;font-weight:500;">' + esc(env) + '</span>';
1237
+      // Restart (green)
1238
+      h += '<button class="btn btn-ghost btn-xs" style="color:#6ee7b7;border-color:rgba(110,231,179,0.3);" '
1239
+        + 'onclick="openLifecycleModal(&apos;restart&apos;,&apos;' + esc(name) + '&apos;,&apos;' + esc(env) + '&apos;)">'
1240
+        + 'Restart</button>';
1241
+      // Rebuild (yellow)
1242
+      h += '<button class="btn btn-ghost btn-xs" style="color:#fbbf24;border-color:rgba(251,191,36,0.3);" '
1243
+        + 'onclick="openLifecycleModal(&apos;rebuild&apos;,&apos;' + esc(name) + '&apos;,&apos;' + esc(env) + '&apos;)">'
1244
+        + 'Rebuild</button>';
1245
+      // Recreate (red)
1246
+      h += '<button class="btn btn-ghost btn-xs" style="color:#f87171;border-color:rgba(248,113,113,0.3);" '
1247
+        + 'onclick="openLifecycleModal(&apos;recreate&apos;,&apos;' + esc(name) + '&apos;,&apos;' + esc(env) + '&apos;)">'
1248
+        + 'Recreate</button>';
1249
+      h += '</div>';
1250
+    }
1251
+
1252
+    h += '</div></div>';
1253
+  }
1254
+  h += '</div></div>';
1255
+
695
1256
   c.innerHTML = h;
696
1257
 }
697
1258
 
698
-async function startRestore() {
699
-  const project = document.getElementById('restore-project').value;
700
-  const env = document.getElementById('restore-env').value;
701
-  const source = document.getElementById('restore-source').value;
702
-  const dryRun = document.getElementById('restore-dry').checked;
703
-  if (!confirm(`Restore ${project}/${env} from ${source}${dryRun ? ' (dry run)' : ''}?`)) return;
1259
+// ---------------------------------------------------------------------------
1260
+// Operations Modal
1261
+// ---------------------------------------------------------------------------
1262
+function openSyncModal(project, envA, envB) {
1263
+  // Show direction picker in the ops modal
1264
+  opsCtx = { type: 'sync', project: project, fromEnv: envA, toEnv: envB };
704
1265
 
705
-  const out = document.getElementById('restore-output');
706
-  const term = document.getElementById('restore-terminal');
707
-  out.style.display = 'block';
708
-  term.textContent = 'Starting restore...\n';
1266
+  if (opsEventSource) { opsEventSource.close(); opsEventSource = null; }
709
1267
 
710
-  const url = `/api/restore/${project}/${env}?source=${source}&dry_run=${dryRun}&token=${encodeURIComponent(getToken())}`;
1268
+  const title = document.getElementById('ops-modal-title');
1269
+  const info = document.getElementById('ops-modal-info');
1270
+  const startBtn = document.getElementById('ops-start-btn');
1271
+
1272
+  title.textContent = 'Sync Data';
1273
+
1274
+  let ih = '<div class="restore-info-row"><span class="restore-info-label">Project</span><span class="restore-info-value">' + esc(project) + '</span></div>';
1275
+  ih += '<div style="margin-top:0.75rem;margin-bottom:0.25rem;font-size:0.8125rem;color:#9ca3af;">Direction</div>';
1276
+  ih += '<div style="display:flex;flex-direction:column;gap:0.5rem;">';
1277
+  ih += '<label style="display:flex;align-items:center;gap:0.5rem;cursor:pointer;padding:0.5rem 0.75rem;border-radius:0.5rem;border:1px solid #374151;'
1278
+    + 'background:rgba(96,165,250,0.1);" id="sync-dir-down">';
1279
+  ih += '<input type="radio" name="sync-dir" value="down" checked onchange="updateSyncDir()" style="accent-color:#60a5fa;">';
1280
+  ih += '<span style="color:#60a5fa;font-weight:600;">' + esc(envA) + '</span>';
1281
+  ih += '<span style="color:#6b7280;">&rarr;</span>';
1282
+  ih += '<span style="color:#fbbf24;font-weight:600;">' + esc(envB) + '</span>';
1283
+  ih += '<span style="font-size:0.75rem;color:#6b7280;margin-left:auto;">content flows down</span>';
1284
+  ih += '</label>';
1285
+  ih += '<label style="display:flex;align-items:center;gap:0.5rem;cursor:pointer;padding:0.5rem 0.75rem;border-radius:0.5rem;border:1px solid #374151;" id="sync-dir-up">';
1286
+  ih += '<input type="radio" name="sync-dir" value="up" onchange="updateSyncDir()" style="accent-color:#fbbf24;">';
1287
+  ih += '<span style="color:#fbbf24;font-weight:600;">' + esc(envB) + '</span>';
1288
+  ih += '<span style="color:#6b7280;">&rarr;</span>';
1289
+  ih += '<span style="color:#60a5fa;font-weight:600;">' + esc(envA) + '</span>';
1290
+  ih += '<span style="font-size:0.75rem;color:#6b7280;margin-left:auto;">content flows up</span>';
1291
+  ih += '</label>';
1292
+  ih += '</div>';
1293
+
1294
+  info.innerHTML = ih;
1295
+  startBtn.className = 'btn btn-primary btn-sm';
1296
+  startBtn.textContent = 'Sync';
1297
+
1298
+  document.getElementById('ops-dry-run').checked = true;
1299
+  document.getElementById('ops-modal-output').style.display = 'none';
1300
+  document.getElementById('ops-modal-terminal').textContent = '';
1301
+  startBtn.disabled = false;
1302
+  document.getElementById('ops-modal').style.display = 'flex';
1303
+}
1304
+
1305
+function updateSyncDir() {
1306
+  const dir = document.querySelector('input[name="sync-dir"]:checked').value;
1307
+  const downLabel = document.getElementById('sync-dir-down');
1308
+  const upLabel = document.getElementById('sync-dir-up');
1309
+  if (dir === 'down') {
1310
+    downLabel.style.background = 'rgba(96,165,250,0.1)';
1311
+    upLabel.style.background = 'transparent';
1312
+    // envA -> envB (default / downward)
1313
+    opsCtx.fromEnv = downLabel.querySelector('span[style*="color:#60a5fa"]').textContent;
1314
+    opsCtx.toEnv = downLabel.querySelector('span[style*="color:#fbbf24"]').textContent;
1315
+  } else {
1316
+    downLabel.style.background = 'transparent';
1317
+    upLabel.style.background = 'rgba(251,191,36,0.1)';
1318
+    // envB -> envA (upward)
1319
+    opsCtx.fromEnv = upLabel.querySelector('span[style*="color:#fbbf24"]').textContent;
1320
+    opsCtx.toEnv = upLabel.querySelector('span[style*="color:#60a5fa"]').textContent;
1321
+  }
1322
+}
1323
+
1324
+function openOpsModal(type, project, fromEnv, toEnv) {
1325
+  opsCtx = { type, project, fromEnv, toEnv };
1326
+
1327
+  if (opsEventSource) { opsEventSource.close(); opsEventSource = null; }
1328
+
1329
+  const title = document.getElementById('ops-modal-title');
1330
+  const info = document.getElementById('ops-modal-info');
1331
+  const startBtn = document.getElementById('ops-start-btn');
1332
+
1333
+  if (type === 'promote') {
1334
+    title.textContent = 'Promote Code';
1335
+    info.innerHTML = '<div class="restore-info-row"><span class="restore-info-label">Project</span><span class="restore-info-value">' + esc(project) + '</span></div>'
1336
+      + '<div class="restore-info-row"><span class="restore-info-label">Direction</span><span class="restore-info-value">' + esc(fromEnv) + ' &rarr; ' + esc(toEnv) + '</span></div>';
1337
+    startBtn.className = 'btn btn-primary btn-sm';
1338
+    startBtn.textContent = 'Promote';
1339
+  } else if (type === 'sync') {
1340
+    title.textContent = 'Sync Data';
1341
+    info.innerHTML = '<div class="restore-info-row"><span class="restore-info-label">Project</span><span class="restore-info-value">' + esc(project) + '</span></div>'
1342
+      + '<div class="restore-info-row"><span class="restore-info-label">Direction</span><span class="restore-info-value">' + esc(fromEnv) + ' &rarr; ' + esc(toEnv) + '</span></div>';
1343
+    startBtn.className = 'btn btn-primary btn-sm';
1344
+    startBtn.textContent = 'Sync';
1345
+  }
1346
+
1347
+  document.getElementById('ops-dry-run').checked = true;
1348
+  document.getElementById('ops-modal-output').style.display = 'none';
1349
+  document.getElementById('ops-modal-terminal').textContent = '';
1350
+  startBtn.disabled = false;
1351
+
1352
+  document.getElementById('ops-modal').style.display = 'flex';
1353
+}
1354
+
1355
+// ---------------------------------------------------------------------------
1356
+// Lifecycle Modal (Restart / Rebuild / Recreate)
1357
+// ---------------------------------------------------------------------------
1358
+function openLifecycleModal(action, project, env) {
1359
+  opsCtx = { type: action, project, fromEnv: env, toEnv: null };
1360
+
1361
+  if (opsEventSource) { opsEventSource.close(); opsEventSource = null; }
1362
+
1363
+  const title = document.getElementById('ops-modal-title');
1364
+  const info = document.getElementById('ops-modal-info');
1365
+  const startBtn = document.getElementById('ops-start-btn');
1366
+  const dryRunRow = document.getElementById('ops-dry-run-row');
1367
+
1368
+  // Hide the dry-run checkbox — lifecycle ops don't use it
1369
+  if (dryRunRow) dryRunRow.style.display = 'none';
1370
+
1371
+  if (action === 'restart') {
1372
+    title.textContent = 'Restart Containers';
1373
+    info.innerHTML = ''
1374
+      + '<div class="restore-info-row"><span class="restore-info-label">Project</span><span class="restore-info-value">' + esc(project) + '</span></div>'
1375
+      + '<div class="restore-info-row"><span class="restore-info-label">Environment</span><span class="restore-info-value">' + esc(env) + '</span></div>'
1376
+      + '<div style="background:rgba(16,185,129,0.08);border:1px solid rgba(16,185,129,0.25);border-radius:0.5rem;padding:0.625rem 0.875rem;font-size:0.8125rem;color:#6ee7b7;margin-top:0.75rem;">'
1377
+      + 'Safe operation. Runs <code>docker restart</code> on each container. No image changes, no data loss.</div>';
1378
+    startBtn.className = 'btn btn-sm';
1379
+    startBtn.style.cssText = 'background:#065f46;color:#6ee7b7;border:1px solid rgba(110,231,179,0.3);';
1380
+    startBtn.textContent = 'Restart';
1381
+
1382
+  } else if (action === 'rebuild') {
1383
+    title.textContent = 'Rebuild Environment';
1384
+    info.innerHTML = ''
1385
+      + '<div class="restore-info-row"><span class="restore-info-label">Project</span><span class="restore-info-value">' + esc(project) + '</span></div>'
1386
+      + '<div class="restore-info-row"><span class="restore-info-label">Environment</span><span class="restore-info-value">' + esc(env) + '</span></div>'
1387
+      + '<div style="background:rgba(251,191,36,0.08);border:1px solid rgba(251,191,36,0.25);border-radius:0.5rem;padding:0.625rem 0.875rem;font-size:0.8125rem;color:#fde68a;margin-top:0.75rem;">'
1388
+      + 'Stops containers via Coolify, rebuilds the Docker image, then starts again. No data loss.</div>';
1389
+    startBtn.className = 'btn btn-sm';
1390
+    startBtn.style.cssText = 'background:#78350f;color:#fde68a;border:1px solid rgba(251,191,36,0.3);';
1391
+    startBtn.textContent = 'Rebuild';
1392
+
1393
+  } else if (action === 'recreate') {
1394
+    title.textContent = 'Recreate Environment';
1395
+    info.innerHTML = ''
1396
+      + '<div class="restore-info-row"><span class="restore-info-label">Project</span><span class="restore-info-value">' + esc(project) + '</span></div>'
1397
+      + '<div class="restore-info-row"><span class="restore-info-label">Environment</span><span class="restore-info-value">' + esc(env) + '</span></div>'
1398
+      + '<div style="background:rgba(220,38,38,0.1);border:1px solid rgba(220,38,38,0.3);border-radius:0.5rem;padding:0.75rem 1rem;font-size:0.8125rem;color:#fca5a5;margin-top:0.75rem;">'
1399
+      + '<strong style="display:block;margin-bottom:0.375rem;">DESTRUCTIVE — Disaster Recovery Only</strong>'
1400
+      + 'Stops containers, wipes all data volumes, rebuilds image, starts fresh. '
1401
+      + 'You must restore a backup afterwards.</div>'
1402
+      + '<div style="margin-top:0.875rem;">'
1403
+      + '<label style="font-size:0.8125rem;color:#9ca3af;display:block;margin-bottom:0.375rem;">Type the environment name to confirm:</label>'
1404
+      + '<input id="recreate-confirm-input" type="text" placeholder="' + esc(env) + '" '
1405
+      + 'style="width:100%;box-sizing:border-box;padding:0.5rem 0.75rem;background:#1f2937;border:1px solid rgba(220,38,38,0.4);border-radius:0.375rem;color:#f3f4f6;font-size:0.875rem;" '
1406
+      + 'oninput="checkRecreateConfirm(\'' + esc(env) + '\')">'
1407
+      + '</div>';
1408
+    startBtn.className = 'btn btn-danger btn-sm';
1409
+    startBtn.style.cssText = '';
1410
+    startBtn.textContent = 'Recreate';
1411
+    startBtn.disabled = true;  // enabled after typing env name
1412
+  }
1413
+
1414
+  document.getElementById('ops-modal-output').style.display = 'none';
1415
+  document.getElementById('ops-modal-terminal').textContent = '';
1416
+
1417
+  document.getElementById('ops-modal').style.display = 'flex';
1418
+  if (action === 'recreate') {
1419
+    setTimeout(() => {
1420
+      const inp = document.getElementById('recreate-confirm-input');
1421
+      if (inp) inp.focus();
1422
+    }, 100);
1423
+  }
1424
+}
1425
+
1426
+function checkRecreateConfirm(expectedEnv) {
1427
+  const inp = document.getElementById('recreate-confirm-input');
1428
+  const startBtn = document.getElementById('ops-start-btn');
1429
+  if (!inp || !startBtn) return;
1430
+  startBtn.disabled = inp.value.trim() !== expectedEnv;
1431
+}
1432
+
1433
+function closeOpsModal() {
1434
+  if (opsEventSource) { opsEventSource.close(); opsEventSource = null; }
1435
+  document.getElementById('ops-modal').style.display = 'none';
1436
+  opsCtx = { type: null, project: null, fromEnv: null, toEnv: null };
1437
+  // Restore dry-run row visibility for promote/sync operations
1438
+  const dryRunRow = document.getElementById('ops-dry-run-row');
1439
+  if (dryRunRow) dryRunRow.style.display = '';
1440
+  // Reset start button style
1441
+  const startBtn = document.getElementById('ops-start-btn');
1442
+  if (startBtn) { startBtn.style.cssText = ''; startBtn.disabled = false; }
1443
+}
1444
+
1445
+function _btnLabelForType(type) {
1446
+  if (type === 'promote') return 'Promote';
1447
+  if (type === 'sync') return 'Sync';
1448
+  if (type === 'restart') return 'Restart';
1449
+  if (type === 'rebuild') return 'Rebuild';
1450
+  if (type === 'recreate') return 'Recreate';
1451
+  return 'Run';
1452
+}
1453
+
1454
+function startOperation() {
1455
+  const { type, project, fromEnv, toEnv } = opsCtx;
1456
+  if (!type || !project) return;
1457
+
1458
+  const dryRun = document.getElementById('ops-dry-run').checked;
1459
+  const startBtn = document.getElementById('ops-start-btn');
1460
+  const outputDiv = document.getElementById('ops-modal-output');
1461
+  const term = document.getElementById('ops-modal-terminal');
1462
+
1463
+  outputDiv.style.display = 'block';
1464
+  term.textContent = 'Starting...\n';
1465
+  startBtn.disabled = true;
1466
+  startBtn.textContent = 'Running...';
1467
+
1468
+  let url;
1469
+  if (type === 'promote') {
1470
+    url = '/api/promote/' + encodeURIComponent(project) + '/' + encodeURIComponent(fromEnv) + '/' + encodeURIComponent(toEnv) + '?dry_run=' + dryRun + '&token=' + encodeURIComponent(getToken());
1471
+  } else if (type === 'sync') {
1472
+    url = '/api/sync/' + encodeURIComponent(project) + '?from=' + encodeURIComponent(fromEnv) + '&to=' + encodeURIComponent(toEnv) + '&dry_run=' + dryRun + '&token=' + encodeURIComponent(getToken());
1473
+  } else if (type === 'restart' || type === 'rebuild' || type === 'recreate') {
1474
+    // All three lifecycle ops go through /api/rebuild/{project}/{env}?action=...
1475
+    url = '/api/rebuild/' + encodeURIComponent(project) + '/' + encodeURIComponent(fromEnv)
1476
+      + '?action=' + encodeURIComponent(type) + '&token=' + encodeURIComponent(getToken());
1477
+  }
1478
+
711
1479
   const es = new EventSource(url);
1480
+  opsEventSource = es;
1481
+  let opDone = false;
1482
+
712
1483
   es.onmessage = function(e) {
713
-    const d = JSON.parse(e.data);
714
-    if (d.done) {
715
-      es.close();
716
-      term.textContent += d.success ? '\n--- Restore complete ---\n' : '\n--- Restore FAILED ---\n';
717
-      toast(d.success ? 'Restore completed' : 'Restore failed', d.success ? 'success' : 'error');
718
-      return;
719
-    }
720
-    if (d.line) { term.textContent += d.line + '\n'; term.scrollTop = term.scrollHeight; }
1484
+    try {
1485
+      const d = JSON.parse(e.data);
1486
+      if (d.done) {
1487
+        opDone = true;
1488
+        es.close();
1489
+        opsEventSource = null;
1490
+        const msg = d.success ? '\n--- Operation complete ---\n' : '\n--- Operation FAILED ---\n';
1491
+        term.textContent += msg;
1492
+        term.scrollTop = term.scrollHeight;
1493
+        toast(d.success ? 'Operation completed' : 'Operation failed', d.success ? 'success' : 'error');
1494
+        startBtn.disabled = false;
1495
+        startBtn.textContent = _btnLabelForType(type);
1496
+
1497
+        // Show "Go to Backups" banner after recreate (or legacy rebuild)
1498
+        const showBackupBanner = (type === 'recreate') && d.success && d.project && d.env;
1499
+        if (showBackupBanner) {
1500
+          const restoreProject = d.project;
1501
+          const restoreEnv = d.env;
1502
+          const banner = document.createElement('div');
1503
+          banner.style.cssText = 'margin-top:1rem;padding:0.75rem 1rem;background:rgba(16,185,129,0.1);border:1px solid rgba(16,185,129,0.3);border-radius:0.5rem;display:flex;align-items:center;gap:0.75rem;';
1504
+          banner.innerHTML = '<span style="color:#6ee7b7;font-size:0.8125rem;flex:1;">Environment recreated. Next step: restore a backup.</span>'
1505
+            + '<button class="btn btn-ghost btn-sm" style="color:#6ee7b7;border-color:rgba(110,231,179,0.3);white-space:nowrap;" '
1506
+            + 'onclick="closeOpsModal();currentPage=\'backups\';backupDrillLevel=2;backupDrillProject=\'' + restoreProject + '\';backupDrillEnv=\'' + restoreEnv + '\';cachedBackups=null;selectedBackups.clear();document.querySelectorAll(\'#sidebar-nav .sidebar-link\').forEach(el=>el.classList.toggle(\'active\',el.dataset.page===\'backups\'));renderPage();pushHash();">'
1507
+            + 'Go to Backups &rarr;</button>';
1508
+          outputDiv.appendChild(banner);
1509
+        }
1510
+
1511
+        return;
1512
+      }
1513
+      if (d.line) {
1514
+        term.textContent += d.line + '\n';
1515
+        term.scrollTop = term.scrollHeight;
1516
+      }
1517
+    } catch (_) {}
721
1518
   };
722
-  es.onerror = function() { es.close(); term.textContent += '\n--- Connection lost ---\n'; toast('Connection lost', 'error'); };
1519
+
1520
+  es.onerror = function() {
1521
+    es.close();
1522
+    opsEventSource = null;
1523
+    if (opDone) return;
1524
+    term.textContent += '\n--- Connection lost ---\n';
1525
+    toast('Connection lost', 'error');
1526
+    startBtn.disabled = false;
1527
+    startBtn.textContent = _btnLabelForType(type);
1528
+  };
723
1529
 }
724
1530
 
725
1531
 // ---------------------------------------------------------------------------
..
..
@@ -758,20 +1564,65 @@
758
1564
   logCtx = { project: null, env: null, service: null };
759
1565
 }
760
1566
 
761
-function setBackupFilter(type, value) {
762
-  if (type === 'project') backupFilterProject = value;
763
-  if (type === 'env') backupFilterEnv = value;
764
-  renderBackups();
765
-}
766
-
767
1567
 async function createBackup(project, env) {
768
1568
   if (!confirm(`Create backup for ${project}/${env}?`)) return;
769
1569
   toast('Creating backup...', 'info');
770
1570
   try {
771
1571
     await api(`/api/backups/${project}/${env}`, { method: 'POST' });
772
1572
     toast('Backup created for ' + project + '/' + env, 'success');
1573
+    cachedBackups = null;
773
1574
     if (currentPage === 'backups') renderBackups();
774
1575
   } catch (e) { toast('Backup failed: ' + e.message, 'error'); }
1576
+}
1577
+
1578
+async function deleteBackup(project, env, name, hasLocal, hasOffsite) {
1579
+  let target;
1580
+  if (hasLocal && hasOffsite) {
1581
+    target = await showDeleteTargetDialog(name);
1582
+    if (!target) return;
1583
+  } else if (hasLocal) {
1584
+    target = 'local';
1585
+  } else {
1586
+    target = 'offsite';
1587
+  }
1588
+  const label = target === 'both' ? 'local + offsite' : target;
1589
+  if (!confirm(`Delete ${label} copy of ${name}?\n\nThis cannot be undone.`)) return;
1590
+  toast('Deleting backup (' + label + ')...', 'info');
1591
+  try {
1592
+    await api(`/api/backups/${encodeURIComponent(project)}/${encodeURIComponent(env)}/${encodeURIComponent(name)}?target=${target}`, { method: 'DELETE' });
1593
+    toast('Backup deleted: ' + name + ' (' + label + ')', 'success');
1594
+    cachedBackups = null;
1595
+    if (currentPage === 'backups') renderBackups();
1596
+  } catch (e) { toast('Delete failed: ' + e.message, 'error'); }
1597
+}
1598
+
1599
+function showDeleteTargetDialog(name) {
1600
+  return new Promise(resolve => {
1601
+    const overlay = document.createElement('div');
1602
+    overlay.style.cssText = 'position:fixed;inset:0;background:rgba(0,0,0,0.6);display:flex;align-items:center;justify-content:center;z-index:9999;';
1603
+    const box = document.createElement('div');
1604
+    box.style.cssText = 'background:#1e293b;border:1px solid #334155;border-radius:0.75rem;padding:1.5rem;min-width:320px;max-width:420px;color:#e2e8f0;';
1605
+    box.innerHTML = `
1606
+      <h3 style="margin:0 0 0.5rem;font-size:1rem;color:#f1f5f9;">Delete from where?</h3>
1607
+      <p style="margin:0 0 1.25rem;font-size:0.85rem;color:#94a3b8;">This backup exists in both local and offsite storage.</p>
1608
+      <div style="display:flex;flex-direction:column;gap:0.5rem;">
1609
+        <button class="btn btn-ghost" style="justify-content:flex-start;color:#f87171;border-color:#7f1d1d;" data-target="local">Local only</button>
1610
+        <button class="btn btn-ghost" style="justify-content:flex-start;color:#a78bfa;border-color:rgba(167,139,250,0.25);" data-target="offsite">Offsite only</button>
1611
+        <button class="btn btn-danger" style="justify-content:flex-start;" data-target="both">Both (local + offsite)</button>
1612
+        <button class="btn btn-ghost" style="justify-content:flex-start;margin-top:0.25rem;" data-target="">Cancel</button>
1613
+      </div>`;
1614
+    overlay.appendChild(box);
1615
+    document.body.appendChild(overlay);
1616
+    box.addEventListener('click', e => {
1617
+      const btn = e.target.closest('[data-target]');
1618
+      if (!btn) return;
1619
+      document.body.removeChild(overlay);
1620
+      resolve(btn.dataset.target || null);
1621
+    });
1622
+    overlay.addEventListener('click', e => {
1623
+      if (e.target === overlay) { document.body.removeChild(overlay); resolve(null); }
1624
+    });
1625
+  });
775
1626
 }
776
1627
 
777
1628
 // ---------------------------------------------------------------------------
..
..
@@ -781,6 +1632,89 @@
781
1632
   const m = {};
782
1633
   for (const item of arr) { const k = item[key] || 'other'; (m[k] = m[k] || []).push(item); }
783
1634
   return m;
1635
+}
1636
+
1637
+// ---------------------------------------------------------------------------
1638
+// URL Hash Routing
1639
+// ---------------------------------------------------------------------------
1640
+function pushHash() {
1641
+  let hash = '';
1642
+  if (currentPage === 'dashboard') {
1643
+    if (viewMode === 'table') {
1644
+      hash = '/dashboard/table';
1645
+      if (tableFilter) hash += '/' + encodeURIComponent(tableFilter);
1646
+    } else if (drillLevel === 2) {
1647
+      hash = '/dashboard/' + encodeURIComponent(drillProject) + '/' + encodeURIComponent(drillEnv);
1648
+    } else if (drillLevel === 1) {
1649
+      hash = '/dashboard/' + encodeURIComponent(drillProject);
1650
+    } else {
1651
+      hash = '/dashboard';
1652
+    }
1653
+  } else if (currentPage === 'backups') {
1654
+    if (backupDrillLevel === 2) {
1655
+      hash = '/backups/' + encodeURIComponent(backupDrillProject) + '/' + encodeURIComponent(backupDrillEnv);
1656
+    } else if (backupDrillLevel === 1) {
1657
+      hash = '/backups/' + encodeURIComponent(backupDrillProject);
1658
+    } else {
1659
+      hash = '/backups';
1660
+    }
1661
+  } else if (currentPage === 'system') {
1662
+    hash = '/system';
1663
+  } else if (currentPage === 'operations') {
1664
+    hash = '/operations';
1665
+  }
1666
+  const newHash = '#' + hash;
1667
+  if (window.location.hash !== newHash) {
1668
+    history.replaceState(null, '', newHash);
1669
+  }
1670
+}
1671
+
1672
+function navigateToHash() {
1673
+  const raw = (window.location.hash || '').replace(/^#\/?/, '');
1674
+  const parts = raw.split('/').map(decodeURIComponent).filter(Boolean);
1675
+
1676
+  if (!parts.length) { showPage('dashboard'); return; }
1677
+
1678
+  const page = parts[0];
1679
+  if (page === 'dashboard') {
1680
+    currentPage = 'dashboard';
1681
+    drillLevel = 0; drillProject = null; drillEnv = null;
1682
+    viewMode = 'cards'; tableFilter = null; tableFilterLabel = '';
1683
+    cachedBackups = null;
1684
+    backupDrillLevel = 0; backupDrillProject = null; backupDrillEnv = null;
1685
+
1686
+    if (parts[1] === 'table') {
1687
+      viewMode = 'table';
1688
+      if (parts[2]) { tableFilter = parts[2]; tableFilterLabel = parts[2]; }
1689
+    } else if (parts[1]) {
1690
+      drillProject = parts[1]; drillLevel = 1;
1691
+      if (parts[2]) { drillEnv = parts[2]; drillLevel = 2; }
1692
+    }
1693
+    document.querySelectorAll('#sidebar-nav .sidebar-link').forEach(el =>
1694
+      el.classList.toggle('active', el.dataset.page === 'dashboard'));
1695
+    renderPage();
1696
+  } else if (page === 'backups') {
1697
+    currentPage = 'backups';
1698
+    drillLevel = 0; drillProject = null; drillEnv = null;
1699
+    viewMode = 'cards'; tableFilter = null; tableFilterLabel = '';
1700
+    cachedBackups = null;
1701
+    backupDrillLevel = 0; backupDrillProject = null; backupDrillEnv = null;
1702
+    selectedBackups.clear();
1703
+
1704
+    if (parts[1]) {
1705
+      backupDrillProject = parts[1]; backupDrillLevel = 1;
1706
+      if (parts[2]) { backupDrillEnv = parts[2]; backupDrillLevel = 2; }
1707
+    }
1708
+    document.querySelectorAll('#sidebar-nav .sidebar-link').forEach(el =>
1709
+      el.classList.toggle('active', el.dataset.page === 'backups'));
1710
+    renderPage();
1711
+  } else if (page === 'system') {
1712
+    showPage('system');
1713
+  } else if (page === 'operations') {
1714
+    showPage('operations');
1715
+  } else {
1716
+    showPage('dashboard');
1717
+  }
784
1718
 }
785
1719
 
786
1720
 // ---------------------------------------------------------------------------
..
..
@@ -795,11 +1729,20 @@
795
1729
         allServices = data;
796
1730
         document.getElementById('login-overlay').style.display = 'none';
797
1731
         document.getElementById('app').style.display = 'flex';
798
-      const vEl = document.getElementById('app-version'); if (vEl && typeof APP_VERSION !== 'undefined') vEl.textContent = APP_VERSION;
799
-        showPage('dashboard');
1732
+        const vEl = document.getElementById('app-version'); if (vEl && typeof APP_VERSION !== 'undefined') vEl.textContent = APP_VERSION;
1733
+        navigateToHash();
800
1734
         startAutoRefresh();
801
1735
       })
802
1736
       .catch(() => { localStorage.removeItem('ops_token'); });
803
1737
   }
804
-  document.addEventListener('keydown', e => { if (e.key === 'Escape') closeLogModal(); });
1738
+  document.addEventListener('keydown', e => {
1739
+    if (e.key === 'Escape') {
1740
+      closeLogModal();
1741
+      closeRestoreModal();
1742
+      closeOpsModal();
1743
+    }
1744
+  });
1745
+  window.addEventListener('hashchange', () => {
1746
+    if (getToken()) navigateToHash();
1747
+  });
805
1748
 })();