| .. | .. |
|---|
| 19 | 19 | |
|---|
| 20 | 20 | from app.auth import verify_token |
|---|
| 21 | 21 | from app.ops_runner import ( |
|---|
| 22 | + OPS_CLI, |
|---|
| 22 | 23 | _BACKUP_TIMEOUT, |
|---|
| 23 | 24 | run_command, |
|---|
| 24 | 25 | run_command_host, |
|---|
| .. | .. |
|---|
| 298 | 299 | |
|---|
| 299 | 300 | async def _op_rebuild(project: str, env: str) -> AsyncGenerator[str, None]: |
|---|
| 300 | 301 | """ |
|---|
| 301 | | - Rebuild: Coolify stop → docker build → Coolify start. |
|---|
| 302 | + Rebuild: docker compose down → build image → docker compose up. |
|---|
| 303 | + Uses `ops rebuild` on the host which handles env files, profiles, and cd correctly. |
|---|
| 302 | 304 | No data loss. For code/Dockerfile changes. |
|---|
| 303 | 305 | """ |
|---|
| 304 | | - try: |
|---|
| 305 | | - uuid = _coolify_uuid(project, env) |
|---|
| 306 | | - build = _build_cfg(project, env) |
|---|
| 307 | | - except ValueError as exc: |
|---|
| 308 | | - yield _line(f"[error] Config error: {exc}") |
|---|
| 309 | | - yield _done(False, project, env, "rebuild") |
|---|
| 310 | | - return |
|---|
| 306 | + yield _line(f"[rebuild] Rebuilding {project}/{env} via ops CLI...") |
|---|
| 311 | 307 | |
|---|
| 312 | | - # Step 1: Stop via Coolify |
|---|
| 313 | | - yield _line(f"[rebuild] Stopping {project}/{env} via Coolify (uuid={uuid})...") |
|---|
| 314 | | - try: |
|---|
| 315 | | - await _coolify_action("stop", uuid) |
|---|
| 316 | | - yield _line(f"[rebuild] Coolify stop queued. Waiting for containers to stop...") |
|---|
| 317 | | - # Step 2: Poll until stopped |
|---|
| 318 | | - stopped = await _poll_until_stopped(project, env) |
|---|
| 319 | | - if not stopped: |
|---|
| 320 | | - yield _line(f"[warn] Containers may still be running after {_POLL_MAX_WAIT}s — proceeding anyway") |
|---|
| 321 | | - else: |
|---|
| 322 | | - yield _line(f"[rebuild] All containers stopped.") |
|---|
| 323 | | - except RuntimeError as exc: |
|---|
| 324 | | - if "already stopped" in str(exc).lower(): |
|---|
| 325 | | - yield _line(f"[rebuild] Service already stopped — continuing with build.") |
|---|
| 326 | | - else: |
|---|
| 327 | | - yield _line(f"[error] Coolify stop failed: {exc}") |
|---|
| 328 | | - yield _done(False, project, env, "rebuild") |
|---|
| 329 | | - return |
|---|
| 330 | | - |
|---|
| 331 | | - # Step 3: Build image (if project uses local images) |
|---|
| 332 | | - if build: |
|---|
| 333 | | - ctx = build["build_context"] |
|---|
| 334 | | - image = f"{build['image_name']}:{env}" |
|---|
| 335 | | - yield _line(f"[rebuild] Building Docker image: {image}") |
|---|
| 336 | | - yield _line(f"[rebuild] Build context: {ctx}") |
|---|
| 337 | | - |
|---|
| 338 | | - async for line in stream_command_host( |
|---|
| 339 | | - ["docker", "build", "-t", image, ctx], |
|---|
| 340 | | - timeout=_BACKUP_TIMEOUT, |
|---|
| 341 | | - ): |
|---|
| 308 | + had_output = False |
|---|
| 309 | + success = True |
|---|
| 310 | + async for line in stream_command_host( |
|---|
| 311 | + [OPS_CLI, "rebuild", project, env], |
|---|
| 312 | + timeout=_BACKUP_TIMEOUT, |
|---|
| 313 | + ): |
|---|
| 314 | + had_output = True |
|---|
| 315 | + if line.startswith("[stderr] "): |
|---|
| 342 | 316 | yield _line(line) |
|---|
| 343 | | - else: |
|---|
| 344 | | - yield _line(f"[rebuild] No local image build needed (registry images only).") |
|---|
| 317 | + elif line.startswith("ERROR") or line.startswith("[error]"): |
|---|
| 318 | + yield _line(f"[error] {line}") |
|---|
| 319 | + success = False |
|---|
| 320 | + else: |
|---|
| 321 | + yield _line(f"[rebuild] {line}") |
|---|
| 345 | 322 | |
|---|
| 346 | | - # Step 4: Start via Coolify |
|---|
| 347 | | - yield _line(f"[rebuild] Starting {project}/{env} via Coolify...") |
|---|
| 348 | | - try: |
|---|
| 349 | | - await _coolify_action("start", uuid) |
|---|
| 350 | | - yield _line(f"[rebuild] Coolify start queued. Waiting for containers...") |
|---|
| 351 | | - except RuntimeError as exc: |
|---|
| 352 | | - yield _line(f"[error] Coolify start failed: {exc}") |
|---|
| 353 | | - yield _done(False, project, env, "rebuild") |
|---|
| 354 | | - return |
|---|
| 323 | + if not had_output: |
|---|
| 324 | + yield _line(f"[error] ops rebuild produced no output — check registry config for {project}") |
|---|
| 325 | + success = False |
|---|
| 355 | 326 | |
|---|
| 356 | | - # Step 5: Poll until running |
|---|
| 357 | | - running = await _poll_until_running(project, env) |
|---|
| 358 | | - if running: |
|---|
| 359 | | - yield _line(f"[rebuild] Containers are up.") |
|---|
| 360 | | - yield _done(True, project, env, "rebuild") |
|---|
| 327 | + if success: |
|---|
| 328 | + # Verify containers came up |
|---|
| 329 | + containers = await _find_containers_for_service(project, env) |
|---|
| 330 | + if containers: |
|---|
| 331 | + yield _line(f"[rebuild] {len(containers)} container(s) running: {', '.join(containers)}") |
|---|
| 332 | + yield _done(True, project, env, "rebuild") |
|---|
| 333 | + else: |
|---|
| 334 | + yield _line(f"[warn] No containers found after rebuild — check docker compose logs") |
|---|
| 335 | + yield _done(False, project, env, "rebuild") |
|---|
| 361 | 336 | else: |
|---|
| 362 | | - yield _line(f"[warn] Containers did not appear healthy within {_POLL_MAX_WAIT}s — check Coolify logs.") |
|---|
| 363 | 337 | yield _done(False, project, env, "rebuild") |
|---|
| 364 | 338 | |
|---|
| 365 | 339 | |
|---|
| .. | .. |
|---|
| 369 | 343 | |
|---|
| 370 | 344 | async def _op_recreate(project: str, env: str) -> AsyncGenerator[str, None]: |
|---|
| 371 | 345 | """ |
|---|
| 372 | | - Recreate: Coolify stop → wipe data → docker build → Coolify start. |
|---|
| 346 | + Recreate: docker compose down → wipe data → docker build → docker compose up. |
|---|
| 373 | 347 | DESTRUCTIVE — wipes all data volumes. Shows "Go to Backups" banner on success. |
|---|
| 374 | 348 | """ |
|---|
| 375 | 349 | try: |
|---|
| 376 | | - uuid = _coolify_uuid(project, env) |
|---|
| 377 | | - build = _build_cfg(project, env) |
|---|
| 378 | 350 | data_dir = _data_dir(project, env) |
|---|
| 379 | 351 | cfg = _project_cfg(project) |
|---|
| 380 | 352 | except ValueError as exc: |
|---|
| .. | .. |
|---|
| 382 | 354 | yield _done(False, project, env, "recreate") |
|---|
| 383 | 355 | return |
|---|
| 384 | 356 | |
|---|
| 385 | | - # Step 1: Stop via Coolify |
|---|
| 386 | | - yield _line(f"[recreate] Stopping {project}/{env} via Coolify (uuid={uuid})...") |
|---|
| 387 | | - try: |
|---|
| 388 | | - await _coolify_action("stop", uuid) |
|---|
| 389 | | - yield _line(f"[recreate] Coolify stop queued. Waiting for containers to stop...") |
|---|
| 390 | | - # Step 2: Poll until stopped |
|---|
| 391 | | - stopped = await _poll_until_stopped(project, env) |
|---|
| 392 | | - if not stopped: |
|---|
| 393 | | - yield _line(f"[warn] Containers may still be running after {_POLL_MAX_WAIT}s — proceeding anyway") |
|---|
| 394 | | - else: |
|---|
| 395 | | - yield _line(f"[recreate] All containers stopped.") |
|---|
| 396 | | - except RuntimeError as exc: |
|---|
| 397 | | - if "already stopped" in str(exc).lower(): |
|---|
| 398 | | - yield _line(f"[recreate] Service already stopped — skipping stop step.") |
|---|
| 399 | | - else: |
|---|
| 400 | | - yield _line(f"[error] Coolify stop failed: {exc}") |
|---|
| 401 | | - yield _done(False, project, env, "recreate") |
|---|
| 402 | | - return |
|---|
| 357 | + # Step 1: Find and stop containers via docker compose |
|---|
| 358 | + code_dir = cfg.get("path", "") + f"/{env}/code" |
|---|
| 359 | + yield _line(f"[recreate] Stopping {project}/{env} containers...") |
|---|
| 403 | 360 | |
|---|
| 404 | | - # Step 3: Wipe data volumes |
|---|
| 405 | | - yield _line(f"[recreate] WARNING: Wiping data directory: {data_dir}") |
|---|
| 406 | | - # Verify THIS env's containers are actually stopped before wiping |
|---|
| 361 | + stop_result = await run_command_host( |
|---|
| 362 | + ["sh", "-c", f"cd {code_dir} && docker compose -p {env}-{cfg.get('name_prefix', project)} --profile {env} down 2>&1 || true"], |
|---|
| 363 | + timeout=120, |
|---|
| 364 | + ) |
|---|
| 365 | + if stop_result["output"].strip(): |
|---|
| 366 | + for line in stop_result["output"].strip().splitlines(): |
|---|
| 367 | + yield _line(line) |
|---|
| 368 | + |
|---|
| 369 | + # Step 2: Verify containers are stopped |
|---|
| 407 | 370 | name_prefix = cfg.get("name_prefix", project) |
|---|
| 408 | | - # Use grep to AND-match both prefix and env (docker --filter uses OR for multiple name filters) |
|---|
| 409 | 371 | verify = await run_command_host( |
|---|
| 410 | | - ["sh", "-c", f"docker ps --format '{{{{.Names}}}}' | grep '^{env}-{name_prefix}\\|^{name_prefix}-{env}' || true"], |
|---|
| 372 | + ["sh", "-c", f"docker ps --format '{{{{.Names}}}}' | grep '^{env}-{name_prefix}-' || true"], |
|---|
| 411 | 373 | timeout=30, |
|---|
| 412 | 374 | ) |
|---|
| 413 | | - running = verify["output"].strip() |
|---|
| 414 | | - if running: |
|---|
| 375 | + running_containers = verify["output"].strip() |
|---|
| 376 | + if running_containers: |
|---|
| 415 | 377 | yield _line(f"[error] Containers still running for {project}/{env}:") |
|---|
| 416 | | - for line in running.splitlines(): |
|---|
| 378 | + for line in running_containers.splitlines(): |
|---|
| 417 | 379 | yield _line(f" {line}") |
|---|
| 418 | 380 | yield _done(False, project, env, "recreate") |
|---|
| 419 | 381 | return |
|---|
| 382 | + yield _line(f"[recreate] All containers stopped.") |
|---|
| 420 | 383 | |
|---|
| 384 | + # Step 3: Wipe data volumes |
|---|
| 385 | + yield _line(f"[recreate] WARNING: Wiping data directory: {data_dir}") |
|---|
| 421 | 386 | wipe_result = await run_command_host( |
|---|
| 422 | 387 | ["sh", "-c", f"rm -r {data_dir}/* 2>&1; echo EXIT_CODE=$?"], |
|---|
| 423 | 388 | timeout=120, |
|---|
| .. | .. |
|---|
| 432 | 397 | yield _done(False, project, env, "recreate") |
|---|
| 433 | 398 | return |
|---|
| 434 | 399 | |
|---|
| 435 | | - # Step 4: Build image (if project uses local images) |
|---|
| 436 | | - if build: |
|---|
| 437 | | - ctx = build["build_context"] |
|---|
| 438 | | - image = f"{build['image_name']}:{env}" |
|---|
| 439 | | - yield _line(f"[recreate] Building Docker image: {image}") |
|---|
| 440 | | - yield _line(f"[recreate] Build context: {ctx}") |
|---|
| 441 | | - |
|---|
| 442 | | - async for line in stream_command_host( |
|---|
| 443 | | - ["docker", "build", "-t", image, ctx], |
|---|
| 444 | | - timeout=_BACKUP_TIMEOUT, |
|---|
| 445 | | - ): |
|---|
| 400 | + # Step 4: Rebuild via ops CLI (handles image build + compose up) |
|---|
| 401 | + yield _line(f"[recreate] Rebuilding containers...") |
|---|
| 402 | + async for line in stream_command_host( |
|---|
| 403 | + [OPS_CLI, "rebuild", project, env], |
|---|
| 404 | + timeout=_BACKUP_TIMEOUT, |
|---|
| 405 | + ): |
|---|
| 406 | + if line.startswith("[stderr] "): |
|---|
| 446 | 407 | yield _line(line) |
|---|
| 447 | | - else: |
|---|
| 448 | | - yield _line(f"[recreate] No local image build needed (registry images only).") |
|---|
| 408 | + else: |
|---|
| 409 | + yield _line(f"[recreate] {line}") |
|---|
| 449 | 410 | |
|---|
| 450 | | - # Step 5: Start via Coolify |
|---|
| 451 | | - yield _line(f"[recreate] Starting {project}/{env} via Coolify...") |
|---|
| 452 | | - try: |
|---|
| 453 | | - await _coolify_action("start", uuid) |
|---|
| 454 | | - yield _line(f"[recreate] Coolify start queued. Waiting for containers...") |
|---|
| 455 | | - except RuntimeError as exc: |
|---|
| 456 | | - yield _line(f"[error] Coolify start failed: {exc}") |
|---|
| 457 | | - yield _done(False, project, env, "recreate") |
|---|
| 458 | | - return |
|---|
| 459 | | - |
|---|
| 460 | | - # Step 6: Poll until running |
|---|
| 461 | | - running = await _poll_until_running(project, env) |
|---|
| 462 | | - if running: |
|---|
| 463 | | - yield _line(f"[recreate] Containers are up. Restore a backup to complete recovery.") |
|---|
| 411 | + # Step 5: Verify containers came up |
|---|
| 412 | + containers = await _find_containers_for_service(project, env) |
|---|
| 413 | + if containers: |
|---|
| 414 | + yield _line(f"[recreate] {len(containers)} container(s) running. Restore a backup to complete recovery.") |
|---|
| 464 | 415 | yield _done(True, project, env, "recreate") |
|---|
| 465 | 416 | else: |
|---|
| 466 | | - yield _line(f"[warn] Containers did not appear within {_POLL_MAX_WAIT}s — check Coolify logs.") |
|---|
| 467 | | - # Still return success=True so the "Go to Backups" banner appears |
|---|
| 417 | + yield _line(f"[warn] No containers found after recreate — check docker compose logs") |
|---|
| 468 | 418 | yield _done(True, project, env, "recreate") |
|---|
| 469 | 419 | |
|---|
| 470 | 420 | |
|---|