Refactor bot and integrate services

services/__init__.py

@@ -0,0 +1 @@
+__all__ = []

services/artifacts.py

@@ -0,0 +1,22 @@
+import json
+from datetime import datetime
+from pathlib import Path
+
+
+def artifact_last(state_file: str) -> str:
+    data = json.loads(Path(state_file).read_text())
+    items = sorted(
+        data.items(),
+        key=lambda x: x[1]["updated_at"],
+        reverse=True
+    )
+    name, info = items[0]
+    t = datetime.fromisoformat(info["updated_at"])
+    age_h = int((datetime.now() - t).total_seconds() / 3600)
+
+    return (
+        "🧉 Last artifact\n\n"
+        f"{name}\n"
+        f"Updated: {t:%Y-%m-%d %H:%M}\n"
+        f"Age: {age_h}h"
+    )

services/backup.py

@@ -0,0 +1,58 @@
+from datetime import datetime, timezone
+from typing import Optional
+import json
+import subprocess
+from services.runner import run_cmd
+
+
+def backup_badge(last_time: datetime) -> str:
+    age = datetime.now(timezone.utc) - last_time
+    hours = age.total_seconds() / 3600
+
+    if hours < 24:
+        return "🟢 Backup: OK"
+    if hours < 72:
+        return "🟡 Backup: stale"
+    return "🔴 Backup: OLD"
+
+
+async def get_last_snapshot() -> Optional[dict]:
+    rc, raw = await run_cmd(
+        ["restic", "snapshots", "--json"],
+        use_restic_env=True,
+        timeout=20
+    )
+    if rc != 0:
+        return None
+
+    snaps = json.loads(raw)
+    if not snaps:
+        return None
+
+    snaps.sort(key=lambda s: s["time"], reverse=True)
+    return snaps[0]
+
+
+def last_backup() -> str:
+    out = subprocess.check_output(
+        ["restic", "snapshots", "--json"],
+        env=None
+    ).decode()
+    snaps = json.loads(out)
+    snaps.sort(key=lambda s: s["time"], reverse=True)
+    s = snaps[0]
+    t = datetime.fromisoformat(s["time"].replace("Z", ""))
+    return (
+        "📦 Last backup\n\n"
+        f"🕒 {t:%Y-%m-%d %H:%M}\n"
+        f"🧉 ID: {s['short_id']}\n"
+        f"📁 Paths: {len(s['paths'])}"
+    )
+
+
+def restore_help() -> str:
+    return (
+        "🧯 Restore help\n\n"
+        "Example:\n"
+        "restic restore <snapshot_id> --target /restore"
+    )

services/docker.py

@@ -0,0 +1,114 @@
+import asyncio
+from datetime import datetime, timezone
+from typing import Dict
+from services.runner import run_cmd
+
+
+def container_uptime(started_at: str) -> str:
+    """
+    started_at: 2026-02-06T21:14:33.123456789Z
+    """
+    try:
+        start = datetime.fromisoformat(
+            started_at.replace("Z", "+00:00")
+        ).astimezone(timezone.utc)
+        delta = datetime.now(timezone.utc) - start
+        days = delta.days
+        hours = delta.seconds // 3600
+        minutes = (delta.seconds % 3600) // 60
+
+        if days > 0:
+            return f"{days}d {hours}h"
+        if hours > 0:
+            return f"{hours}h {minutes}m"
+        return f"{minutes}m"
+    except Exception:
+        return "unknown"
+
+
+async def build_docker_map(cfg) -> Dict[str, str]:
+    docker_cfg = cfg.get("docker", {})
+
+    result: Dict[str, str] = {}
+
+    # 1. autodiscovery
+    if docker_cfg.get("autodiscovery"):
+        rc, raw = await run_cmd(
+            ["sudo", "docker", "ps", "--format", "{{.Names}}"],
+            timeout=20
+        )
+        if rc == 0:
+            names = raw.splitlines()
+            patterns = docker_cfg.get("match", [])
+            for name in names:
+                if any(p in name for p in patterns):
+                    result[name] = name
+
+    # 2. aliases override
+    aliases = docker_cfg.get("aliases", {})
+    for alias, real in aliases.items():
+        result[alias] = real
+
+    return result
+
+
+async def discover_containers(cfg) -> Dict[str, str]:
+    """
+    returns: alias -> real container name
+    """
+    docker_cfg = cfg.get("docker", {})
+    result: Dict[str, str] = {}
+
+    # --- autodiscovery ---
+    if docker_cfg.get("autodiscovery"):
+        rc, raw = await run_cmd(
+            ["sudo", "docker", "ps", "--format", "{{.Names}}"],
+            timeout=20
+        )
+
+        if rc == 0:
+            found = raw.splitlines()
+
+            label = docker_cfg.get("label")
+            patterns = docker_cfg.get("match", [])
+
+            for name in found:
+                # label-based discovery
+                if label:
+                    key, val = label.split("=", 1)
+                    rc2, lbl = await run_cmd([
+                        "sudo", "docker", "inspect",
+                        "-f", f"{{{{ index .Config.Labels \"{key}\" }}}}",
+                        name
+                    ])
+                    if rc2 == 0 and lbl.strip() == val:
+                        result[name] = name
+                    continue
+
+                # name-pattern discovery
+                if any(p in name for p in patterns):
+                    result[name] = name
+
+    # --- manual aliases ALWAYS override ---
+    aliases = docker_cfg.get("aliases", {})
+    for alias, real in aliases.items():
+        result[alias] = real
+
+    return result
+
+
+async def docker_watchdog(container_map, notify, bot, chat_id):
+    last = {}
+    while True:
+        for alias, real in container_map.items():
+            rc, state = await run_cmd(
+                ["docker", "inspect", "-f", "{{.State.Status}}", real],
+                timeout=10
+            )
+            if rc != 0:
+                state = "error"
+            state = state.strip()
+            if last.get(alias) != state:
+                await notify(bot, chat_id, f"🐳 {alias}: {state}")
+                last[alias] = state
+        await asyncio.sleep(120)

services/health.py

@@ -0,0 +1,40 @@
+import subprocess
+import psutil
+from services.system import worst_disk_usage
+
+
+def _containers_from_cfg(cfg) -> dict:
+    return cfg.get("docker", {}).get("containers", {})
+
+
+def health(cfg, container_map: dict | None = None) -> str:
+    lines = ["🩺 Health check\n"]
+
+    try:
+        subprocess.check_output(["restic", "snapshots"], timeout=10)
+        lines.append("🟢 Backup repo reachable")
+    except Exception:
+        lines.append("🔴 Backup repo unreachable")
+
+    containers = container_map if container_map is not None else _containers_from_cfg(cfg)
+    for alias, real in containers.items():
+        out = subprocess.getoutput(
+            f"docker inspect -f '{{{{.State.Status}}}}' {real}"
+        )
+        if out.strip() != "running":
+            lines.append(f"🔴 {alias} down")
+        else:
+            lines.append(f"🟢 {alias} OK")
+
+    usage, mount = worst_disk_usage()
+    if usage is None:
+        lines.append("⚠️ Disk n/a")
+    elif usage > cfg["thresholds"]["disk_warn"]:
+        lines.append(f"🟡 Disk {usage}% ({mount})")
+    else:
+        lines.append(f"🟢 Disk {usage}% ({mount})")
+
+    load = psutil.getloadavg()[0]
+    lines.append(f"{'🟢' if load < cfg['thresholds']['load_warn'] else '🟡'} Load {load}")
+
+    return "\n".join(lines)

services/notify.py

@@ -0,0 +1,8 @@
+from aiogram import Bot
+
+
+async def notify(bot: Bot, chat_id: int, text: str):
+    try:
+        await bot.send_message(chat_id, text)
+    except Exception:
+        pass

services/runner.py

@@ -0,0 +1,24 @@
+import asyncio
+import os
+from app import RESTIC_ENV
+
+
+async def run_cmd(cmd: list[str], *, use_restic_env: bool = False, timeout: int = 60):
+    env = os.environ.copy()
+    env["PATH"] = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+    if use_restic_env:
+        env.update(RESTIC_ENV)
+
+    proc = await asyncio.create_subprocess_exec(
+        *cmd,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.STDOUT,
+        env=env,
+    )
+
+    try:
+        out, _ = await asyncio.wait_for(proc.communicate(), timeout=timeout)
+        return proc.returncode, out.decode(errors="ignore")[-3500:]
+    except asyncio.TimeoutError:
+        proc.kill()
+        return 124, "❌ timeout"

services/system.py

@@ -0,0 +1,69 @@
+import psutil
+
+
+def format_disks() -> str:
+    parts = psutil.disk_partitions(all=False)
+    lines = []
+
+    skip_prefixes = (
+        "/snap",
+        "/proc",
+        "/sys",
+        "/run",
+        "/boot/efi",
+    )
+
+    for p in parts:
+        mp = p.mountpoint
+        if mp.startswith(skip_prefixes):
+            continue
+
+        try:
+            usage = psutil.disk_usage(mp)
+        except PermissionError:
+            continue
+
+        icon = "🟢"
+        if usage.percent > 90:
+            icon = "🔴"
+        elif usage.percent > 80:
+            icon = "🟡"
+
+        lines.append(
+            f"{icon} **{mp}**: "
+            f"{usage.used // (1024**3)} / {usage.total // (1024**3)} GiB "
+            f"({usage.percent}%)"
+        )
+
+    if not lines:
+        return "💽 Disks: n/a"
+
+    return "💽 **Disks**\n" + "\n".join(lines)
+
+
+def worst_disk_usage() -> tuple[int | None, str | None]:
+    parts = psutil.disk_partitions(all=False)
+    skip_prefixes = (
+        "/snap",
+        "/proc",
+        "/sys",
+        "/run",
+        "/boot/efi",
+    )
+
+    worst_percent = None
+    worst_mount = None
+
+    for p in parts:
+        mp = p.mountpoint
+        if mp.startswith(skip_prefixes):
+            continue
+        try:
+            usage = psutil.disk_usage(mp)
+        except PermissionError:
+            continue
+        if worst_percent is None or usage.percent > worst_percent:
+            worst_percent = int(usage.percent)
+            worst_mount = mp
+
+    return worst_percent, worst_mount