Add runtime state, auto-mute schedules, and backup retries

services/alert_mute.py

@@ -1,37 +1,53 @@
 import time
 from typing import Dict
+from services.runtime_state import get_state, set_state
 
 # category -> unix timestamp until muted
-_MUTES: Dict[str, float] = {}
+
+
+def _mutes() -> Dict[str, float]:
+    return get_state().get("mutes", {})
+
+
+def _save(mutes: Dict[str, float]):
+    set_state("mutes", mutes)
 
 
 def _cleanup() -> None:
+    mutes = _mutes()
     now = time.time()
-    expired = [k for k, until in _MUTES.items() if until <= now]
+    expired = [k for k, until in mutes.items() if until <= now]
     for k in expired:
-        _MUTES.pop(k, None)
+        mutes.pop(k, None)
+    _save(mutes)
 
 
 def set_mute(category: str, seconds: int) -> float:
     _cleanup()
+    mutes = _mutes()
     until = time.time() + max(0, seconds)
-    _MUTES[category] = until
+    mutes[category] = until
+    _save(mutes)
     return until
 
 
 def clear_mute(category: str) -> None:
-    _MUTES.pop(category, None)
+    mutes = _mutes()
+    mutes.pop(category, None)
+    _save(mutes)
 
 
 def is_muted(category: str | None) -> bool:
     if not category:
         return False
     _cleanup()
-    until = _MUTES.get(category)
+    mutes = _mutes()
+    until = mutes.get(category)
     if until is None:
         return False
     if until <= time.time():
-        _MUTES.pop(category, None)
+        mutes.pop(category, None)
+        _save(mutes)
         return False
     return True
 
@@ -39,4 +55,39 @@ def is_muted(category: str | None) -> bool:
 def list_mutes() -> dict[str, int]:
     _cleanup()
     now = time.time()
-    return {k: int(until - now) for k, until in _MUTES.items()}
+    mutes = _mutes()
+    return {k: int(until - now) for k, until in mutes.items()}
+
+
+def is_auto_muted(cfg: dict, category: str | None) -> bool:
+    if not category:
+        return False
+    auto_list = cfg.get("alerts", {}).get("auto_mute", [])
+    if not isinstance(auto_list, list):
+        return False
+    now = time.localtime()
+    now_minutes = now.tm_hour * 60 + now.tm_min
+    for item in auto_list:
+        if not isinstance(item, dict):
+            continue
+        cat = item.get("category")
+        if cat != category:
+            continue
+        start = item.get("start", "00:00")
+        end = item.get("end", "00:00")
+        try:
+            sh, sm = [int(x) for x in start.split(":")]
+            eh, em = [int(x) for x in end.split(":")]
+        except Exception:
+            continue
+        start_min = sh * 60 + sm
+        end_min = eh * 60 + em
+        if start_min == end_min:
+            continue
+        if start_min < end_min:
+            if start_min <= now_minutes < end_min:
+                return True
+        else:
+            if now_minutes >= start_min or now_minutes < end_min:
+                return True
+    return False

services/notify.py

@@ -2,7 +2,7 @@ import time
 from datetime import datetime
 from aiogram import Bot
 from app import cfg
-from services.alert_mute import is_muted
+from services.alert_mute import is_muted, is_auto_muted
 from services.incidents import log_incident
 
 
@@ -49,6 +49,8 @@ async def notify(
     alerts_cfg = cfg.get("alerts", {})
     if category and is_muted(category):
         return
+    if category and is_auto_muted(cfg, category):
+        return
     if _in_quiet_hours(alerts_cfg):
         allow_critical = bool(alerts_cfg.get("quiet_hours", {}).get("allow_critical", True))
         if not (allow_critical and level == "critical"):

services/openwrt.py

@@ -447,4 +447,6 @@ async def get_openwrt_status(cfg: dict[str, Any], mode: str = "full") -> str:
         return "\n".join(header)
     if mode == "clients":
         return "\n".join(header + wifi_section)
+    if mode == "leases":
+        return "\n".join(header + lease_section)
     return "\n".join(header + wifi_section + lease_section)

services/queue.py

@@ -2,12 +2,24 @@ import asyncio
 import time
 from collections import deque
 from typing import Awaitable, Callable, Any
+from services import runtime_state
 
 
 _queue: asyncio.Queue = asyncio.Queue()
 _current_label: str | None = None
 _current_meta: dict[str, Any] | None = None
 _pending: deque[tuple[str, float]] = deque()
+_stats: dict[str, Any] = runtime_state.get("queue_stats", {}) or {
+    "processed": 0,
+    "avg_wait_sec": 0.0,
+    "avg_runtime_sec": 0.0,
+    "last_label": "",
+    "last_finished_at": 0.0,
+}
+
+
+def _save_stats():
+    runtime_state.set_state("queue_stats", _stats)
 
 
 async def enqueue(label: str, job: Callable[[], Awaitable[None]]) -> int:
@@ -34,6 +46,21 @@ async def worker():
         try:
             await job()
         finally:
+            finished_at = time.time()
+            if _current_meta:
+                wait_sec = max(0.0, _current_meta["started_at"] - _current_meta["enqueued_at"])
+                runtime_sec = max(0.0, finished_at - _current_meta["started_at"])
+                n_prev = int(_stats.get("processed", 0))
+                _stats["processed"] = n_prev + 1
+                _stats["avg_wait_sec"] = (
+                    (_stats.get("avg_wait_sec", 0.0) * n_prev) + wait_sec
+                ) / _stats["processed"]
+                _stats["avg_runtime_sec"] = (
+                    (_stats.get("avg_runtime_sec", 0.0) * n_prev) + runtime_sec
+                ) / _stats["processed"]
+                _stats["last_label"] = label
+                _stats["last_finished_at"] = finished_at
+                _save_stats()
             _current_label = None
             _current_meta = None
             _queue.task_done()
@@ -47,6 +74,12 @@ def format_status() -> str:
     if pending:
         preview = ", ".join([p[0] for p in pending[:5]])
         lines.append(f"➡️ Next: {preview}")
+    if _stats.get("processed"):
+        lines.append(
+            f"📈 Done: {_stats.get('processed')} | "
+            f"avg wait {int(_stats.get('avg_wait_sec', 0))}s | "
+            f"avg run {int(_stats.get('avg_runtime_sec', 0))}s"
+        )
     return "\n".join(lines)
 
 
@@ -67,4 +100,15 @@ def format_details(limit: int = 10) -> str:
         for i, (label, enqueued_at) in enumerate(pending[:limit], start=1):
             wait = int(now - enqueued_at)
             lines.append(f"{i:>3} | {label} | {wait}s")
+    if _stats.get("processed"):
+        lines.append("")
+        lines.append(
+            "📈 Stats: "
+            f"{_stats.get('processed')} done, "
+            f"avg wait {int(_stats.get('avg_wait_sec', 0))}s, "
+            f"avg run {int(_stats.get('avg_runtime_sec', 0))}s"
+        )
+        last_label = _stats.get("last_label")
+        if last_label:
+            lines.append(f"Last: {last_label}")
     return "\n".join(lines)

services/runtime_state.py

@@ -0,0 +1,52 @@
+import json
+import os
+from typing import Any, Dict
+
+_PATH = "/var/server-bot/runtime.json"
+_STATE: Dict[str, Any] = {}
+
+
+def configure(path: str | None):
+    global _PATH
+    if path:
+        _PATH = path
+
+
+def _load_from_disk():
+    global _STATE
+    if not os.path.exists(_PATH):
+        _STATE = {}
+        return
+    try:
+        with open(_PATH, "r", encoding="utf-8") as f:
+            _STATE = json.load(f)
+    except Exception:
+        _STATE = {}
+
+
+def _save():
+    os.makedirs(os.path.dirname(_PATH), exist_ok=True)
+    try:
+        with open(_PATH, "w", encoding="utf-8") as f:
+            json.dump(_STATE, f)
+    except Exception:
+        pass
+
+
+def get_state() -> Dict[str, Any]:
+    if not _STATE:
+        _load_from_disk()
+    return _STATE
+
+
+def set_state(key: str, value: Any):
+    if not _STATE:
+        _load_from_disk()
+    _STATE[key] = value
+    _save()
+
+
+def get(key: str, default: Any = None) -> Any:
+    if not _STATE:
+        _load_from_disk()
+    return _STATE.get(key, default)