198 lines
6.4 KiB
Python
198 lines
6.4 KiB
Python
import asyncio
|
|
import json
|
|
import socket
|
|
import time
|
|
import psutil
|
|
from aiogram import F
|
|
from aiogram.types import Message
|
|
from app import dp, cfg
|
|
from auth import is_admin_msg
|
|
from keyboards import menu_kb
|
|
from services.system import format_disks
|
|
from services.health import health
|
|
from state import DOCKER_MAP
|
|
from services.runner import run_cmd_full
|
|
|
|
|
|
async def cmd_status(msg: Message):
|
|
now = time.time()
|
|
uptime_sec = int(now - psutil.boot_time())
|
|
|
|
days, rem = divmod(uptime_sec, 86400)
|
|
hours, rem = divmod(rem, 3600)
|
|
minutes, _ = divmod(rem, 60)
|
|
|
|
load1 = psutil.getloadavg()[0]
|
|
load_warn = float(cfg.get("thresholds", {}).get("load_warn", 2.0))
|
|
high_warn = float(cfg.get("thresholds", {}).get("high_load_warn", load_warn * 1.5))
|
|
|
|
cpu_icon = "🟢"
|
|
if load1 > high_warn:
|
|
cpu_icon = "🔴"
|
|
elif load1 > load_warn:
|
|
cpu_icon = "🟡"
|
|
|
|
mem = psutil.virtual_memory()
|
|
cpu_percent = psutil.cpu_percent(interval=None)
|
|
|
|
disks = format_disks()
|
|
net_lines = await _network_snapshot()
|
|
|
|
await msg.answer(
|
|
"📊 **Server status**\n\n"
|
|
f"🖥 **Host:** `{socket.gethostname()}`\n"
|
|
f"⏱ **Uptime:** {days}d {hours}h {minutes}m\n"
|
|
f"{cpu_icon} **Load (1m):** {load1:.2f}\n"
|
|
f"🧮 **CPU:** {cpu_percent:.0f}%\n"
|
|
f"🧠 **RAM:** {mem.used // (1024**3)} / {mem.total // (1024**3)} GiB ({mem.percent}%)\n\n"
|
|
f"{disks}\n\n"
|
|
f"{net_lines}",
|
|
reply_markup=menu_kb,
|
|
parse_mode="Markdown",
|
|
)
|
|
|
|
|
|
async def cmd_health(msg: Message):
|
|
await msg.answer("⏳ Health-check…", reply_markup=menu_kb)
|
|
|
|
async def worker():
|
|
try:
|
|
text = await asyncio.to_thread(health, cfg, DOCKER_MAP)
|
|
except Exception as e:
|
|
await msg.answer(f"❌ Health failed: {type(e).__name__}: {e}", reply_markup=menu_kb)
|
|
return
|
|
await msg.answer(text, reply_markup=menu_kb)
|
|
|
|
asyncio.create_task(worker())
|
|
|
|
|
|
@dp.message(F.text == "🩺 Health")
|
|
async def h(msg: Message):
|
|
if is_admin_msg(msg):
|
|
await cmd_health(msg)
|
|
|
|
|
|
@dp.message(F.text == "📊 Статус")
|
|
async def st(msg: Message):
|
|
if is_admin_msg(msg):
|
|
await cmd_status(msg)
|
|
|
|
|
|
@dp.message(F.text == "/status_short")
|
|
async def st_short(msg: Message):
|
|
if not is_admin_msg(msg):
|
|
return
|
|
now = time.time()
|
|
uptime_sec = int(now - psutil.boot_time())
|
|
days, rem = divmod(uptime_sec, 86400)
|
|
hours, rem = divmod(rem, 3600)
|
|
minutes, _ = divmod(rem, 60)
|
|
load1, load5, load15 = psutil.getloadavg()
|
|
mem = psutil.virtual_memory()
|
|
disks = format_disks().splitlines()
|
|
disk_line = disks[1] if len(disks) > 1 else "Disks: n/a"
|
|
await msg.answer(
|
|
"📋 **Status (short)**\n"
|
|
f"🖥 `{socket.gethostname()}`\n"
|
|
f"⏱ Uptime: {days}d {hours}h {minutes}m\n"
|
|
f"⚙️ Load: {load1:.2f} {load5:.2f} {load15:.2f}\n"
|
|
f"🧠 RAM: {mem.percent}% ({mem.used // (1024**3)} / {mem.total // (1024**3)} GiB)\n"
|
|
f"💾 {disk_line}",
|
|
reply_markup=menu_kb,
|
|
parse_mode="Markdown",
|
|
)
|
|
|
|
|
|
@dp.message(F.text == "/health_short")
|
|
async def health_short(msg: Message):
|
|
if not is_admin_msg(msg):
|
|
return
|
|
try:
|
|
text = await asyncio.to_thread(health, cfg, DOCKER_MAP)
|
|
except Exception as e:
|
|
await msg.answer(f"❌ Health failed: {type(e).__name__}: {e}", reply_markup=menu_kb)
|
|
return
|
|
lines = [ln for ln in text.splitlines() if ln.strip()]
|
|
brief = " | ".join(lines[1:5]) if len(lines) > 1 else text
|
|
await msg.answer(f"🩺 Health (short)\n{brief}", reply_markup=menu_kb)
|
|
|
|
|
|
@dp.message(F.text.in_({"🧪 Self-test", "/selftest"}))
|
|
async def selftest(msg: Message):
|
|
if not is_admin_msg(msg):
|
|
return
|
|
|
|
await msg.answer("⏳ Self-test…", reply_markup=menu_kb)
|
|
|
|
async def worker():
|
|
lines = ["🧪 Self-test"]
|
|
# health
|
|
try:
|
|
htext = await asyncio.to_thread(health, cfg, DOCKER_MAP)
|
|
h_lines = [ln for ln in htext.splitlines() if ln.strip()]
|
|
brief = " | ".join(h_lines[1:5]) if len(h_lines) > 1 else h_lines[0] if h_lines else "n/a"
|
|
lines.append(f"🟢 Health: {brief}")
|
|
except Exception as e:
|
|
lines.append(f"🔴 Health failed: {e}")
|
|
|
|
# restic snapshots check
|
|
rc, out = await run_cmd_full(["restic", "snapshots", "--json"], use_restic_env=True, timeout=40)
|
|
if rc == 0:
|
|
try:
|
|
snaps = json.loads(out)
|
|
if isinstance(snaps, list) and snaps:
|
|
snaps.sort(key=lambda s: s.get("time", ""), reverse=True)
|
|
last = snaps[0]
|
|
t = last.get("time", "?").replace("Z", "").replace("T", " ")[:16]
|
|
lines.append(f"🟢 Restic snapshots: {len(snaps)}, last {t}")
|
|
else:
|
|
lines.append("🟡 Restic snapshots: empty")
|
|
except Exception:
|
|
lines.append("🟡 Restic snapshots: invalid JSON")
|
|
else:
|
|
lines.append(f"🔴 Restic snapshots error: {out.strip() or rc}")
|
|
|
|
await msg.answer("\n".join(lines), reply_markup=menu_kb)
|
|
|
|
asyncio.create_task(worker())
|
|
|
|
|
|
def _rate_str(value: float) -> str:
|
|
if value >= 1024 * 1024:
|
|
return f"{value / (1024 * 1024):.2f} MiB/s"
|
|
if value >= 1024:
|
|
return f"{value / 1024:.1f} KiB/s"
|
|
return f"{value:.0f} B/s"
|
|
|
|
|
|
async def _network_snapshot(interval: float = 1.0) -> str:
|
|
start = psutil.net_io_counters(pernic=True)
|
|
await asyncio.sleep(interval)
|
|
end = psutil.net_io_counters(pernic=True)
|
|
|
|
rows = []
|
|
for nic, s in end.items():
|
|
if nic.startswith("lo"):
|
|
continue
|
|
if not nic.startswith("enp"):
|
|
continue
|
|
e = start.get(nic)
|
|
if not e:
|
|
continue
|
|
rx = max(0, s.bytes_recv - e.bytes_recv)
|
|
tx = max(0, s.bytes_sent - e.bytes_sent)
|
|
err = max(0, (s.errin - e.errin) + (s.errout - e.errout))
|
|
score = rx + tx + (err * 1024)
|
|
rows.append((score, nic, rx, tx, err))
|
|
|
|
rows.sort(reverse=True)
|
|
top = rows[:3]
|
|
if not top:
|
|
return "📡 **Network (1s):** no data"
|
|
|
|
lines = ["📡 **Network (1s):**"]
|
|
for _score, nic, rx, tx, err in top:
|
|
err_part = f", err {err}" if err else ""
|
|
lines.append(f"- {nic}: RX {_rate_str(rx / interval)}, TX {_rate_str(tx / interval)}{err_part}")
|
|
return "\n".join(lines)
|