Add runtime state, auto-mute schedules, and backup retries

handlers/backup.py

@@ -63,6 +63,17 @@ def _format_backup_result(rc: int, out: str) -> str:
     return base
 
 
+def _tail(path: str, lines: int = 120) -> str:
+    if not os.path.exists(path):
+        return f"⚠️ Log not found: {path}"
+    try:
+        with open(path, "r", encoding="utf-8", errors="replace") as f:
+            data = f.readlines()[-lines:]
+    except Exception as e:
+        return f"⚠️ Failed to read log: {e}"
+    return "".join(data).strip() or "(empty)"
+
+
 def _load_json(raw: str, label: str) -> tuple[bool, object | None, str]:
     if not raw or not raw.strip():
         return False, None, f"? {label} returned empty output"
@@ -225,6 +236,10 @@ async def cmd_backup_status(msg: Message):
 
 
 async def cmd_backup_now(msg: Message):
+    await schedule_backup(msg)
+
+
+async def schedule_backup(msg: Message):
     async def job():
         if cfg.get("safety", {}).get("dry_run", False):
             await msg.answer("🧪 Dry-run: backup skipped", reply_markup=backup_kb)
@@ -241,7 +256,14 @@ async def cmd_backup_now(msg: Message):
                 use_restic_env=True,
                 timeout=6 * 3600,
             )
-            await msg.answer(_format_backup_result(rc, out), reply_markup=backup_kb)
+            kb = backup_kb
+            if rc != 0:
+                kb = InlineKeyboardMarkup(
+                    inline_keyboard=[
+                        [InlineKeyboardButton(text="🔁 Retry backup", callback_data="backup:retry")]
+                    ]
+                )
+            await msg.answer(_format_backup_result(rc, out), reply_markup=kb)
         finally:
             release_lock("backup")
 
@@ -352,7 +374,7 @@ async def br(msg: Message):
 @dp.message(F.text == "/backup_run")
 async def br_cmd(msg: Message):
     if is_admin_msg(msg):
-        await cmd_backup_now(msg)
+        await schedule_backup(msg)
 
 
 @dp.message(F.text == "🧪 Restic check")
@@ -367,7 +389,14 @@ async def rc(msg: Message):
             use_restic_env=True,
             timeout=6 * 3600,
         )
-        await msg.answer(("✅ OK\n" if rc2 == 0 else "❌ FAIL\n") + out, reply_markup=backup_kb)
+        kb = backup_kb
+        if rc2 != 0:
+            kb = InlineKeyboardMarkup(
+                inline_keyboard=[
+                    [InlineKeyboardButton(text="🔁 Retry restic check", callback_data="backup:retry_check")]
+                ]
+            )
+        await msg.answer(("✅ OK\n" if rc2 == 0 else "❌ FAIL\n") + out, reply_markup=kb)
 
     pos = await enqueue("restic-check", job)
     await msg.answer(f"🕓 Restic check queued (#{pos})", reply_markup=backup_kb)
@@ -395,3 +424,36 @@ async def wr(msg: Message):
 async def rh(msg: Message):
     if is_admin_msg(msg):
         await msg.answer(restore_help(), reply_markup=backup_kb)
+
+
+@dp.message(F.text == "📜 History")
+@dp.message(F.text == "/backup_history")
+async def backup_history(msg: Message):
+    if not is_admin_msg(msg):
+        return
+    log_path = "/var/log/backup-auto.log"
+    content = _tail(log_path, lines=160)
+    if content.startswith("⚠️"):
+        await msg.answer(content, reply_markup=backup_kb)
+        return
+    await msg.answer(
+        f"📜 Backup history (tail)\n`{log_path}`\n```\n{content}\n```",
+        reply_markup=backup_kb,
+        parse_mode="Markdown",
+    )
+
+
+@dp.callback_query(F.data == "backup:retry")
+async def backup_retry(cb: CallbackQuery):
+    if not is_admin_cb(cb):
+        return
+    await cb.answer("Queuing backup…")
+    await schedule_backup(cb.message)
+
+
+@dp.callback_query(F.data == "backup:retry_check")
+async def backup_retry_check(cb: CallbackQuery):
+    if not is_admin_cb(cb):
+        return
+    await cb.answer("Queuing restic check…")
+    await rc(cb.message)