feat: operational resilience hardening — server blocking, caching, key refresh, async inbox (v2.14.0)

- Server-level blocking: O(1) Redis SISMEMBER check in all inbox listeners,
  admin UI for blocking/unblocking servers by hostname, MongoDB fallback
- Redis caching for collection dispatchers: 300s TTL on followers/following/liked
  counters and paginated pages, one-shot followers recipients cache
- Proactive key refresh: daily cron re-fetches actor documents for followers
  with 7+ day stale keys using lookupWithSecurity()
- Async inbox processing: MongoDB-backed queue with 3s polling, retry (3 attempts),
  24h TTL auto-prune. Follow keeps synchronous Accept, Block keeps synchronous
  follower removal. All other activity types fully deferred to background processor.

Inspired by wafrn's battle-tested multi-user AP implementation.

Confab-Link: http://localhost:8080/sessions/af5f8b45-6b8d-442d-8f25-78c326190709

index.js

@@ -36,6 +36,8 @@ import {
   unmuteController,
   blockController,
   unblockController,
+  blockServerController,
+  unblockServerController,
   moderationController,
   filterModeController,
 } from "./lib/controllers/moderation.js";
@@ -103,6 +105,9 @@ import { startBatchRefollow } from "./lib/batch-refollow.js";
 import { logActivity } from "./lib/activity-log.js";
 import { scheduleCleanup } from "./lib/timeline-cleanup.js";
 import { runSeparateMentionsMigration } from "./lib/migrations/separate-mentions.js";
+import { loadBlockedServersToRedis } from "./lib/storage/server-blocks.js";
+import { scheduleKeyRefresh } from "./lib/key-refresh.js";
+import { startInboxProcessor } from "./lib/inbox-queue.js";
 import { deleteFederationController } from "./lib/controllers/federation-delete.js";
 import {
   federationMgmtController,
@@ -308,6 +313,8 @@ export default class ActivityPubEndpoint {
     router.post("/admin/reader/unmute", unmuteController(mp, this));
     router.post("/admin/reader/block", blockController(mp, this));
     router.post("/admin/reader/unblock", unblockController(mp, this));
+    router.post("/admin/reader/block-server", blockServerController(mp));
+    router.post("/admin/reader/unblock-server", unblockServerController(mp));
     router.get("/admin/followers", followersController(mp));
     router.post("/admin/followers/approve", approveFollowController(mp, this));
     router.post("/admin/followers/reject", rejectFollowController(mp, this));
@@ -1124,6 +1131,12 @@ export default class ActivityPubEndpoint {
     Indiekit.addCollection("ap_reports");
     // Pending follow requests (manual approval)
     Indiekit.addCollection("ap_pending_follows");
+    // Server-level blocks
+    Indiekit.addCollection("ap_blocked_servers");
+    // Key freshness tracking for proactive refresh
+    Indiekit.addCollection("ap_key_freshness");
+    // Async inbox processing queue
+    Indiekit.addCollection("ap_inbox_queue");
 
     // Store collection references (posts resolved lazily)
     const indiekitCollections = Indiekit.collections;
@@ -1151,6 +1164,12 @@ export default class ActivityPubEndpoint {
       ap_reports: indiekitCollections.get("ap_reports"),
       // Pending follow requests (manual approval)
       ap_pending_follows: indiekitCollections.get("ap_pending_follows"),
+      // Server-level blocks
+      ap_blocked_servers: indiekitCollections.get("ap_blocked_servers"),
+      // Key freshness tracking
+      ap_key_freshness: indiekitCollections.get("ap_key_freshness"),
+      // Async inbox processing queue
+      ap_inbox_queue: indiekitCollections.get("ap_inbox_queue"),
       get posts() {
         return indiekitCollections.get("posts");
       },
@@ -1351,6 +1370,27 @@ export default class ActivityPubEndpoint {
         { requestedAt: -1 },
         { background: true },
       );
+      // Server-level blocks
+      this._collections.ap_blocked_servers.createIndex(
+        { hostname: 1 },
+        { unique: true, background: true },
+      );
+      // Key freshness tracking
+      this._collections.ap_key_freshness.createIndex(
+        { actorUrl: 1 },
+        { unique: true, background: true },
+      );
+
+      // Inbox queue indexes
+      this._collections.ap_inbox_queue.createIndex(
+        { status: 1, receivedAt: 1 },
+        { background: true },
+      );
+      // TTL: auto-prune completed items after 24h
+      this._collections.ap_inbox_queue.createIndex(
+        { processedAt: 1 },
+        { expireAfterSeconds: 86_400, background: true },
+      );
     } catch {
       // Index creation failed — collections not yet available.
       // Indexes already exist from previous startups; non-fatal.
@@ -1446,6 +1486,34 @@ export default class ActivityPubEndpoint {
     if (this.options.timelineRetention > 0) {
       scheduleCleanup(this._collections, this.options.timelineRetention);
     }
+
+    // Load server blocks into Redis for fast inbox checks
+    loadBlockedServersToRedis(this._collections).catch((error) => {
+      console.warn("[ActivityPub] Failed to load blocked servers to Redis:", error.message);
+    });
+
+    // Schedule proactive key refresh for stale follower keys (runs on startup + every 24h)
+    const keyRefreshHandle = this.options.actor.handle;
+    const keyRefreshFederation = this._federation;
+    const keyRefreshPubUrl = this._publicationUrl;
+    scheduleKeyRefresh(
+      this._collections,
+      () => keyRefreshFederation?.createContext(new URL(keyRefreshPubUrl), {
+        handle: keyRefreshHandle,
+        publicationUrl: keyRefreshPubUrl,
+      }),
+      keyRefreshHandle,
+    );
+
+    // Start async inbox queue processor (processes one item every 3s)
+    this._inboxProcessorInterval = startInboxProcessor(
+      this._collections,
+      () => this._federation?.createContext(new URL(this._publicationUrl), {
+        handle: this.options.actor.handle,
+        publicationUrl: this._publicationUrl,
+      }),
+      this.options.actor.handle,
+    );
   }
 
   /**