Add AP inbox diagnostics: surface signature errors and request logging
All checks were successful
Deploy Indiekit Server / deploy (push) Successful in 1m20s

- patch-ap-inbox-delivery-debug: two fixes for diagnosing missing inbound
  AP interactions (likes, boosts, replies not appearing in notifications)

  Fix A (federation-setup.js): change ["fedify","federation","inbox"] log
  category from lowestLevel "fatal" → "error" so HTTP Signature verification
  failures are now visible in server logs instead of being silently swallowed.
  The original "fatal" level was hiding real delivery rejections (401s) that
  cause remote servers to stop retrying.

  Fix B (federation-bridge.js): add a pre-signature-check console.info for
  every inbox POST when AP_DEBUG=1 or AP_LOG_LEVEL=debug. Confirms whether
  remote servers are reaching our inbox at all (nginx/routing check).

- memory/project_activitypub.md: document full inbound activity pipeline,
  _publicationUrl dependency, body buffering, and how to use new diagnostics

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Sven
2026-04-01 16:32:06 +02:00
parent e791c06b79
commit 8b1b5d990a
3 changed files with 155 additions and 2 deletions

View File

@@ -105,6 +105,43 @@ would fail `findTimelineItemById` → `inReplyTo = null` → no `in-reply-to` in
The `"reply"` post type in `indiekit.config.mjs` has no `discovery` field — standard PTD spec applies.
## Inbound AP Activity Pipeline
Activities from remote servers follow this path:
```
Remote server → nginx → Express (body buffered in createFedifyMiddleware)
→ Fedify signature check (uses req._rawBody for digest)
→ Fedify Redis message queue (if Redis configured)
→ Fedify queue worker → inbox listener (inbox-listeners.js)
→ enqueueActivity() → ap_inbox_queue (MongoDB)
→ startInboxProcessor() (1s poll) → routeToHandler()
→ handleLike / handleAnnounce / handleCreate
→ addNotification() → ap_notifications
```
**Critical: `collections._publicationUrl`** is set in `index.js` (`_publicationUrl: this._publicationUrl`)
AND by `patch-ap-inbox-publication-url` in `federation-setup.js`. Both set `"https://blog.giersig.eu/"`.
Notification conditions gate on `pubUrl && objectId.startsWith(pubUrl)`:
- `handleLike`: only notifies for likes of our own content
- `handleAnnounce` PATH 1: only notifies for boosts of our content
- `handleCreate`: only notifies for replies to our posts (`inReplyTo.startsWith(pubUrl)`)
**Body buffering** (`createFedifyMiddleware`): `application/activity+json` bodies are buffered
into `req._rawBody` before `express.json()` (which only handles `application/json`) touches them.
`fromExpressRequest` passes `req._rawBody` verbatim to the Fedify `Request` object so the
HTTP Signature Digest check passes.
**Fedify inbox log suppression**: `["fedify","federation","inbox"]` was hardcoded to `"fatal"`
(`patch-ap-inbox-delivery-debug` fixes this to `"error"` so real failures are visible).
**Diagnosing inbox delivery issues:**
- Set `AP_DEBUG=1` → logs `[AP-inbox] POST /activitypub/users/svemagie/inbox ct=... body=...B`
BEFORE Fedify's signature check. If this doesn't appear, activities aren't reaching our server.
- With inbox log level now `"error"`: signature failures show as Fedify error logs.
- Queue processing failures: `[inbox-queue] Failed processing ...` — always logged.
## detectProtocol() in Microsub Reader
`detectProtocol(url)` in `reader.js` classifies URLs for syndication auto-selection:

View File

@@ -5,8 +5,8 @@
"main": "index.js",
"scripts": {
"preinstall": "node scripts/setup-gitea-url-rewrite.mjs",
"postinstall": "xattr -w com.apple.fileprovider.ignore#P 1 node_modules 2>/dev/null || true && node scripts/patch-lightningcss.mjs && node scripts/patch-endpoint-media-scope.mjs && node scripts/patch-endpoint-media-sharp-runtime.mjs && node scripts/patch-frontend-sharp-runtime.mjs && node scripts/patch-endpoint-files-upload-route.mjs && node scripts/patch-endpoint-files-upload-locales.mjs && node scripts/patch-endpoint-activitypub-locales.mjs && node scripts/patch-endpoint-homepage-locales.mjs && node scripts/patch-endpoint-homepage-identity-defaults.mjs && node scripts/patch-federation-unlisted-guards.mjs && node scripts/patch-endpoint-micropub-where-note-visibility.mjs && node scripts/patch-endpoint-podroll-opml-upload.mjs && node scripts/patch-frontend-serviceworker-file.mjs && node scripts/patch-endpoint-comments-locales.mjs && node scripts/patch-endpoint-posts-locales.mjs && node scripts/patch-endpoint-conversations-locales.mjs && node scripts/patch-conversations-collection-guards.mjs && node scripts/patch-indiekit-routes-rate-limits.mjs && node scripts/patch-indiekit-error-production-stack.mjs && node scripts/patch-indieauth-devmode-guard.mjs && node scripts/patch-listening-endpoint-runtime-guards.mjs && node scripts/patch-endpoint-github-changelog-categories.mjs && node scripts/patch-endpoint-github-contributions-log.mjs && node scripts/patch-store-github-error-message.mjs && node scripts/patch-store-github-update-fallback.mjs && node scripts/patch-store-github-gitea-methods.mjs && node scripts/patch-store-github-content-type.mjs && node scripts/patch-endpoint-blogroll-feeds-alias.mjs && node scripts/patch-endpoint-posts-uid-lookup.mjs && node scripts/patch-conversations-bluesky-self-filter.mjs && node scripts/patch-conversations-bluesky-cursor-fix.mjs && node scripts/patch-endpoint-micropub-source-filter.mjs && node scripts/patch-syndicate-force-checked-default.mjs && node scripts/patch-syndicate-normalize-syndication-array.mjs && node scripts/patch-endpoint-posts-fetch-diagnostic.mjs && node scripts/patch-micropub-fetch-internal-url.mjs && node scripts/patch-micropub-session-token.mjs && node scripts/patch-indiekit-endpoint-urls-protocol.mjs && node scripts/patch-webmention-sender-hentry-syntax.mjs && node scripts/patch-webmention-sender-retry.mjs && node scripts/patch-webmention-sender-livefetch.mjs && node scripts/patch-webmention-sender-empty-details.mjs && node scripts/patch-bluesky-syndicator-internal-url.mjs && node scripts/patch-bluesky-syndicator-media-type-guard.mjs && node scripts/patch-ap-skip-draft-syndication.mjs && node scripts/patch-ap-og-image.mjs && node scripts/patch-ap-webfinger-before-auth.mjs && node scripts/patch-ap-federation-bridge-base-url.mjs && node scripts/patch-ap-compose-default-checked.mjs && node scripts/patch-ap-mastodon-reply-threading.mjs && node scripts/patch-ap-mastodon-status-id.mjs && node scripts/patch-ap-interactions-send-guard.mjs && node scripts/patch-ap-syndicate-dedup.mjs && node scripts/patch-ap-mastodon-delete-fix.mjs && node scripts/patch-ap-status-reply-id.mjs && node scripts/patch-ap-inbox-publication-url.mjs && node scripts/patch-bluesky-syndicator-delete.mjs && node scripts/patch-micropub-delete-propagation.mjs && node scripts/patch-micropub-gitea-dispatch-conditional.mjs",
"serve": "export NODE_ENV=${NODE_ENV:-production} INDIEKIT_DEBUG=${INDIEKIT_DEBUG:-0} && node scripts/preflight-production-security.mjs && node scripts/preflight-mongo-connection.mjs && node scripts/preflight-activitypub-rsa-key.mjs && node scripts/preflight-activitypub-profile-urls.mjs && node scripts/patch-lightningcss.mjs && node scripts/patch-endpoint-media-scope.mjs && node scripts/patch-endpoint-media-sharp-runtime.mjs && node scripts/patch-frontend-sharp-runtime.mjs && node scripts/patch-endpoint-files-upload-route.mjs && node scripts/patch-endpoint-files-upload-locales.mjs && node scripts/patch-endpoint-activitypub-locales.mjs && node scripts/patch-endpoint-homepage-locales.mjs && node scripts/patch-endpoint-homepage-identity-defaults.mjs && node scripts/patch-federation-unlisted-guards.mjs && node scripts/patch-endpoint-micropub-where-note-visibility.mjs && node scripts/patch-endpoint-podroll-opml-upload.mjs && node scripts/patch-frontend-serviceworker-file.mjs && node scripts/patch-endpoint-comments-locales.mjs && node scripts/patch-endpoint-posts-locales.mjs && node scripts/patch-endpoint-conversations-locales.mjs && node scripts/patch-conversations-collection-guards.mjs && node scripts/patch-indiekit-routes-rate-limits.mjs && node scripts/patch-indiekit-error-production-stack.mjs && node scripts/patch-indieauth-devmode-guard.mjs && node scripts/patch-listening-endpoint-runtime-guards.mjs && node scripts/patch-endpoint-github-changelog-categories.mjs && node scripts/patch-endpoint-github-contributions-log.mjs && node scripts/patch-store-github-error-message.mjs && node scripts/patch-store-github-update-fallback.mjs && node scripts/patch-store-github-gitea-methods.mjs && node scripts/patch-store-github-content-type.mjs && node scripts/patch-microsub-reader-ap-dispatch.mjs && node scripts/patch-microsub-compose-draft-guard.mjs && node scripts/patch-endpoint-blogroll-feeds-alias.mjs && node scripts/patch-endpoint-posts-uid-lookup.mjs && node scripts/patch-conversations-bluesky-self-filter.mjs && node scripts/patch-conversations-bluesky-cursor-fix.mjs && node scripts/patch-endpoint-micropub-source-filter.mjs && node scripts/patch-syndicate-force-checked-default.mjs && node scripts/patch-syndicate-normalize-syndication-array.mjs && node scripts/patch-endpoint-posts-fetch-diagnostic.mjs && node scripts/patch-micropub-fetch-internal-url.mjs && node scripts/patch-micropub-session-token.mjs && node scripts/patch-indiekit-endpoint-urls-protocol.mjs && node scripts/patch-webmention-sender-hentry-syntax.mjs && node scripts/patch-webmention-sender-retry.mjs && node scripts/patch-webmention-sender-livefetch.mjs && node scripts/patch-webmention-sender-empty-details.mjs && node scripts/patch-bluesky-syndicator-internal-url.mjs && node scripts/patch-bluesky-syndicator-media-type-guard.mjs && node scripts/patch-ap-skip-draft-syndication.mjs && node scripts/patch-ap-og-image.mjs && node scripts/patch-ap-webfinger-before-auth.mjs && node scripts/patch-ap-federation-bridge-base-url.mjs && node scripts/patch-ap-compose-default-checked.mjs && node scripts/patch-ap-mastodon-reply-threading.mjs && node scripts/patch-ap-mastodon-status-id.mjs && node scripts/patch-ap-interactions-send-guard.mjs && node scripts/patch-ap-syndicate-dedup.mjs && node scripts/patch-ap-mastodon-delete-fix.mjs && node scripts/patch-ap-status-reply-id.mjs && node scripts/patch-ap-inbox-publication-url.mjs && node scripts/patch-bluesky-syndicator-delete.mjs && node scripts/patch-micropub-delete-propagation.mjs && node --require ./metrics-shim.cjs node_modules/@indiekit/indiekit/bin/cli.js serve --config indiekit.config.mjs",
"postinstall": "xattr -w com.apple.fileprovider.ignore#P 1 node_modules 2>/dev/null || true && node scripts/patch-lightningcss.mjs && node scripts/patch-endpoint-media-scope.mjs && node scripts/patch-endpoint-media-sharp-runtime.mjs && node scripts/patch-frontend-sharp-runtime.mjs && node scripts/patch-endpoint-files-upload-route.mjs && node scripts/patch-endpoint-files-upload-locales.mjs && node scripts/patch-endpoint-activitypub-locales.mjs && node scripts/patch-endpoint-homepage-locales.mjs && node scripts/patch-endpoint-homepage-identity-defaults.mjs && node scripts/patch-federation-unlisted-guards.mjs && node scripts/patch-endpoint-micropub-where-note-visibility.mjs && node scripts/patch-endpoint-podroll-opml-upload.mjs && node scripts/patch-frontend-serviceworker-file.mjs && node scripts/patch-endpoint-comments-locales.mjs && node scripts/patch-endpoint-posts-locales.mjs && node scripts/patch-endpoint-conversations-locales.mjs && node scripts/patch-conversations-collection-guards.mjs && node scripts/patch-indiekit-routes-rate-limits.mjs && node scripts/patch-indiekit-error-production-stack.mjs && node scripts/patch-indieauth-devmode-guard.mjs && node scripts/patch-listening-endpoint-runtime-guards.mjs && node scripts/patch-endpoint-github-changelog-categories.mjs && node scripts/patch-endpoint-github-contributions-log.mjs && node scripts/patch-store-github-error-message.mjs && node scripts/patch-store-github-update-fallback.mjs && node scripts/patch-store-github-gitea-methods.mjs && node scripts/patch-store-github-content-type.mjs && node scripts/patch-endpoint-blogroll-feeds-alias.mjs && node scripts/patch-endpoint-posts-uid-lookup.mjs && node scripts/patch-conversations-bluesky-self-filter.mjs && node scripts/patch-conversations-bluesky-cursor-fix.mjs && node scripts/patch-endpoint-micropub-source-filter.mjs && node scripts/patch-syndicate-force-checked-default.mjs && node scripts/patch-syndicate-normalize-syndication-array.mjs && node scripts/patch-endpoint-posts-fetch-diagnostic.mjs && node scripts/patch-micropub-fetch-internal-url.mjs && node scripts/patch-micropub-session-token.mjs && node scripts/patch-indiekit-endpoint-urls-protocol.mjs && node scripts/patch-webmention-sender-hentry-syntax.mjs && node scripts/patch-webmention-sender-retry.mjs && node scripts/patch-webmention-sender-livefetch.mjs && node scripts/patch-webmention-sender-empty-details.mjs && node scripts/patch-bluesky-syndicator-internal-url.mjs && node scripts/patch-bluesky-syndicator-media-type-guard.mjs && node scripts/patch-ap-skip-draft-syndication.mjs && node scripts/patch-ap-og-image.mjs && node scripts/patch-ap-webfinger-before-auth.mjs && node scripts/patch-ap-federation-bridge-base-url.mjs && node scripts/patch-ap-compose-default-checked.mjs && node scripts/patch-ap-mastodon-reply-threading.mjs && node scripts/patch-ap-mastodon-status-id.mjs && node scripts/patch-ap-interactions-send-guard.mjs && node scripts/patch-ap-syndicate-dedup.mjs && node scripts/patch-ap-mastodon-delete-fix.mjs && node scripts/patch-ap-status-reply-id.mjs && node scripts/patch-ap-inbox-publication-url.mjs && node scripts/patch-bluesky-syndicator-delete.mjs && node scripts/patch-micropub-delete-propagation.mjs && node scripts/patch-micropub-gitea-dispatch-conditional.mjs && node scripts/patch-ap-inbox-delivery-debug.mjs",
"serve": "export NODE_ENV=${NODE_ENV:-production} INDIEKIT_DEBUG=${INDIEKIT_DEBUG:-0} && node scripts/preflight-production-security.mjs && node scripts/preflight-mongo-connection.mjs && node scripts/preflight-activitypub-rsa-key.mjs && node scripts/preflight-activitypub-profile-urls.mjs && node scripts/patch-lightningcss.mjs && node scripts/patch-endpoint-media-scope.mjs && node scripts/patch-endpoint-media-sharp-runtime.mjs && node scripts/patch-frontend-sharp-runtime.mjs && node scripts/patch-endpoint-files-upload-route.mjs && node scripts/patch-endpoint-files-upload-locales.mjs && node scripts/patch-endpoint-activitypub-locales.mjs && node scripts/patch-endpoint-homepage-locales.mjs && node scripts/patch-endpoint-homepage-identity-defaults.mjs && node scripts/patch-federation-unlisted-guards.mjs && node scripts/patch-endpoint-micropub-where-note-visibility.mjs && node scripts/patch-endpoint-podroll-opml-upload.mjs && node scripts/patch-frontend-serviceworker-file.mjs && node scripts/patch-endpoint-comments-locales.mjs && node scripts/patch-endpoint-posts-locales.mjs && node scripts/patch-endpoint-conversations-locales.mjs && node scripts/patch-conversations-collection-guards.mjs && node scripts/patch-indiekit-routes-rate-limits.mjs && node scripts/patch-indiekit-error-production-stack.mjs && node scripts/patch-indieauth-devmode-guard.mjs && node scripts/patch-listening-endpoint-runtime-guards.mjs && node scripts/patch-endpoint-github-changelog-categories.mjs && node scripts/patch-endpoint-github-contributions-log.mjs && node scripts/patch-store-github-error-message.mjs && node scripts/patch-store-github-update-fallback.mjs && node scripts/patch-store-github-gitea-methods.mjs && node scripts/patch-store-github-content-type.mjs && node scripts/patch-microsub-reader-ap-dispatch.mjs && node scripts/patch-microsub-compose-draft-guard.mjs && node scripts/patch-endpoint-blogroll-feeds-alias.mjs && node scripts/patch-endpoint-posts-uid-lookup.mjs && node scripts/patch-conversations-bluesky-self-filter.mjs && node scripts/patch-conversations-bluesky-cursor-fix.mjs && node scripts/patch-endpoint-micropub-source-filter.mjs && node scripts/patch-syndicate-force-checked-default.mjs && node scripts/patch-syndicate-normalize-syndication-array.mjs && node scripts/patch-endpoint-posts-fetch-diagnostic.mjs && node scripts/patch-micropub-fetch-internal-url.mjs && node scripts/patch-micropub-session-token.mjs && node scripts/patch-indiekit-endpoint-urls-protocol.mjs && node scripts/patch-webmention-sender-hentry-syntax.mjs && node scripts/patch-webmention-sender-retry.mjs && node scripts/patch-webmention-sender-livefetch.mjs && node scripts/patch-webmention-sender-empty-details.mjs && node scripts/patch-bluesky-syndicator-internal-url.mjs && node scripts/patch-bluesky-syndicator-media-type-guard.mjs && node scripts/patch-ap-skip-draft-syndication.mjs && node scripts/patch-ap-og-image.mjs && node scripts/patch-ap-webfinger-before-auth.mjs && node scripts/patch-ap-federation-bridge-base-url.mjs && node scripts/patch-ap-compose-default-checked.mjs && node scripts/patch-ap-mastodon-reply-threading.mjs && node scripts/patch-ap-mastodon-status-id.mjs && node scripts/patch-ap-interactions-send-guard.mjs && node scripts/patch-ap-syndicate-dedup.mjs && node scripts/patch-ap-mastodon-delete-fix.mjs && node scripts/patch-ap-status-reply-id.mjs && node scripts/patch-ap-inbox-publication-url.mjs && node scripts/patch-bluesky-syndicator-delete.mjs && node scripts/patch-micropub-delete-propagation.mjs && node scripts/patch-ap-inbox-delivery-debug.mjs && node --require ./metrics-shim.cjs node_modules/@indiekit/indiekit/bin/cli.js serve --config indiekit.config.mjs",
"test": "echo \"Error: no test specified\" && exit 1"
},
"keywords": [],

View File

@@ -0,0 +1,116 @@
/**
* Patch: add inbox delivery diagnostics.
*
* Problems:
* 1. The ["fedify","federation","inbox"] LogTape category is hardcoded to
* lowestLevel "fatal", hiding HTTP Signature verification failures (401s).
* Remote servers that receive 401s stop retrying → activities are lost.
* 2. No request-level logging for incoming inbox POSTs, so we can't tell
* whether remote servers are even attempting delivery.
*
* Fix A (federation-setup.js):
* Change inbox log category from "fatal" → "error".
* Real verification failures (wrong key, clock skew, digest mismatch) surface.
* High-volume 404/410 key-fetch warnings from deleted actors stay silent.
*
* Fix B (federation-bridge.js):
* Add a console.info before fromExpressRequest() that logs every POST to an
* inbox path (path + content-type + raw body length). Fires BEFORE Fedify's
* signature check, confirming whether remote servers reach our inbox at all.
* Guarded by AP_LOG_LEVEL=debug or AP_DEBUG=1 to keep production logs quiet.
*/
import { access, readFile, writeFile } from "node:fs/promises";
const MARKER_A = "// [patch] ap-inbox-delivery-debug-A";
const MARKER_B = "// [patch] ap-inbox-delivery-debug-B";
// ── Fix A: federation-setup.js — inbox logger level ──────────────────────────
const setupCandidates = [
"node_modules/@rmdes/indiekit-endpoint-activitypub/lib/federation-setup.js",
"node_modules/@indiekit/indiekit/node_modules/@rmdes/indiekit-endpoint-activitypub/lib/federation-setup.js",
];
const OLD_INBOX_LOGGER = ` {
// Noise guard: HTTP Signature verification failures are expected for
// incoming activities from servers with expired/gone keys (e.g. deleted
// actors, migrated servers). These produce high log volume with no
// actionable signal — suppress everything below fatal.
category: ["fedify", "federation", "inbox"],
sinks: ["console"],
lowestLevel: "fatal",
},`;
const NEW_INBOX_LOGGER = ` {
// Surfacing real verification failures (wrong key, clock skew, digest
// mismatch) at "error" level while keeping high-volume key-fetch
// 404/410 warnings from deleted actors silent. ${MARKER_A}
category: ["fedify", "federation", "inbox"],
sinks: ["console"],
lowestLevel: "error",
},`;
// ── Fix B: federation-bridge.js — request-level inbox logging ────────────────
const bridgeCandidates = [
"node_modules/@rmdes/indiekit-endpoint-activitypub/lib/federation-bridge.js",
"node_modules/@indiekit/indiekit/node_modules/@rmdes/indiekit-endpoint-activitypub/lib/federation-bridge.js",
];
// Insert a debug log right before "const request = fromExpressRequest(req, publicationUrl);"
// This is patched by ap-base-url already so that comment marker is present.
const OLD_BRIDGE_REQUEST = ` const request = fromExpressRequest(req, publicationUrl); // ap-base-url patch`;
const NEW_BRIDGE_REQUEST = ` // Log incoming inbox POSTs before Fedify signature check. ${MARKER_B}
// Enabled by AP_LOG_LEVEL=debug or AP_DEBUG=1.
if (
(process.env.AP_LOG_LEVEL === "debug" || process.env.AP_DEBUG === "1") &&
req.method === "POST" &&
(req.path.includes("/inbox") || req.path.includes("/users/"))
) {
const _bct = (req.headers["content-type"] || "").split(";")[0].trim();
const _bsz = req._rawBody?.length ?? (req.body ? "pre-parsed" : "none");
console.info(\`[AP-inbox] POST \${req.path} ct=\${_bct} body=\${_bsz}B\`);
}
const request = fromExpressRequest(req, publicationUrl); // ap-base-url patch`;
async function exists(p) {
try { await access(p); return true; } catch { return false; }
}
async function applyPatch(candidates, oldSnippet, newSnippet, label, marker) {
let checked = 0;
let patched = 0;
for (const filePath of candidates) {
if (!(await exists(filePath))) continue;
checked++;
const source = await readFile(filePath, "utf8");
if (source.includes(marker)) {
console.log(`[postinstall] patch-ap-inbox-delivery-debug: ${label} already applied to ${filePath}`);
continue;
}
if (!source.includes(oldSnippet)) {
console.warn(`[postinstall] patch-ap-inbox-delivery-debug: ${label} snippet not found in ${filePath}`);
continue;
}
await writeFile(filePath, source.replace(oldSnippet, newSnippet), "utf8");
patched++;
console.log(`[postinstall] Applied patch-ap-inbox-delivery-debug (${label}) to ${filePath}`);
}
return { checked, patched };
}
const a = await applyPatch(setupCandidates, OLD_INBOX_LOGGER, NEW_INBOX_LOGGER, "inbox-logger-level", MARKER_A);
const b = await applyPatch(bridgeCandidates, OLD_BRIDGE_REQUEST, NEW_BRIDGE_REQUEST, "bridge-request-log", MARKER_B);
const total = a.checked + b.checked;
const totalPatched = a.patched + b.patched;
if (total === 0) {
console.log("[postinstall] patch-ap-inbox-delivery-debug: no target files found");
} else if (totalPatched === 0) {
console.log("[postinstall] patch-ap-inbox-delivery-debug: already up to date");
} else {
console.log(`[postinstall] patch-ap-inbox-delivery-debug: patched ${totalPatched} file(s)`);
}