diff --git a/memory/project_activitypub.md b/memory/project_activitypub.md index 2b883602..26dac44d 100644 --- a/memory/project_activitypub.md +++ b/memory/project_activitypub.md @@ -105,6 +105,43 @@ would fail `findTimelineItemById` → `inReplyTo = null` → no `in-reply-to` in The `"reply"` post type in `indiekit.config.mjs` has no `discovery` field — standard PTD spec applies. +## Inbound AP Activity Pipeline + +Activities from remote servers follow this path: + +``` +Remote server → nginx → Express (body buffered in createFedifyMiddleware) + → Fedify signature check (uses req._rawBody for digest) + → Fedify Redis message queue (if Redis configured) + → Fedify queue worker → inbox listener (inbox-listeners.js) + → enqueueActivity() → ap_inbox_queue (MongoDB) + → startInboxProcessor() (1s poll) → routeToHandler() + → handleLike / handleAnnounce / handleCreate + → addNotification() → ap_notifications +``` + +**Critical: `collections._publicationUrl`** is set in `index.js` (`_publicationUrl: this._publicationUrl`) +AND by `patch-ap-inbox-publication-url` in `federation-setup.js`. Both set `"https://blog.giersig.eu/"`. + +Notification conditions gate on `pubUrl && objectId.startsWith(pubUrl)`: +- `handleLike`: only notifies for likes of our own content +- `handleAnnounce` PATH 1: only notifies for boosts of our content +- `handleCreate`: only notifies for replies to our posts (`inReplyTo.startsWith(pubUrl)`) + +**Body buffering** (`createFedifyMiddleware`): `application/activity+json` bodies are buffered +into `req._rawBody` before `express.json()` (which only handles `application/json`) touches them. +`fromExpressRequest` passes `req._rawBody` verbatim to the Fedify `Request` object so the +HTTP Signature Digest check passes. + +**Fedify inbox log suppression**: `["fedify","federation","inbox"]` was hardcoded to `"fatal"` +(`patch-ap-inbox-delivery-debug` fixes this to `"error"` so real failures are visible). + +**Diagnosing inbox delivery issues:** +- Set `AP_DEBUG=1` → logs `[AP-inbox] POST /activitypub/users/svemagie/inbox ct=... body=...B` + BEFORE Fedify's signature check. If this doesn't appear, activities aren't reaching our server. +- With inbox log level now `"error"`: signature failures show as Fedify error logs. +- Queue processing failures: `[inbox-queue] Failed processing ...` — always logged. + ## detectProtocol() in Microsub Reader `detectProtocol(url)` in `reader.js` classifies URLs for syndication auto-selection: diff --git a/package.json b/package.json index d1e35a25..d5b3e383 100644 --- a/package.json +++ b/package.json @@ -5,8 +5,8 @@ "main": "index.js", "scripts": { "preinstall": "node scripts/setup-gitea-url-rewrite.mjs", - "postinstall": "xattr -w com.apple.fileprovider.ignore#P 1 node_modules 2>/dev/null || true && node scripts/patch-lightningcss.mjs && node scripts/patch-endpoint-media-scope.mjs && node scripts/patch-endpoint-media-sharp-runtime.mjs && node scripts/patch-frontend-sharp-runtime.mjs && node scripts/patch-endpoint-files-upload-route.mjs && node scripts/patch-endpoint-files-upload-locales.mjs && node scripts/patch-endpoint-activitypub-locales.mjs && node scripts/patch-endpoint-homepage-locales.mjs && node scripts/patch-endpoint-homepage-identity-defaults.mjs && node scripts/patch-federation-unlisted-guards.mjs && node scripts/patch-endpoint-micropub-where-note-visibility.mjs && node scripts/patch-endpoint-podroll-opml-upload.mjs && node scripts/patch-frontend-serviceworker-file.mjs && node scripts/patch-endpoint-comments-locales.mjs && node scripts/patch-endpoint-posts-locales.mjs && node scripts/patch-endpoint-conversations-locales.mjs && node scripts/patch-conversations-collection-guards.mjs && node scripts/patch-indiekit-routes-rate-limits.mjs && node scripts/patch-indiekit-error-production-stack.mjs && node scripts/patch-indieauth-devmode-guard.mjs && node scripts/patch-listening-endpoint-runtime-guards.mjs && node scripts/patch-endpoint-github-changelog-categories.mjs && node scripts/patch-endpoint-github-contributions-log.mjs && node scripts/patch-store-github-error-message.mjs && node scripts/patch-store-github-update-fallback.mjs && node scripts/patch-store-github-gitea-methods.mjs && node scripts/patch-store-github-content-type.mjs && node scripts/patch-endpoint-blogroll-feeds-alias.mjs && node scripts/patch-endpoint-posts-uid-lookup.mjs && node scripts/patch-conversations-bluesky-self-filter.mjs && node scripts/patch-conversations-bluesky-cursor-fix.mjs && node scripts/patch-endpoint-micropub-source-filter.mjs && node scripts/patch-syndicate-force-checked-default.mjs && node scripts/patch-syndicate-normalize-syndication-array.mjs && node scripts/patch-endpoint-posts-fetch-diagnostic.mjs && node scripts/patch-micropub-fetch-internal-url.mjs && node scripts/patch-micropub-session-token.mjs && node scripts/patch-indiekit-endpoint-urls-protocol.mjs && node scripts/patch-webmention-sender-hentry-syntax.mjs && node scripts/patch-webmention-sender-retry.mjs && node scripts/patch-webmention-sender-livefetch.mjs && node scripts/patch-webmention-sender-empty-details.mjs && node scripts/patch-bluesky-syndicator-internal-url.mjs && node scripts/patch-bluesky-syndicator-media-type-guard.mjs && node scripts/patch-ap-skip-draft-syndication.mjs && node scripts/patch-ap-og-image.mjs && node scripts/patch-ap-webfinger-before-auth.mjs && node scripts/patch-ap-federation-bridge-base-url.mjs && node scripts/patch-ap-compose-default-checked.mjs && node scripts/patch-ap-mastodon-reply-threading.mjs && node scripts/patch-ap-mastodon-status-id.mjs && node scripts/patch-ap-interactions-send-guard.mjs && node scripts/patch-ap-syndicate-dedup.mjs && node scripts/patch-ap-mastodon-delete-fix.mjs && node scripts/patch-ap-status-reply-id.mjs && node scripts/patch-ap-inbox-publication-url.mjs && node scripts/patch-bluesky-syndicator-delete.mjs && node scripts/patch-micropub-delete-propagation.mjs && node scripts/patch-micropub-gitea-dispatch-conditional.mjs", - "serve": "export NODE_ENV=${NODE_ENV:-production} INDIEKIT_DEBUG=${INDIEKIT_DEBUG:-0} && node scripts/preflight-production-security.mjs && node scripts/preflight-mongo-connection.mjs && node scripts/preflight-activitypub-rsa-key.mjs && node scripts/preflight-activitypub-profile-urls.mjs && node scripts/patch-lightningcss.mjs && node scripts/patch-endpoint-media-scope.mjs && node scripts/patch-endpoint-media-sharp-runtime.mjs && node scripts/patch-frontend-sharp-runtime.mjs && node scripts/patch-endpoint-files-upload-route.mjs && node scripts/patch-endpoint-files-upload-locales.mjs && node scripts/patch-endpoint-activitypub-locales.mjs && node scripts/patch-endpoint-homepage-locales.mjs && node scripts/patch-endpoint-homepage-identity-defaults.mjs && node scripts/patch-federation-unlisted-guards.mjs && node scripts/patch-endpoint-micropub-where-note-visibility.mjs && node scripts/patch-endpoint-podroll-opml-upload.mjs && node scripts/patch-frontend-serviceworker-file.mjs && node scripts/patch-endpoint-comments-locales.mjs && node scripts/patch-endpoint-posts-locales.mjs && node scripts/patch-endpoint-conversations-locales.mjs && node scripts/patch-conversations-collection-guards.mjs && node scripts/patch-indiekit-routes-rate-limits.mjs && node scripts/patch-indiekit-error-production-stack.mjs && node scripts/patch-indieauth-devmode-guard.mjs && node scripts/patch-listening-endpoint-runtime-guards.mjs && node scripts/patch-endpoint-github-changelog-categories.mjs && node scripts/patch-endpoint-github-contributions-log.mjs && node scripts/patch-store-github-error-message.mjs && node scripts/patch-store-github-update-fallback.mjs && node scripts/patch-store-github-gitea-methods.mjs && node scripts/patch-store-github-content-type.mjs && node scripts/patch-microsub-reader-ap-dispatch.mjs && node scripts/patch-microsub-compose-draft-guard.mjs && node scripts/patch-endpoint-blogroll-feeds-alias.mjs && node scripts/patch-endpoint-posts-uid-lookup.mjs && node scripts/patch-conversations-bluesky-self-filter.mjs && node scripts/patch-conversations-bluesky-cursor-fix.mjs && node scripts/patch-endpoint-micropub-source-filter.mjs && node scripts/patch-syndicate-force-checked-default.mjs && node scripts/patch-syndicate-normalize-syndication-array.mjs && node scripts/patch-endpoint-posts-fetch-diagnostic.mjs && node scripts/patch-micropub-fetch-internal-url.mjs && node scripts/patch-micropub-session-token.mjs && node scripts/patch-indiekit-endpoint-urls-protocol.mjs && node scripts/patch-webmention-sender-hentry-syntax.mjs && node scripts/patch-webmention-sender-retry.mjs && node scripts/patch-webmention-sender-livefetch.mjs && node scripts/patch-webmention-sender-empty-details.mjs && node scripts/patch-bluesky-syndicator-internal-url.mjs && node scripts/patch-bluesky-syndicator-media-type-guard.mjs && node scripts/patch-ap-skip-draft-syndication.mjs && node scripts/patch-ap-og-image.mjs && node scripts/patch-ap-webfinger-before-auth.mjs && node scripts/patch-ap-federation-bridge-base-url.mjs && node scripts/patch-ap-compose-default-checked.mjs && node scripts/patch-ap-mastodon-reply-threading.mjs && node scripts/patch-ap-mastodon-status-id.mjs && node scripts/patch-ap-interactions-send-guard.mjs && node scripts/patch-ap-syndicate-dedup.mjs && node scripts/patch-ap-mastodon-delete-fix.mjs && node scripts/patch-ap-status-reply-id.mjs && node scripts/patch-ap-inbox-publication-url.mjs && node scripts/patch-bluesky-syndicator-delete.mjs && node scripts/patch-micropub-delete-propagation.mjs && node --require ./metrics-shim.cjs node_modules/@indiekit/indiekit/bin/cli.js serve --config indiekit.config.mjs", + "postinstall": "xattr -w com.apple.fileprovider.ignore#P 1 node_modules 2>/dev/null || true && node scripts/patch-lightningcss.mjs && node scripts/patch-endpoint-media-scope.mjs && node scripts/patch-endpoint-media-sharp-runtime.mjs && node scripts/patch-frontend-sharp-runtime.mjs && node scripts/patch-endpoint-files-upload-route.mjs && node scripts/patch-endpoint-files-upload-locales.mjs && node scripts/patch-endpoint-activitypub-locales.mjs && node scripts/patch-endpoint-homepage-locales.mjs && node scripts/patch-endpoint-homepage-identity-defaults.mjs && node scripts/patch-federation-unlisted-guards.mjs && node scripts/patch-endpoint-micropub-where-note-visibility.mjs && node scripts/patch-endpoint-podroll-opml-upload.mjs && node scripts/patch-frontend-serviceworker-file.mjs && node scripts/patch-endpoint-comments-locales.mjs && node scripts/patch-endpoint-posts-locales.mjs && node scripts/patch-endpoint-conversations-locales.mjs && node scripts/patch-conversations-collection-guards.mjs && node scripts/patch-indiekit-routes-rate-limits.mjs && node scripts/patch-indiekit-error-production-stack.mjs && node scripts/patch-indieauth-devmode-guard.mjs && node scripts/patch-listening-endpoint-runtime-guards.mjs && node scripts/patch-endpoint-github-changelog-categories.mjs && node scripts/patch-endpoint-github-contributions-log.mjs && node scripts/patch-store-github-error-message.mjs && node scripts/patch-store-github-update-fallback.mjs && node scripts/patch-store-github-gitea-methods.mjs && node scripts/patch-store-github-content-type.mjs && node scripts/patch-endpoint-blogroll-feeds-alias.mjs && node scripts/patch-endpoint-posts-uid-lookup.mjs && node scripts/patch-conversations-bluesky-self-filter.mjs && node scripts/patch-conversations-bluesky-cursor-fix.mjs && node scripts/patch-endpoint-micropub-source-filter.mjs && node scripts/patch-syndicate-force-checked-default.mjs && node scripts/patch-syndicate-normalize-syndication-array.mjs && node scripts/patch-endpoint-posts-fetch-diagnostic.mjs && node scripts/patch-micropub-fetch-internal-url.mjs && node scripts/patch-micropub-session-token.mjs && node scripts/patch-indiekit-endpoint-urls-protocol.mjs && node scripts/patch-webmention-sender-hentry-syntax.mjs && node scripts/patch-webmention-sender-retry.mjs && node scripts/patch-webmention-sender-livefetch.mjs && node scripts/patch-webmention-sender-empty-details.mjs && node scripts/patch-bluesky-syndicator-internal-url.mjs && node scripts/patch-bluesky-syndicator-media-type-guard.mjs && node scripts/patch-ap-skip-draft-syndication.mjs && node scripts/patch-ap-og-image.mjs && node scripts/patch-ap-webfinger-before-auth.mjs && node scripts/patch-ap-federation-bridge-base-url.mjs && node scripts/patch-ap-compose-default-checked.mjs && node scripts/patch-ap-mastodon-reply-threading.mjs && node scripts/patch-ap-mastodon-status-id.mjs && node scripts/patch-ap-interactions-send-guard.mjs && node scripts/patch-ap-syndicate-dedup.mjs && node scripts/patch-ap-mastodon-delete-fix.mjs && node scripts/patch-ap-status-reply-id.mjs && node scripts/patch-ap-inbox-publication-url.mjs && node scripts/patch-bluesky-syndicator-delete.mjs && node scripts/patch-micropub-delete-propagation.mjs && node scripts/patch-micropub-gitea-dispatch-conditional.mjs && node scripts/patch-ap-inbox-delivery-debug.mjs", + "serve": "export NODE_ENV=${NODE_ENV:-production} INDIEKIT_DEBUG=${INDIEKIT_DEBUG:-0} && node scripts/preflight-production-security.mjs && node scripts/preflight-mongo-connection.mjs && node scripts/preflight-activitypub-rsa-key.mjs && node scripts/preflight-activitypub-profile-urls.mjs && node scripts/patch-lightningcss.mjs && node scripts/patch-endpoint-media-scope.mjs && node scripts/patch-endpoint-media-sharp-runtime.mjs && node scripts/patch-frontend-sharp-runtime.mjs && node scripts/patch-endpoint-files-upload-route.mjs && node scripts/patch-endpoint-files-upload-locales.mjs && node scripts/patch-endpoint-activitypub-locales.mjs && node scripts/patch-endpoint-homepage-locales.mjs && node scripts/patch-endpoint-homepage-identity-defaults.mjs && node scripts/patch-federation-unlisted-guards.mjs && node scripts/patch-endpoint-micropub-where-note-visibility.mjs && node scripts/patch-endpoint-podroll-opml-upload.mjs && node scripts/patch-frontend-serviceworker-file.mjs && node scripts/patch-endpoint-comments-locales.mjs && node scripts/patch-endpoint-posts-locales.mjs && node scripts/patch-endpoint-conversations-locales.mjs && node scripts/patch-conversations-collection-guards.mjs && node scripts/patch-indiekit-routes-rate-limits.mjs && node scripts/patch-indiekit-error-production-stack.mjs && node scripts/patch-indieauth-devmode-guard.mjs && node scripts/patch-listening-endpoint-runtime-guards.mjs && node scripts/patch-endpoint-github-changelog-categories.mjs && node scripts/patch-endpoint-github-contributions-log.mjs && node scripts/patch-store-github-error-message.mjs && node scripts/patch-store-github-update-fallback.mjs && node scripts/patch-store-github-gitea-methods.mjs && node scripts/patch-store-github-content-type.mjs && node scripts/patch-microsub-reader-ap-dispatch.mjs && node scripts/patch-microsub-compose-draft-guard.mjs && node scripts/patch-endpoint-blogroll-feeds-alias.mjs && node scripts/patch-endpoint-posts-uid-lookup.mjs && node scripts/patch-conversations-bluesky-self-filter.mjs && node scripts/patch-conversations-bluesky-cursor-fix.mjs && node scripts/patch-endpoint-micropub-source-filter.mjs && node scripts/patch-syndicate-force-checked-default.mjs && node scripts/patch-syndicate-normalize-syndication-array.mjs && node scripts/patch-endpoint-posts-fetch-diagnostic.mjs && node scripts/patch-micropub-fetch-internal-url.mjs && node scripts/patch-micropub-session-token.mjs && node scripts/patch-indiekit-endpoint-urls-protocol.mjs && node scripts/patch-webmention-sender-hentry-syntax.mjs && node scripts/patch-webmention-sender-retry.mjs && node scripts/patch-webmention-sender-livefetch.mjs && node scripts/patch-webmention-sender-empty-details.mjs && node scripts/patch-bluesky-syndicator-internal-url.mjs && node scripts/patch-bluesky-syndicator-media-type-guard.mjs && node scripts/patch-ap-skip-draft-syndication.mjs && node scripts/patch-ap-og-image.mjs && node scripts/patch-ap-webfinger-before-auth.mjs && node scripts/patch-ap-federation-bridge-base-url.mjs && node scripts/patch-ap-compose-default-checked.mjs && node scripts/patch-ap-mastodon-reply-threading.mjs && node scripts/patch-ap-mastodon-status-id.mjs && node scripts/patch-ap-interactions-send-guard.mjs && node scripts/patch-ap-syndicate-dedup.mjs && node scripts/patch-ap-mastodon-delete-fix.mjs && node scripts/patch-ap-status-reply-id.mjs && node scripts/patch-ap-inbox-publication-url.mjs && node scripts/patch-bluesky-syndicator-delete.mjs && node scripts/patch-micropub-delete-propagation.mjs && node scripts/patch-ap-inbox-delivery-debug.mjs && node --require ./metrics-shim.cjs node_modules/@indiekit/indiekit/bin/cli.js serve --config indiekit.config.mjs", "test": "echo \"Error: no test specified\" && exit 1" }, "keywords": [], diff --git a/scripts/patch-ap-inbox-delivery-debug.mjs b/scripts/patch-ap-inbox-delivery-debug.mjs new file mode 100644 index 00000000..6061ca18 --- /dev/null +++ b/scripts/patch-ap-inbox-delivery-debug.mjs @@ -0,0 +1,116 @@ +/** + * Patch: add inbox delivery diagnostics. + * + * Problems: + * 1. The ["fedify","federation","inbox"] LogTape category is hardcoded to + * lowestLevel "fatal", hiding HTTP Signature verification failures (401s). + * Remote servers that receive 401s stop retrying → activities are lost. + * 2. No request-level logging for incoming inbox POSTs, so we can't tell + * whether remote servers are even attempting delivery. + * + * Fix A (federation-setup.js): + * Change inbox log category from "fatal" → "error". + * Real verification failures (wrong key, clock skew, digest mismatch) surface. + * High-volume 404/410 key-fetch warnings from deleted actors stay silent. + * + * Fix B (federation-bridge.js): + * Add a console.info before fromExpressRequest() that logs every POST to an + * inbox path (path + content-type + raw body length). Fires BEFORE Fedify's + * signature check, confirming whether remote servers reach our inbox at all. + * Guarded by AP_LOG_LEVEL=debug or AP_DEBUG=1 to keep production logs quiet. + */ + +import { access, readFile, writeFile } from "node:fs/promises"; + +const MARKER_A = "// [patch] ap-inbox-delivery-debug-A"; +const MARKER_B = "// [patch] ap-inbox-delivery-debug-B"; + +// ── Fix A: federation-setup.js — inbox logger level ────────────────────────── + +const setupCandidates = [ + "node_modules/@rmdes/indiekit-endpoint-activitypub/lib/federation-setup.js", + "node_modules/@indiekit/indiekit/node_modules/@rmdes/indiekit-endpoint-activitypub/lib/federation-setup.js", +]; + +const OLD_INBOX_LOGGER = ` { + // Noise guard: HTTP Signature verification failures are expected for + // incoming activities from servers with expired/gone keys (e.g. deleted + // actors, migrated servers). These produce high log volume with no + // actionable signal — suppress everything below fatal. + category: ["fedify", "federation", "inbox"], + sinks: ["console"], + lowestLevel: "fatal", + },`; + +const NEW_INBOX_LOGGER = ` { + // Surfacing real verification failures (wrong key, clock skew, digest + // mismatch) at "error" level while keeping high-volume key-fetch + // 404/410 warnings from deleted actors silent. ${MARKER_A} + category: ["fedify", "federation", "inbox"], + sinks: ["console"], + lowestLevel: "error", + },`; + +// ── Fix B: federation-bridge.js — request-level inbox logging ──────────────── + +const bridgeCandidates = [ + "node_modules/@rmdes/indiekit-endpoint-activitypub/lib/federation-bridge.js", + "node_modules/@indiekit/indiekit/node_modules/@rmdes/indiekit-endpoint-activitypub/lib/federation-bridge.js", +]; + +// Insert a debug log right before "const request = fromExpressRequest(req, publicationUrl);" +// This is patched by ap-base-url already so that comment marker is present. +const OLD_BRIDGE_REQUEST = ` const request = fromExpressRequest(req, publicationUrl); // ap-base-url patch`; + +const NEW_BRIDGE_REQUEST = ` // Log incoming inbox POSTs before Fedify signature check. ${MARKER_B} + // Enabled by AP_LOG_LEVEL=debug or AP_DEBUG=1. + if ( + (process.env.AP_LOG_LEVEL === "debug" || process.env.AP_DEBUG === "1") && + req.method === "POST" && + (req.path.includes("/inbox") || req.path.includes("/users/")) + ) { + const _bct = (req.headers["content-type"] || "").split(";")[0].trim(); + const _bsz = req._rawBody?.length ?? (req.body ? "pre-parsed" : "none"); + console.info(\`[AP-inbox] POST \${req.path} ct=\${_bct} body=\${_bsz}B\`); + } + const request = fromExpressRequest(req, publicationUrl); // ap-base-url patch`; + +async function exists(p) { + try { await access(p); return true; } catch { return false; } +} + +async function applyPatch(candidates, oldSnippet, newSnippet, label, marker) { + let checked = 0; + let patched = 0; + for (const filePath of candidates) { + if (!(await exists(filePath))) continue; + checked++; + const source = await readFile(filePath, "utf8"); + if (source.includes(marker)) { + console.log(`[postinstall] patch-ap-inbox-delivery-debug: ${label} already applied to ${filePath}`); + continue; + } + if (!source.includes(oldSnippet)) { + console.warn(`[postinstall] patch-ap-inbox-delivery-debug: ${label} snippet not found in ${filePath}`); + continue; + } + await writeFile(filePath, source.replace(oldSnippet, newSnippet), "utf8"); + patched++; + console.log(`[postinstall] Applied patch-ap-inbox-delivery-debug (${label}) to ${filePath}`); + } + return { checked, patched }; +} + +const a = await applyPatch(setupCandidates, OLD_INBOX_LOGGER, NEW_INBOX_LOGGER, "inbox-logger-level", MARKER_A); +const b = await applyPatch(bridgeCandidates, OLD_BRIDGE_REQUEST, NEW_BRIDGE_REQUEST, "bridge-request-log", MARKER_B); + +const total = a.checked + b.checked; +const totalPatched = a.patched + b.patched; + +if (total === 0) { + console.log("[postinstall] patch-ap-inbox-delivery-debug: no target files found"); +} else if (totalPatched === 0) { + console.log("[postinstall] patch-ap-inbox-delivery-debug: already up to date"); +} else { + console.log(`[postinstall] patch-ap-inbox-delivery-debug: patched ${totalPatched} file(s)`); +}