From 97a902bda1045ab57f2082e3bed4cd6c5d6c7577 Mon Sep 17 00:00:00 2001 From: svemagie <869694+svemagie@users.noreply.github.com> Date: Sat, 21 Mar 2026 20:22:04 +0100 Subject: [PATCH] =?UTF-8?q?feat:=20merge=20upstream=20v3.7.1=E2=80=93v3.7.?= =?UTF-8?q?5=20into=20svemagie/main?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All five 3.7.x releases published 2026-03-21 in one pass. Changes from upstream: - lib/lookup-helpers.js: lookupWithSecurity → async with signed→unsigned fallback (handles servers like tags.pub that return 400 on signed GETs) - lib/mastodon/helpers/account-cache.js: add reverse lookup map (hashId → actorUrl) populated by cacheAccountStats(); export getActorUrlFromId() for follow/unfollow resolution - lib/mastodon/helpers/enrich-accounts.js: NEW — enrichAccountStats() enriches embedded account objects in serialized statuses with real follower/following/post counts; Phanpy never calls /accounts/:id so counts were always 0 without this - lib/mastodon/routes/timelines.js: call enrichAccountStats() after serialising home, public, and hashtag timelines - lib/mastodon/routes/statuses.js: processStatusContent() linkifies bare URLs and converts @user@domain mentions to links; extractMentions() builds mention list; date lookup now tries both .000Z and bare Z suffixes - lib/mastodon/routes/stubs.js: /api/v1/domain_blocks now returns real blocked-server hostnames from ap_blocked_servers instead of [] - lib/mastodon/routes/accounts.js: /accounts/relationships computes domain_blocking using ap_blocked_servers; resolveActorUrl() falls back to getActorUrlFromId() cache for timeline-author resolution - lib/controllers/federation-mgmt.js: fetch blocked servers, blocked accounts, and muted accounts in parallel; pass to template - views/activitypub-federation-mgmt.njk: add Moderation section showing blocked servers, blocked accounts, and muted accounts - package.json: bump version 3.6.8 → 3.7.5 Co-Authored-By: Claude Sonnet 4.6 --- lib/controllers/federation-mgmt.js | 11 ++- lib/lookup-helpers.js | 32 ++++++-- lib/mastodon/helpers/account-cache.js | 18 +++++ lib/mastodon/helpers/enrich-accounts.js | 97 +++++++++++++++++++++++++ lib/mastodon/routes/accounts.js | 25 ++++++- lib/mastodon/routes/statuses.js | 84 ++++++++++++++++++++- lib/mastodon/routes/stubs.js | 11 ++- lib/mastodon/routes/timelines.js | 12 +++ package.json | 2 +- views/activitypub-federation-mgmt.njk | 47 ++++++++++++ 10 files changed, 324 insertions(+), 15 deletions(-) create mode 100644 lib/mastodon/helpers/enrich-accounts.js diff --git a/lib/controllers/federation-mgmt.js b/lib/controllers/federation-mgmt.js index bda6a73..36ce931 100644 --- a/lib/controllers/federation-mgmt.js +++ b/lib/controllers/federation-mgmt.js @@ -41,12 +41,16 @@ export function federationMgmtController(mountPath, plugin) { const redisUrl = plugin.options.redisUrl || ""; - // Parallel: collection stats + posts + recent activities - const [collectionStats, postsResult, recentActivities] = + // Parallel: collection stats + posts + recent activities + moderation data + const pluginCollections = plugin._collections || {}; + const [collectionStats, postsResult, recentActivities, blockedServers, blockedAccounts, mutedAccounts] = await Promise.all([ getCollectionStats(collections, { redisUrl }), getPaginatedPosts(collections, request.query.page), getRecentActivities(collections), + pluginCollections.ap_blocked_servers?.find({}).sort({ blockedAt: -1 }).toArray() || [], + pluginCollections.ap_blocked?.find({}).sort({ blockedAt: -1 }).toArray() || [], + pluginCollections.ap_muted?.find({}).sort({ mutedAt: -1 }).toArray() || [], ]); const csrfToken = getToken(request.session); @@ -62,6 +66,9 @@ export function federationMgmtController(mountPath, plugin) { posts: postsResult.posts, cursor: postsResult.cursor, recentActivities, + blockedServers: blockedServers || [], + blockedAccounts: blockedAccounts || [], + mutedAccounts: mutedAccounts || [], csrfToken, mountPath, publicationUrl: plugin._publicationUrl, diff --git a/lib/lookup-helpers.js b/lib/lookup-helpers.js index 149c932..c542fc5 100644 --- a/lib/lookup-helpers.js +++ b/lib/lookup-helpers.js @@ -14,14 +14,36 @@ * Using `crossOrigin: "ignore"` tells Fedify to silently discard objects * whose id doesn't match the fetch origin, rather than throwing. * + * When an authenticated document loader is provided (for Authorized Fetch + * compatibility), the lookup is tried with it first. If it fails (some + * servers like tags.pub return 400 for signed GETs), a fallback to the + * default unsigned loader is attempted automatically. + * * @param {object} ctx - Fedify Context * @param {string|URL} input - URL or handle to look up * @param {object} [options] - Additional options passed to lookupObject * @returns {Promise} Resolved object or null */ -export function lookupWithSecurity(ctx, input, options = {}) { - return ctx.lookupObject(input, { - crossOrigin: "ignore", - ...options, - }); +export async function lookupWithSecurity(ctx, input, options = {}) { + const baseOptions = { crossOrigin: "ignore", ...options }; + + let result = null; + try { + result = await ctx.lookupObject(input, baseOptions); + } catch { + // signed lookup threw — fall through to unsigned + } + + // If signed lookup failed and we used a custom documentLoader, + // retry without it (unsigned GET) + if (!result && options.documentLoader) { + try { + const { documentLoader: _, ...unsignedOptions } = baseOptions; + result = await ctx.lookupObject(input, unsignedOptions); + } catch { + // unsigned also failed — return null + } + } + + return result; } diff --git a/lib/mastodon/helpers/account-cache.js b/lib/mastodon/helpers/account-cache.js index 0407855..f4d3a21 100644 --- a/lib/mastodon/helpers/account-cache.js +++ b/lib/mastodon/helpers/account-cache.js @@ -6,6 +6,7 @@ * * LRU-style with TTL — entries expire after 1 hour. */ +import { remoteActorId } from "./id-mapping.js"; const CACHE_TTL_MS = 60 * 60 * 1000; // 1 hour const MAX_ENTRIES = 500; @@ -13,6 +14,10 @@ const MAX_ENTRIES = 500; // Map const cache = new Map(); +// Reverse map: accountId (hash) → actorUrl +// Populated alongside the stats cache for follow/unfollow lookups +const idToUrl = new Map(); + /** * Store account stats in cache. * @param {string} actorUrl - The actor's URL (cache key) @@ -28,6 +33,10 @@ export function cacheAccountStats(actorUrl, stats) { } cache.set(actorUrl, { ...stats, cachedAt: Date.now() }); + + // Maintain reverse lookup + const hashId = remoteActorId(actorUrl); + if (hashId) idToUrl.set(hashId, actorUrl); } /** @@ -49,3 +58,12 @@ export function getCachedAccountStats(actorUrl) { return entry; } + +/** + * Reverse lookup: get actor URL from account hash ID. + * @param {string} hashId - The 24-char hex account ID + * @returns {string|null} Actor URL or null + */ +export function getActorUrlFromId(hashId) { + return idToUrl.get(hashId) || null; +} diff --git a/lib/mastodon/helpers/enrich-accounts.js b/lib/mastodon/helpers/enrich-accounts.js new file mode 100644 index 0000000..9ecc04e --- /dev/null +++ b/lib/mastodon/helpers/enrich-accounts.js @@ -0,0 +1,97 @@ +/** + * Enrich embedded account objects in serialized statuses with real + * follower/following/post counts from remote AP collections. + * + * Phanpy (and some other clients) never call /accounts/:id — they + * trust the account object embedded in each status. Without enrichment, + * these show 0/0/0 for all remote accounts. + * + * Uses the account stats cache to avoid redundant fetches. Only resolves + * unique authors with 0 counts that aren't already cached. + */ +import { getCachedAccountStats } from "./account-cache.js"; +import { resolveRemoteAccount } from "./resolve-account.js"; + +/** + * Enrich account objects in a list of serialized statuses. + * Resolves unique authors in parallel (max 5 concurrent). + * + * @param {Array} statuses - Serialized Mastodon Status objects (mutated in place) + * @param {object} pluginOptions - Plugin options with federation context + * @param {string} baseUrl - Server base URL + */ +export async function enrichAccountStats(statuses, pluginOptions, baseUrl) { + if (!statuses?.length || !pluginOptions?.federation) return; + + // Collect unique author URLs that need enrichment + const accountsToEnrich = new Map(); // url -> [account references] + for (const status of statuses) { + collectAccount(status.account, accountsToEnrich); + if (status.reblog?.account) { + collectAccount(status.reblog.account, accountsToEnrich); + } + } + + if (accountsToEnrich.size === 0) return; + + // Resolve in parallel with concurrency limit + const entries = [...accountsToEnrich.entries()]; + const CONCURRENCY = 5; + for (let i = 0; i < entries.length; i += CONCURRENCY) { + const batch = entries.slice(i, i + CONCURRENCY); + await Promise.all( + batch.map(async ([url, accounts]) => { + try { + const resolved = await resolveRemoteAccount(url, pluginOptions, baseUrl); + if (resolved) { + for (const account of accounts) { + account.followers_count = resolved.followers_count; + account.following_count = resolved.following_count; + account.statuses_count = resolved.statuses_count; + if (resolved.created_at && account.created_at) { + account.created_at = resolved.created_at; + } + if (resolved.note) account.note = resolved.note; + if (resolved.fields?.length) account.fields = resolved.fields; + if (resolved.avatar && resolved.avatar !== account.avatar) { + account.avatar = resolved.avatar; + account.avatar_static = resolved.avatar; + } + if (resolved.header) { + account.header = resolved.header; + account.header_static = resolved.header; + } + } + } + } catch { + // Silently skip failed resolutions + } + }), + ); + } +} + +/** + * Collect an account reference for enrichment if it has 0 counts + * and isn't already cached. + */ +function collectAccount(account, map) { + if (!account?.url) return; + if (account.followers_count > 0 || account.statuses_count > 0) return; + + // Check cache first — if cached, apply immediately + const cached = getCachedAccountStats(account.url); + if (cached) { + account.followers_count = cached.followersCount || 0; + account.following_count = cached.followingCount || 0; + account.statuses_count = cached.statusesCount || 0; + if (cached.createdAt) account.created_at = cached.createdAt; + return; + } + + // Queue for remote resolution + if (!map.has(account.url)) { + map.set(account.url, []); + } + map.get(account.url).push(account); +} diff --git a/lib/mastodon/routes/accounts.js b/lib/mastodon/routes/accounts.js index a5cfb8e..dae75d1 100644 --- a/lib/mastodon/routes/accounts.js +++ b/lib/mastodon/routes/accounts.js @@ -8,6 +8,7 @@ import express from "express"; import { serializeCredentialAccount, serializeAccount } from "../entities/account.js"; import { serializeStatus } from "../entities/status.js"; import { accountId, remoteActorId } from "../helpers/id-mapping.js"; +import { getActorUrlFromId } from "../helpers/account-cache.js"; import { buildPaginationQuery, parseLimit, setPaginationHeaders } from "../helpers/pagination.js"; import { resolveRemoteAccount } from "../helpers/resolve-account.js"; @@ -169,11 +170,12 @@ router.get("/api/v1/accounts/relationships", async (req, res, next) => { const collections = req.app.locals.mastodonCollections; - const [followers, following, blocked, muted] = await Promise.all([ + const [followers, following, blocked, muted, blockedServers] = await Promise.all([ collections.ap_followers.find({}).toArray(), collections.ap_following.find({}).toArray(), collections.ap_blocked.find({}).toArray(), collections.ap_muted.find({}).toArray(), + collections.ap_blocked_servers?.find({}).toArray() || [], ]); const followerIds = new Set(followers.map((f) => remoteActorId(f.actorUrl))); @@ -181,6 +183,21 @@ router.get("/api/v1/accounts/relationships", async (req, res, next) => { const blockedIds = new Set(blocked.map((b) => remoteActorId(b.url))); const mutedIds = new Set(muted.filter((m) => m.url).map((m) => remoteActorId(m.url))); + // Build domain-blocked actor ID set by checking known actors against blocked server hostnames + const blockedDomains = new Set(blockedServers.map((s) => s.hostname).filter(Boolean)); + const domainBlockedIds = new Set(); + if (blockedDomains.size > 0) { + const allActors = [...followers, ...following]; + for (const actor of allActors) { + try { + const domain = new URL(actor.actorUrl).hostname; + if (blockedDomains.has(domain)) { + domainBlockedIds.add(remoteActorId(actor.actorUrl)); + } + } catch { /* skip invalid URLs */ } + } + } + const relationships = ids.map((id) => ({ id, following: followingIds.has(id), @@ -194,7 +211,7 @@ router.get("/api/v1/accounts/relationships", async (req, res, next) => { muting_notifications: mutedIds.has(id), requested: false, requested_by: false, - domain_blocking: false, + domain_blocking: domainBlockedIds.has(id), endorsed: false, note: "", })); @@ -746,6 +763,10 @@ async function resolveActorUrl(id, collections) { } } + // Check account cache reverse lookup (populated by resolveRemoteAccount) + const cachedUrl = getActorUrlFromId(id); + if (cachedUrl) return cachedUrl; + return null; } diff --git a/lib/mastodon/routes/statuses.js b/lib/mastodon/routes/statuses.js index c22102f..47a6789 100644 --- a/lib/mastodon/routes/statuses.js +++ b/lib/mastodon/routes/statuses.js @@ -247,12 +247,17 @@ router.post("/api/v1/statuses", async (req, res, next) => { }); }; + // Process content: linkify URLs and extract @mentions + const rawContent = data.properties.content || { text: statusText || "", html: "" }; + const processedContent = processStatusContent(rawContent, statusText || ""); + const mentions = extractMentions(statusText || ""); + const now = new Date().toISOString(); const timelineItem = await addTimelineItem(collections, { uid: postUrl, url: postUrl, type: data.properties["post-type"] || "note", - content: data.properties.content || { text: statusText || "", html: "" }, + content: processedContent, summary: spoilerText || "", sensitive: sensitive === true || sensitive === "true", visibility: visibility || "public", @@ -274,7 +279,7 @@ router.post("/api/v1/statuses", async (req, res, next) => { category: categories, counts: { replies: 0, boosts: 0, likes: 0 }, linkPreviews: [], - mentions: [], + mentions, emojis: [], }); @@ -567,8 +572,17 @@ async function findTimelineItemById(collection, id) { // Try cursor-based lookup first (published date from ms-since-epoch) const publishedDate = decodeCursor(id); if (publishedDate) { - const item = await collection.findOne({ published: publishedDate }); + // Try exact match first (with .000Z suffix from toISOString) + let item = await collection.findOne({ published: publishedDate }); if (item) return item; + + // Try without milliseconds — stored dates often lack .000Z + // e.g., "2026-03-21T15:33:50Z" vs "2026-03-21T15:33:50.000Z" + const withoutMs = publishedDate.replace(/\.000Z$/, "Z"); + if (withoutMs !== publishedDate) { + item = await collection.findOne({ published: withoutMs }); + if (item) return item; + } } // Fall back to ObjectId lookup (legacy IDs) @@ -627,4 +641,68 @@ async function loadItemInteractions(collections, item) { return { favouritedIds, rebloggedIds, bookmarkedIds }; } +/** + * Process status content: linkify bare URLs and convert @mentions to links. + * + * Mastodon clients send plain text — the server is responsible for + * converting URLs and mentions into HTML links. + * + * @param {object} content - { text, html } from Micropub pipeline + * @param {string} rawText - Original status text from client + * @returns {object} { text, html } with linkified content + */ +function processStatusContent(content, rawText) { + let html = content.html || content.text || rawText || ""; + + // If the HTML is just plain text wrapped in

, process it + // Don't touch HTML that already has links (from Micropub rendering) + if (!html.includes(""')\]]+)/g, + '$1', + ); + + // Convert @user@domain mentions to profile links + html = html.replace( + /(?:^|\s)(@([a-zA-Z0-9_]+)@([a-zA-Z0-9.-]+\.[a-zA-Z]{2,}))/g, + (match, full, username, domain) => + match.replace( + full, + `@${username}@${domain}`, + ), + ); + } + + return { + text: content.text || rawText || "", + html, + }; +} + +/** + * Extract @user@domain mentions from text into mention objects. + * + * @param {string} text - Status text + * @returns {Array<{name: string, url: string}>} Mention objects + */ +function extractMentions(text) { + if (!text) return []; + const mentionRegex = /@([a-zA-Z0-9_]+)@([a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/g; + const mentions = []; + const seen = new Set(); + let match; + while ((match = mentionRegex.exec(text)) !== null) { + const [, username, domain] = match; + const key = `${username}@${domain}`.toLowerCase(); + if (seen.has(key)) continue; + seen.add(key); + mentions.push({ + name: `@${username}@${domain}`, + url: `https://${domain}/@${username}`, + }); + } + return mentions; +} + export default router; diff --git a/lib/mastodon/routes/stubs.js b/lib/mastodon/routes/stubs.js index eb9a73e..70c9458 100644 --- a/lib/mastodon/routes/stubs.js +++ b/lib/mastodon/routes/stubs.js @@ -314,8 +314,15 @@ router.get("/api/v1/conversations", (req, res) => { // ─── Domain blocks ────────────────────────────────────────────────────────── -router.get("/api/v1/domain_blocks", (req, res) => { - res.json([]); +router.get("/api/v1/domain_blocks", async (req, res) => { + try { + const collections = req.app.locals.mastodonCollections; + if (!collections?.ap_blocked_servers) return res.json([]); + const docs = await collections.ap_blocked_servers.find({}).toArray(); + res.json(docs.map((d) => d.hostname).filter(Boolean)); + } catch { + res.json([]); + } }); // ─── Endorsements ─────────────────────────────────────────────────────────── diff --git a/lib/mastodon/routes/timelines.js b/lib/mastodon/routes/timelines.js index 4a4763e..54b182f 100644 --- a/lib/mastodon/routes/timelines.js +++ b/lib/mastodon/routes/timelines.js @@ -9,6 +9,7 @@ import express from "express"; import { serializeStatus } from "../entities/status.js"; import { buildPaginationQuery, parseLimit, setPaginationHeaders } from "../helpers/pagination.js"; import { loadModerationData, applyModerationFilters } from "../../item-processing.js"; +import { enrichAccountStats } from "../helpers/enrich-accounts.js"; const router = express.Router(); // eslint-disable-line new-cap @@ -76,6 +77,11 @@ router.get("/api/v1/timelines/home", async (req, res, next) => { }), ); + // Enrich embedded account objects with real follower/following/post counts. + // Phanpy never calls /accounts/:id — it trusts embedded account data. + const pluginOptions = req.app.locals.mastodonPluginOptions || {}; + await enrichAccountStats(statuses, pluginOptions, baseUrl); + // Set pagination Link headers setPaginationHeaders(res, req, items, limit); @@ -155,6 +161,9 @@ router.get("/api/v1/timelines/public", async (req, res, next) => { }), ); + const pluginOpts = req.app.locals.mastodonPluginOptions || {}; + await enrichAccountStats(statuses, pluginOpts, baseUrl); + setPaginationHeaders(res, req, items, limit); res.json(statuses); } catch (error) { @@ -215,6 +224,9 @@ router.get("/api/v1/timelines/tag/:hashtag", async (req, res, next) => { }), ); + const pluginOpts = req.app.locals.mastodonPluginOptions || {}; + await enrichAccountStats(statuses, pluginOpts, baseUrl); + setPaginationHeaders(res, req, items, limit); res.json(statuses); } catch (error) { diff --git a/package.json b/package.json index ec9a191..9798d17 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@rmdes/indiekit-endpoint-activitypub", - "version": "3.6.8", + "version": "3.7.5", "description": "ActivityPub federation endpoint for Indiekit via Fedify. Adds full fediverse support: actor, inbox, outbox, followers, following, syndication, and Mastodon migration.", "keywords": [ "indiekit", diff --git a/views/activitypub-federation-mgmt.njk b/views/activitypub-federation-mgmt.njk index dd52522..d76b37e 100644 --- a/views/activitypub-federation-mgmt.njk +++ b/views/activitypub-federation-mgmt.njk @@ -116,6 +116,53 @@ {% endif %} + {# --- Moderation Overview --- #} +

+

Moderation

+ {% if blockedServers.length > 0 %} +

Blocked servers ({{ blockedServers.length }})

+
+ {% for server in blockedServers %} +
+ 🚫 {{ server.hostname }} + {% if server.blockedAt %} + {{ server.blockedAt | date("PPp") }} + {% endif %} +
+ {% endfor %} +
+ {% else %} + {{ prose({ text: "No servers blocked." }) }} + {% endif %} + + {% if blockedAccounts.length > 0 %} +

Blocked accounts ({{ blockedAccounts.length }})

+
+ {% for account in blockedAccounts %} +
+ 🚫 {{ account.url or account.handle or "Unknown" }} + {% if account.blockedAt %} + {{ account.blockedAt | date("PPp") }} + {% endif %} +
+ {% endfor %} +
+ {% else %} + {{ prose({ text: "No accounts blocked." }) }} + {% endif %} + + {% if mutedAccounts.length > 0 %} +

Muted ({{ mutedAccounts.length }})

+
+ {% for muted in mutedAccounts %} +
+ 🔇 {{ muted.url or muted.keyword or "Unknown" }} +
+ {% endfor %} +
+ {% endif %} +
+ {# --- JSON Modal --- #}