From af2f89907349f7620424fffc863983d8f4a29a4d Mon Sep 17 00:00:00 2001 From: Ricardo Date: Tue, 3 Mar 2026 12:48:40 +0100 Subject: [PATCH] refactor: unify reader and explore processing pipeline (Release 0) Extract shared item-processing.js module with postProcessItems(), applyModerationFilters(), buildInteractionMap(), applyTabFilter(), renderItemCards(), and loadModerationData(). All controllers (reader, api-timeline, explore, hashtag-explore, tag-timeline) now flow through the same pipeline. Unify Alpine.js infinite scroll into single parameterized apInfiniteScroll component configured via data attributes, replacing the separate apExploreScroll component. Also adds fetchAndStoreQuote() for quote enrichment and on-demand quote fetching in post-detail controller. Bump version to 2.5.0. Confab-Link: http://localhost:8080/sessions/e9d666ac-3c90-4298-9e92-9ac9d142bc06 --- assets/reader-infinite-scroll.js | 176 +++++++--------------- assets/reader-tabs.js | 10 ++ assets/reader.css | 12 +- lib/controllers/api-timeline.js | 123 ++-------------- lib/controllers/explore-utils.js | 43 +++++- lib/controllers/explore.js | 161 ++++++++------------ lib/controllers/hashtag-explore.js | 28 +--- lib/controllers/post-detail.js | 41 +++++- lib/controllers/reader.js | 132 ++--------------- lib/controllers/tag-timeline.js | 88 +---------- lib/item-processing.js | 226 +++++++++++++++++++++++++++++ lib/og-unfurl.js | 59 +++++++- package.json | 2 +- views/activitypub-explore.njk | 46 ++++-- views/activitypub-reader.njk | 13 +- views/activitypub-tag-timeline.njk | 14 +- 16 files changed, 585 insertions(+), 589 deletions(-) create mode 100644 lib/item-processing.js diff --git a/assets/reader-infinite-scroll.js b/assets/reader-infinite-scroll.js index 5f616e6..4ae09cc 100644 --- a/assets/reader-infinite-scroll.js +++ b/assets/reader-infinite-scroll.js @@ -1,121 +1,55 @@ /** - * Infinite scroll — AlpineJS component for AJAX load-more on the timeline - * Registers the `apInfiniteScroll` Alpine data component. + * Infinite scroll — unified AlpineJS component for AJAX load-more. + * Works for both reader timeline and explore view via data attributes. + * + * Required data attributes on the component element: + * data-cursor — initial pagination cursor value + * data-api-url — API endpoint URL (e.g., /activitypub/admin/reader/api/timeline) + * data-cursor-param — query param name for the cursor (e.g., "before" or "max_id") + * data-cursor-field — response JSON field for the next cursor (e.g., "before" or "maxId") + * data-timeline-id — DOM ID of the timeline container to append HTML into + * + * Optional: + * data-extra-params — JSON-encoded object of additional query params + * data-hide-pagination — CSS selector of no-JS pagination to hide */ document.addEventListener("alpine:init", () => { - // eslint-disable-next-line no-undef - Alpine.data("apExploreScroll", () => ({ - loading: false, - done: false, - maxId: null, - instance: "", - scope: "local", - hashtag: "", - observer: null, - - init() { - const el = this.$el; - this.maxId = el.dataset.maxId || null; - this.instance = el.dataset.instance || ""; - this.scope = el.dataset.scope || "local"; - this.hashtag = el.dataset.hashtag || ""; - - if (!this.maxId) { - this.done = true; - return; - } - - this.observer = new IntersectionObserver( - (entries) => { - for (const entry of entries) { - if (entry.isIntersecting && !this.loading && !this.done) { - this.loadMore(); - } - } - }, - { rootMargin: "200px" } - ); - - if (this.$refs.sentinel) { - this.observer.observe(this.$refs.sentinel); - } - }, - - async loadMore() { - if (this.loading || this.done || !this.maxId) return; - - this.loading = true; - - const timeline = document.getElementById("ap-explore-timeline"); - const mountPath = timeline ? timeline.dataset.mountPath : ""; - - const params = new URLSearchParams({ - instance: this.instance, - scope: this.scope, - max_id: this.maxId, - }); - // Pass hashtag when in hashtag mode so infinite scroll stays on tag timeline - if (this.hashtag) { - params.set("hashtag", this.hashtag); - } - - try { - const res = await fetch( - `${mountPath}/admin/reader/api/explore?${params.toString()}`, - { headers: { Accept: "application/json" } } - ); - - if (!res.ok) throw new Error(`HTTP ${res.status}`); - - const data = await res.json(); - - if (data.html && timeline) { - timeline.insertAdjacentHTML("beforeend", data.html); - } - - if (data.maxId) { - this.maxId = data.maxId; - } else { - this.done = true; - if (this.observer) this.observer.disconnect(); - } - } catch (err) { - console.error("[ap-explore-scroll] load failed:", err.message); - } finally { - this.loading = false; - } - }, - - destroy() { - if (this.observer) this.observer.disconnect(); - }, - })); - // eslint-disable-next-line no-undef Alpine.data("apInfiniteScroll", () => ({ loading: false, done: false, - before: null, - tab: "", - tag: "", + cursor: null, + apiUrl: "", + cursorParam: "before", + cursorField: "before", + timelineId: "", + extraParams: {}, observer: null, init() { const el = this.$el; - this.before = el.dataset.before || null; - this.tab = el.dataset.tab || ""; - this.tag = el.dataset.tag || ""; + this.cursor = el.dataset.cursor || null; + this.apiUrl = el.dataset.apiUrl || ""; + this.cursorParam = el.dataset.cursorParam || "before"; + this.cursorField = el.dataset.cursorField || "before"; + this.timelineId = el.dataset.timelineId || ""; - // Hide the no-JS pagination fallback now that JS is active - const paginationEl = - document.getElementById("ap-reader-pagination") || - document.getElementById("ap-tag-pagination"); - if (paginationEl) { - paginationEl.style.display = "none"; + // Parse extra params from JSON data attribute + try { + this.extraParams = JSON.parse(el.dataset.extraParams || "{}"); + } catch { + this.extraParams = {}; } - if (!this.before) { + // Hide the no-JS pagination fallback now that JS is active + const hideSel = el.dataset.hidePagination; + if (hideSel) { + const paginationEl = document.getElementById(hideSel); + if (paginationEl) paginationEl.style.display = "none"; + } + + if (!this.cursor) { this.done = true; return; } @@ -129,7 +63,7 @@ document.addEventListener("alpine:init", () => { } } }, - { rootMargin: "200px" } + { rootMargin: "200px" }, ); if (this.$refs.sentinel) { @@ -138,36 +72,36 @@ document.addEventListener("alpine:init", () => { }, async loadMore() { - if (this.loading || this.done || !this.before) return; + if (this.loading || this.done || !this.cursor) return; this.loading = true; - const timeline = document.getElementById("ap-timeline"); - const mountPath = timeline ? timeline.dataset.mountPath : ""; - - const params = new URLSearchParams({ before: this.before }); - if (this.tab) params.set("tab", this.tab); - if (this.tag) params.set("tag", this.tag); + const params = new URLSearchParams({ + [this.cursorParam]: this.cursor, + ...this.extraParams, + }); try { const res = await fetch( - `${mountPath}/admin/reader/api/timeline?${params.toString()}`, - { headers: { Accept: "application/json" } } + `${this.apiUrl}?${params.toString()}`, + { headers: { Accept: "application/json" } }, ); if (!res.ok) throw new Error(`HTTP ${res.status}`); const data = await res.json(); + const timeline = this.timelineId + ? document.getElementById(this.timelineId) + : null; + if (data.html && timeline) { - // Append the returned pre-rendered HTML timeline.insertAdjacentHTML("beforeend", data.html); } - if (data.before) { - this.before = data.before; + if (data[this.cursorField]) { + this.cursor = data[this.cursorField]; } else { - // No more items this.done = true; if (this.observer) this.observer.disconnect(); } @@ -178,10 +112,6 @@ document.addEventListener("alpine:init", () => { } }, - appendItems(/* detail */) { - // Custom event hook — not used in this implementation but kept for extensibility - }, - destroy() { if (this.observer) this.observer.disconnect(); }, @@ -282,7 +212,9 @@ document.addEventListener("alpine:init", () => { const card = entry.target; const uid = card.dataset.uid; if (uid && !card.classList.contains("ap-card--read")) { - card.classList.add("ap-card--read"); + // Mark read server-side but DON'T dim visually in this session. + // Cards only appear dimmed when they arrive from the server + // with item.read=true on a subsequent page load. this._batch.push(uid); } this._observer.unobserve(card); diff --git a/assets/reader-tabs.js b/assets/reader-tabs.js index e25191b..da58aad 100644 --- a/assets/reader-tabs.js +++ b/assets/reader-tabs.js @@ -366,6 +366,16 @@ document.addEventListener("alpine:init", () => { } }, + // ── Public load-more method (called by button click) ──────────────────── + + loadMoreTab(tab) { + if (tab.type === "instance") { + this._loadMoreInstanceTab(tab); + } else if (tab.type === "hashtag") { + this._loadMoreHashtagTab(tab); + } + }, + // ── Infinite scroll for tab panels ─────────────────────────────────────── _setupScrollObserver(tab) { diff --git a/assets/reader.css b/assets/reader.css index dfe19db..dd7d04c 100644 --- a/assets/reader.css +++ b/assets/reader.css @@ -2486,15 +2486,21 @@ } .ap-quote-embed__content { - -webkit-box-orient: vertical; - -webkit-line-clamp: 6; color: var(--color-on-background); - display: -webkit-box; font-size: var(--font-size-s); line-height: 1.5; + max-height: calc(1.5em * 6); overflow: hidden; } +.ap-quote-embed__content a { + display: inline; +} + +.ap-quote-embed__content a span { + display: inline; +} + .ap-quote-embed__content p { margin: 0 0 var(--space-xs); } diff --git a/lib/controllers/api-timeline.js b/lib/controllers/api-timeline.js index ffc5e07..1dbb897 100644 --- a/lib/controllers/api-timeline.js +++ b/lib/controllers/api-timeline.js @@ -4,12 +4,7 @@ import { getTimelineItems, countNewItems, markItemsRead } from "../storage/timeline.js"; import { getToken, validateToken } from "../csrf.js"; -import { - getMutedUrls, - getMutedKeywords, - getBlockedUrls, - getFilterMode, -} from "../storage/moderation.js"; +import { postProcessItems, applyTabFilter, loadModerationData, renderItemCards } from "../item-processing.js"; export function apiTimelineController(mountPath) { return async (request, response, next) => { @@ -25,7 +20,7 @@ export function apiTimelineController(mountPath) { const before = request.query.before; const limit = 20; - // Build storage query options (same logic as readerController) + // Build storage query options const unread = request.query.unread === "1"; const options = { before, limit, unread }; @@ -44,124 +39,32 @@ export function apiTimelineController(mountPath) { const result = await getTimelineItems(collections, options); - // Client-side tab filtering for types not supported by storage - let items = result.items; - if (!tag) { - if (tab === "replies") { - items = items.filter((item) => item.inReplyTo); - } else if (tab === "media") { - items = items.filter( - (item) => - (item.photo && item.photo.length > 0) || - (item.video && item.video.length > 0) || - (item.audio && item.audio.length > 0) - ); - } - } + // Tab filtering for types not supported by storage layer + const tabFiltered = tag ? result.items : applyTabFilter(result.items, tab); - // Apply moderation filters + // Shared processing pipeline: moderation, quote stripping, interactions const modCollections = { ap_muted: application?.collections?.get("ap_muted"), ap_blocked: application?.collections?.get("ap_blocked"), ap_profile: application?.collections?.get("ap_profile"), }; - const [mutedUrls, mutedKeywords, blockedUrls, filterMode] = - await Promise.all([ - getMutedUrls(modCollections), - getMutedKeywords(modCollections), - getBlockedUrls(modCollections), - getFilterMode(modCollections), - ]); - const blockedSet = new Set(blockedUrls); - const mutedSet = new Set(mutedUrls); + const moderation = await loadModerationData(modCollections); - if (blockedSet.size > 0 || mutedSet.size > 0 || mutedKeywords.length > 0) { - items = items.filter((item) => { - if (item.author?.url && blockedSet.has(item.author.url)) { - return false; - } - - const isMutedActor = item.author?.url && mutedSet.has(item.author.url); - let matchedKeyword = null; - if (mutedKeywords.length > 0) { - const searchable = [item.content?.text, item.name, item.summary] - .filter(Boolean) - .join(" ") - .toLowerCase(); - if (searchable) { - matchedKeyword = mutedKeywords.find((kw) => - searchable.includes(kw.toLowerCase()) - ); - } - } - - if (isMutedActor || matchedKeyword) { - if (filterMode === "warn") { - item._moderated = true; - item._moderationReason = isMutedActor ? "muted_account" : "muted_keyword"; - if (matchedKeyword) item._moderationKeyword = matchedKeyword; - return true; - } - return false; - } - - return true; - }); - } - - // Get interaction state - const interactionsCol = application?.collections?.get("ap_interactions"); - const interactionMap = {}; - - if (interactionsCol) { - const lookupUrls = new Set(); - const objectUrlToUid = new Map(); - for (const item of items) { - const uid = item.uid; - const displayUrl = item.url || item.originalUrl; - if (uid) { lookupUrls.add(uid); objectUrlToUid.set(uid, uid); } - if (displayUrl) { lookupUrls.add(displayUrl); objectUrlToUid.set(displayUrl, uid || displayUrl); } - } - if (lookupUrls.size > 0) { - const interactions = await interactionsCol - .find({ objectUrl: { $in: [...lookupUrls] } }) - .toArray(); - for (const interaction of interactions) { - const key = objectUrlToUid.get(interaction.objectUrl) || interaction.objectUrl; - if (!interactionMap[key]) interactionMap[key] = {}; - interactionMap[key][interaction.type] = true; - } - } - } + const { items, interactionMap } = await postProcessItems(tabFiltered, { + moderation, + interactionsCol: application?.collections?.get("ap_interactions"), + }); const csrfToken = getToken(request.session); - - // Render each card server-side using the same Nunjucks template - // Merge response.locals so that i18n (__), mountPath, etc. are available - const templateData = { + const html = await renderItemCards(items, request, { ...response.locals, mountPath, csrfToken, interactionMap, - }; - - const htmlParts = await Promise.all( - items.map((item) => { - return new Promise((resolve, reject) => { - request.app.render( - "partials/ap-item-card.njk", - { ...templateData, item }, - (err, html) => { - if (err) reject(err); - else resolve(html); - } - ); - }); - }) - ); + }); response.json({ - html: htmlParts.join(""), + html, before: result.before, }); } catch (error) { diff --git a/lib/controllers/explore-utils.js b/lib/controllers/explore-utils.js index 7907cf1..ff31c9f 100644 --- a/lib/controllers/explore-utils.js +++ b/lib/controllers/explore-utils.js @@ -92,7 +92,7 @@ export function mapMastodonStatusToItem(status, instance) { } } - return { + const item = { uid: status.url || status.uri || "", url: status.url || status.uri || "", type: "note", @@ -119,4 +119,45 @@ export function mapMastodonStatusToItem(status, instance) { createdAt: new Date().toISOString(), _explore: true, }; + + // Map quoted post data if present (Mastodon 4.3+ quote support) + // Mastodon API wraps the quoted status: { state: "accepted", quoted_status: { ...fullStatus } } + const quotedStatus = status.quote?.quoted_status || null; + if (quotedStatus) { + item.quoteUrl = quotedStatus.url || quotedStatus.uri || ""; + + const q = quotedStatus; + const qAccount = q.account || {}; + const qAcct = qAccount.acct || ""; + const qHandle = qAcct.includes("@") ? `@${qAcct}` : `@${qAcct}@${instance}`; + const qPhoto = []; + for (const att of q.media_attachments || []) { + const attUrl = att.url || att.remote_url || ""; + if (attUrl && (att.type === "image" || att.type === "gifv")) { + qPhoto.push(attUrl); + } + } + + item.quote = { + url: q.url || q.uri || "", + uid: q.uri || q.url || "", + author: { + name: sanitizeHtml(qAccount.display_name || qAccount.username || "Unknown", { allowedTags: [], allowedAttributes: {} }), + url: qAccount.url || "", + photo: qAccount.avatar || qAccount.avatar_static || "", + handle: qHandle, + }, + content: { + text: (q.content || "").replace(/<[^>]*>/g, ""), + html: sanitizeContent(q.content || ""), + }, + published: q.created_at || "", + name: "", + photo: qPhoto.slice(0, 1), + }; + } else { + item.quoteUrl = ""; + } + + return item; } diff --git a/lib/controllers/explore.js b/lib/controllers/explore.js index 0f19bce..6e266a6 100644 --- a/lib/controllers/explore.js +++ b/lib/controllers/explore.js @@ -8,6 +8,7 @@ import { searchInstances, checkInstanceTimeline, getPopularAccounts } from "../fedidb.js"; import { getToken } from "../csrf.js"; import { validateInstance, validateHashtag, mapMastodonStatusToItem } from "./explore-utils.js"; +import { postProcessItems, renderItemCards } from "../item-processing.js"; const FETCH_TIMEOUT_MS = 10_000; const MAX_RESULTS = 20; @@ -15,6 +16,57 @@ const MAX_RESULTS = 20; // Re-export validateInstance for backward compatibility (used by tabs.js, index.js) export { validateInstance } from "./explore-utils.js"; +/** + * Fetch statuses from a remote Mastodon-compatible instance. + * + * @param {string} instance - Validated hostname + * @param {object} options + * @param {string} [options.scope] - "local" or "federated" + * @param {string} [options.hashtag] - Validated hashtag (no #) + * @param {string} [options.maxId] - Pagination cursor + * @param {number} [options.limit] - Max results + * @returns {Promise<{ items: Array, nextMaxId: string|null }>} + */ +export async function fetchMastodonTimeline(instance, { scope = "local", hashtag, maxId, limit = MAX_RESULTS } = {}) { + const isLocal = scope === "local"; + let apiUrl; + if (hashtag) { + apiUrl = new URL(`https://${instance}/api/v1/timelines/tag/${encodeURIComponent(hashtag)}`); + } else { + apiUrl = new URL(`https://${instance}/api/v1/timelines/public`); + } + apiUrl.searchParams.set("local", isLocal ? "true" : "false"); + apiUrl.searchParams.set("limit", String(limit)); + if (maxId) apiUrl.searchParams.set("max_id", maxId); + + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS); + + const fetchRes = await fetch(apiUrl.toString(), { + headers: { Accept: "application/json" }, + signal: controller.signal, + }); + clearTimeout(timeoutId); + + if (!fetchRes.ok) { + throw new Error(`Remote instance returned HTTP ${fetchRes.status}`); + } + + const statuses = await fetchRes.json(); + if (!Array.isArray(statuses)) { + throw new Error("Unexpected API response format"); + } + + const items = statuses.map((s) => mapMastodonStatusToItem(s, instance)); + + const nextMaxId = + statuses.length === limit && statuses.length > 0 + ? statuses[statuses.length - 1].id || null + : null; + + return { items, nextMaxId }; +} + export function exploreController(mountPath) { return async (request, response, next) => { try { @@ -59,50 +111,15 @@ export function exploreController(mountPath) { }); } - // Build API URL: hashtag timeline or public timeline - const isLocal = scope === "local"; - let apiUrl; - if (hashtag) { - apiUrl = new URL(`https://${instance}/api/v1/timelines/tag/${encodeURIComponent(hashtag)}`); - apiUrl.searchParams.set("local", isLocal ? "true" : "false"); - } else { - apiUrl = new URL(`https://${instance}/api/v1/timelines/public`); - apiUrl.searchParams.set("local", isLocal ? "true" : "false"); - } - apiUrl.searchParams.set("limit", String(MAX_RESULTS)); - if (maxId) apiUrl.searchParams.set("max_id", maxId); - let items = []; let nextMaxId = null; let error = null; try { - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS); - - const fetchRes = await fetch(apiUrl.toString(), { - headers: { Accept: "application/json" }, - signal: controller.signal, - }); - clearTimeout(timeoutId); - - if (!fetchRes.ok) { - throw new Error(`Remote instance returned HTTP ${fetchRes.status}`); - } - - const statuses = await fetchRes.json(); - - if (!Array.isArray(statuses)) { - throw new Error("Unexpected API response format"); - } - - items = statuses.map((s) => mapMastodonStatusToItem(s, instance)); - - // Get next max_id from last item for pagination - if (statuses.length === MAX_RESULTS && statuses.length > 0) { - const last = statuses[statuses.length - 1]; - nextMaxId = last.id || null; - } + const result = await fetchMastodonTimeline(instance, { scope, hashtag, maxId }); + const processed = await postProcessItems(result.items); + items = processed.items; + nextMaxId = result.nextMaxId; } catch (fetchError) { const msg = fetchError.name === "AbortError" ? response.locals.__("activitypub.reader.explore.timeout") @@ -121,7 +138,6 @@ export function exploreController(mountPath) { error, mountPath, csrfToken, - // Pass empty interactionMap — explore posts are not in our DB interactionMap: {}, }); } catch (error) { @@ -148,73 +164,18 @@ export function exploreApiController(mountPath) { return response.status(400).json({ error: "Invalid instance" }); } - // Build API URL: hashtag timeline or public timeline - const isLocal = scope === "local"; - let apiUrl; - if (hashtag) { - apiUrl = new URL(`https://${instance}/api/v1/timelines/tag/${encodeURIComponent(hashtag)}`); - apiUrl.searchParams.set("local", isLocal ? "true" : "false"); - } else { - apiUrl = new URL(`https://${instance}/api/v1/timelines/public`); - apiUrl.searchParams.set("local", isLocal ? "true" : "false"); - } - apiUrl.searchParams.set("limit", String(MAX_RESULTS)); - if (maxId) apiUrl.searchParams.set("max_id", maxId); + const { items: rawItems, nextMaxId } = await fetchMastodonTimeline(instance, { scope, hashtag, maxId }); + const { items } = await postProcessItems(rawItems); - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS); - - const fetchRes = await fetch(apiUrl.toString(), { - headers: { Accept: "application/json" }, - signal: controller.signal, - }); - clearTimeout(timeoutId); - - if (!fetchRes.ok) { - return response.status(502).json({ error: `Remote returned ${fetchRes.status}` }); - } - - const statuses = await fetchRes.json(); - if (!Array.isArray(statuses)) { - return response.status(502).json({ error: "Unexpected API response" }); - } - - const items = statuses.map((s) => mapMastodonStatusToItem(s, instance)); - - let nextMaxId = null; - if (statuses.length === MAX_RESULTS && statuses.length > 0) { - const last = statuses[statuses.length - 1]; - nextMaxId = last.id || null; - } - - // Render each card server-side const csrfToken = getToken(request.session); - const templateData = { + const html = await renderItemCards(items, request, { ...response.locals, mountPath, csrfToken, interactionMap: {}, - }; - - const htmlParts = await Promise.all( - items.map((item) => { - return new Promise((resolve, reject) => { - request.app.render( - "partials/ap-item-card.njk", - { ...templateData, item }, - (err, html) => { - if (err) reject(err); - else resolve(html); - } - ); - }); - }) - ); - - response.json({ - html: htmlParts.join(""), - maxId: nextMaxId, }); + + response.json({ html, maxId: nextMaxId }); } catch (error) { next(error); } diff --git a/lib/controllers/hashtag-explore.js b/lib/controllers/hashtag-explore.js index cfc39f9..6d509ef 100644 --- a/lib/controllers/hashtag-explore.js +++ b/lib/controllers/hashtag-explore.js @@ -18,6 +18,7 @@ import { validateHashtag, mapMastodonStatusToItem } from "./explore-utils.js"; import { getToken } from "../csrf.js"; +import { postProcessItems, renderItemCards } from "../item-processing.js"; const FETCH_TIMEOUT_MS = 10_000; const PAGE_SIZE = 20; @@ -179,39 +180,26 @@ export function hashtagExploreApiController(mountPath) { const pageItems = dedupedItems.slice(0, PAGE_SIZE); // Map to timeline item format - const items = pageItems.map(({ status, domain }) => + const rawItems = pageItems.map(({ status, domain }) => mapMastodonStatusToItem(status, domain) ); + // Shared processing pipeline (quote stripping, etc.) + const { items } = await postProcessItems(rawItems); + // Render HTML AFTER merge/dedup/paginate (don't waste CPU on discarded items) const csrfToken = getToken(request.session); - const templateData = { + const html = await renderItemCards(items, request, { ...response.locals, mountPath, csrfToken, interactionMap: {}, - }; - - const htmlParts = await Promise.all( - items.map( - (item) => - new Promise((resolve, reject) => { - request.app.render( - "partials/ap-item-card.njk", - { ...templateData, item }, - (err, html) => { - if (err) reject(err); - else resolve(html); - } - ); - }) - ) - ); + }); const instanceLabels = instanceTabs.map((t) => t.domain); response.json({ - html: htmlParts.join(""), + html, cursors: updatedCursors, sources, instancesQueried: instanceTabs.length, diff --git a/lib/controllers/post-detail.js b/lib/controllers/post-detail.js index c48af5e..90acafd 100644 --- a/lib/controllers/post-detail.js +++ b/lib/controllers/post-detail.js @@ -1,9 +1,9 @@ // Post detail controller — view individual AP posts/notes/articles import { Article, Note, Person, Service, Application } from "@fedify/fedify/vocab"; import { getToken } from "../csrf.js"; -import { extractObjectData } from "../timeline-store.js"; +import { extractObjectData, extractActorInfo } from "../timeline-store.js"; import { getCached, setCache } from "../lookup-cache.js"; -import { fetchAndStoreQuote } from "../og-unfurl.js"; +import { fetchAndStoreQuote, stripQuoteReferenceHtml } from "../og-unfurl.js"; // Load parent posts (inReplyTo chain) up to maxDepth levels async function loadParentChain(ctx, documentLoader, timelineCol, parentUrl, maxDepth = 5) { @@ -332,6 +332,20 @@ export function postDetailController(mountPath, plugin) { if (quoteObject) { const quoteData = await extractObjectData(quoteObject, { documentLoader: qLoader }); + + // If author photo is empty, try fetching the actor directly + if (!quoteData.author.photo && quoteData.author.url) { + try { + const actor = await qCtx.lookupObject(new URL(quoteData.author.url), { documentLoader: qLoader }); + if (actor) { + const actorInfo = await extractActorInfo(actor, { documentLoader: qLoader }); + if (actorInfo.photo) quoteData.author.photo = actorInfo.photo; + } + } catch { + // Actor fetch failed — keep existing author data + } + } + timelineItem.quote = { url: quoteData.url || quoteData.uid, uid: quoteData.uid, @@ -342,11 +356,24 @@ export function postDetailController(mountPath, plugin) { photo: quoteData.photo?.slice(0, 1) || [], }; + // Strip RE: paragraph from parent content + const quoteRef = timelineItem.quoteUrl || timelineItem.quote.url || timelineItem.quote.uid; + if (timelineItem.content?.html && quoteRef) { + timelineItem.content.html = stripQuoteReferenceHtml( + timelineItem.content.html, + quoteRef, + ); + } + // Persist for future requests (fire-and-forget) if (timelineCol) { + const persistUpdate = { $set: { quote: timelineItem.quote } }; + if (timelineItem.content?.html) { + persistUpdate.$set["content.html"] = timelineItem.content.html; + } timelineCol.updateOne( { $or: [{ uid: objectUrl }, { url: objectUrl }] }, - { $set: { quote: timelineItem.quote } }, + persistUpdate, ).catch(() => {}); } } @@ -355,6 +382,14 @@ export function postDetailController(mountPath, plugin) { } } + // Strip RE: paragraph for items with existing quote data (render-time cleanup) + if (timelineItem.quote && timelineItem.content?.html) { + const quoteRef = timelineItem.quoteUrl || timelineItem.quote.url || timelineItem.quote.uid; + if (quoteRef) { + timelineItem.content.html = stripQuoteReferenceHtml(timelineItem.content.html, quoteRef); + } + } + const csrfToken = getToken(request.session); response.render("activitypub-post-detail", { diff --git a/lib/controllers/reader.js b/lib/controllers/reader.js index fed54f2..f330276 100644 --- a/lib/controllers/reader.js +++ b/lib/controllers/reader.js @@ -12,13 +12,8 @@ import { deleteNotification, } from "../storage/notifications.js"; import { getToken, validateToken } from "../csrf.js"; -import { - getMutedUrls, - getMutedKeywords, - getBlockedUrls, - getFilterMode, -} from "../storage/moderation.js"; import { getFollowedTags } from "../storage/followed-tags.js"; +import { postProcessItems, applyTabFilter, loadModerationData } from "../item-processing.js"; // Re-export controllers from split modules for backward compatibility export { @@ -54,7 +49,7 @@ export function readerController(mountPath) { // Build query options const options = { before, after, limit, unread }; - // Tab filtering + // Tab filtering at storage level if (tab === "notes") { options.type = "note"; options.excludeReplies = true; @@ -67,82 +62,21 @@ export function readerController(mountPath) { // Get timeline items const result = await getTimelineItems(collections, options); - // Apply client-side filtering for tabs not supported by storage layer - let items = result.items; - if (tab === "replies") { - items = items.filter((item) => item.inReplyTo); - } else if (tab === "media") { - items = items.filter( - (item) => - (item.photo && item.photo.length > 0) || - (item.video && item.video.length > 0) || - (item.audio && item.audio.length > 0), - ); - } + // Tab filtering for types not supported by storage layer + const tabFiltered = applyTabFilter(result.items, tab); - // Apply moderation filters (muted actors, keywords, blocked actors) + // Load moderation data + interactions, apply shared pipeline const modCollections = { ap_muted: application?.collections?.get("ap_muted"), ap_blocked: application?.collections?.get("ap_blocked"), ap_profile: application?.collections?.get("ap_profile"), }; - const [mutedUrls, mutedKeywords, blockedUrls, filterMode] = - await Promise.all([ - getMutedUrls(modCollections), - getMutedKeywords(modCollections), - getBlockedUrls(modCollections), - getFilterMode(modCollections), - ]); - const blockedSet = new Set(blockedUrls); - const mutedSet = new Set(mutedUrls); + const moderation = await loadModerationData(modCollections); - if (blockedSet.size > 0 || mutedSet.size > 0 || mutedKeywords.length > 0) { - items = items.filter((item) => { - // Blocked actors are ALWAYS hidden - if (item.author?.url && blockedSet.has(item.author.url)) { - return false; - } - - // Check muted actor - const isMutedActor = - item.author?.url && mutedSet.has(item.author.url); - - // Check muted keywords against content, title, and summary - let matchedKeyword = null; - if (mutedKeywords.length > 0) { - const searchable = [ - item.content?.text, - item.name, - item.summary, - ] - .filter(Boolean) - .join(" ") - .toLowerCase(); - if (searchable) { - matchedKeyword = mutedKeywords.find((kw) => - searchable.includes(kw.toLowerCase()), - ); - } - } - - if (isMutedActor || matchedKeyword) { - if (filterMode === "warn") { - // Mark for content warning instead of hiding - item._moderated = true; - item._moderationReason = isMutedActor - ? "muted_account" - : "muted_keyword"; - if (matchedKeyword) { - item._moderationKeyword = matchedKeyword; - } - return true; - } - return false; - } - - return true; - }); - } + const { items, interactionMap } = await postProcessItems(tabFiltered, { + moderation, + interactionsCol: application?.collections?.get("ap_interactions"), + }); // Get unread notification count for badge + unread timeline count for toggle const [unreadCount, unreadTimelineCount] = await Promise.all([ @@ -150,52 +84,6 @@ export function readerController(mountPath) { countUnreadItems(collections), ]); - // Get interaction state for liked/boosted indicators - // Interactions are keyed by canonical AP uid (new) or display url (legacy). - // Query by both, normalize map keys to uid for template lookup. - const interactionsCol = - application?.collections?.get("ap_interactions"); - const interactionMap = {}; - - if (interactionsCol) { - const lookupUrls = new Set(); - const objectUrlToUid = new Map(); - - for (const item of items) { - const uid = item.uid; - const displayUrl = item.url || item.originalUrl; - - if (uid) { - lookupUrls.add(uid); - objectUrlToUid.set(uid, uid); - } - - if (displayUrl) { - lookupUrls.add(displayUrl); - objectUrlToUid.set(displayUrl, uid || displayUrl); - } - } - - if (lookupUrls.size > 0) { - const interactions = await interactionsCol - .find({ objectUrl: { $in: [...lookupUrls] } }) - .toArray(); - - for (const interaction of interactions) { - // Normalize to uid so template can look up by itemUid - const key = - objectUrlToUid.get(interaction.objectUrl) || - interaction.objectUrl; - - if (!interactionMap[key]) { - interactionMap[key] = {}; - } - - interactionMap[key][interaction.type] = true; - } - } - } - // CSRF token for interaction forms const csrfToken = getToken(request.session); diff --git a/lib/controllers/tag-timeline.js b/lib/controllers/tag-timeline.js index cd0ac3d..b61b200 100644 --- a/lib/controllers/tag-timeline.js +++ b/lib/controllers/tag-timeline.js @@ -4,12 +4,7 @@ import { getTimelineItems } from "../storage/timeline.js"; import { getToken } from "../csrf.js"; -import { - getMutedUrls, - getMutedKeywords, - getBlockedUrls, - getFilterMode, -} from "../storage/moderation.js"; +import { postProcessItems, loadModerationData } from "../item-processing.js"; export function tagTimelineController(mountPath) { return async (request, response, next) => { @@ -36,88 +31,21 @@ export function tagTimelineController(mountPath) { // Get timeline items filtered by tag const result = await getTimelineItems(collections, { before, after, limit, tag }); - let items = result.items; - // Apply moderation filters (same as main reader) + // Shared processing pipeline: moderation, quote stripping, interactions const modCollections = { ap_muted: application?.collections?.get("ap_muted"), ap_blocked: application?.collections?.get("ap_blocked"), ap_profile: application?.collections?.get("ap_profile"), }; - const [mutedUrls, mutedKeywords, blockedUrls, filterMode] = - await Promise.all([ - getMutedUrls(modCollections), - getMutedKeywords(modCollections), - getBlockedUrls(modCollections), - getFilterMode(modCollections), - ]); - const blockedSet = new Set(blockedUrls); - const mutedSet = new Set(mutedUrls); + const moderation = await loadModerationData(modCollections); - if (blockedSet.size > 0 || mutedSet.size > 0 || mutedKeywords.length > 0) { - items = items.filter((item) => { - if (item.author?.url && blockedSet.has(item.author.url)) { - return false; - } + const { items, interactionMap } = await postProcessItems(result.items, { + moderation, + interactionsCol: application?.collections?.get("ap_interactions"), + }); - const isMutedActor = item.author?.url && mutedSet.has(item.author.url); - - let matchedKeyword = null; - if (mutedKeywords.length > 0) { - const searchable = [item.content?.text, item.name, item.summary] - .filter(Boolean) - .join(" ") - .toLowerCase(); - if (searchable) { - matchedKeyword = mutedKeywords.find((kw) => - searchable.includes(kw.toLowerCase()) - ); - } - } - - if (isMutedActor || matchedKeyword) { - if (filterMode === "warn") { - item._moderated = true; - item._moderationReason = isMutedActor ? "muted_account" : "muted_keyword"; - if (matchedKeyword) item._moderationKeyword = matchedKeyword; - return true; - } - return false; - } - - return true; - }); - } - - // Get interaction state for liked/boosted indicators - const interactionsCol = application?.collections?.get("ap_interactions"); - const interactionMap = {}; - - if (interactionsCol) { - const lookupUrls = new Set(); - const objectUrlToUid = new Map(); - - for (const item of items) { - const uid = item.uid; - const displayUrl = item.url || item.originalUrl; - if (uid) { lookupUrls.add(uid); objectUrlToUid.set(uid, uid); } - if (displayUrl) { lookupUrls.add(displayUrl); objectUrlToUid.set(displayUrl, uid || displayUrl); } - } - - if (lookupUrls.size > 0) { - const interactions = await interactionsCol - .find({ objectUrl: { $in: [...lookupUrls] } }) - .toArray(); - - for (const interaction of interactions) { - const key = objectUrlToUid.get(interaction.objectUrl) || interaction.objectUrl; - if (!interactionMap[key]) interactionMap[key] = {}; - interactionMap[key][interaction.type] = true; - } - } - } - - // Check if this hashtag is followed (Task 7 will populate ap_followed_tags) + // Check if this hashtag is followed const followedTagsCol = application?.collections?.get("ap_followed_tags"); let isFollowed = false; if (followedTagsCol) { diff --git a/lib/item-processing.js b/lib/item-processing.js new file mode 100644 index 0000000..cf96a6f --- /dev/null +++ b/lib/item-processing.js @@ -0,0 +1,226 @@ +/** + * Shared item processing pipeline for the ActivityPub reader. + * + * Both the reader (inbox-sourced items) and explore (Mastodon API items) + * flow through these functions. This ensures every enhancement (emoji, + * quote stripping, moderation, etc.) is implemented once. + */ + +import { stripQuoteReferenceHtml } from "./og-unfurl.js"; + +/** + * Post-process timeline items for rendering. + * Called after items are loaded from any source (MongoDB or Mastodon API). + * + * @param {Array} items - Timeline items (from DB or Mastodon API mapping) + * @param {object} [options] + * @param {object} [options.moderation] - { mutedUrls, mutedKeywords, blockedUrls, filterMode } + * @param {object} [options.interactionsCol] - MongoDB collection for interaction state + * @returns {Promise<{ items: Array, interactionMap: object }>} + */ +export async function postProcessItems(items, options = {}) { + // 1. Moderation filters (muted actors, keywords, blocked actors) + if (options.moderation) { + items = applyModerationFilters(items, options.moderation); + } + + // 2. Strip "RE:" paragraphs from items with quote embeds + stripQuoteReferences(items); + + // 3. Build interaction map (likes/boosts) — empty when no collection + const interactionMap = options.interactionsCol + ? await buildInteractionMap(items, options.interactionsCol) + : {}; + + return { items, interactionMap }; +} + +/** + * Apply moderation filters to items. + * Blocked actors are always hidden. Muted actors/keywords are hidden or + * marked with a content warning depending on filterMode. + * + * @param {Array} items + * @param {object} moderation + * @param {string[]} moderation.mutedUrls + * @param {string[]} moderation.mutedKeywords + * @param {string[]} moderation.blockedUrls + * @param {string} moderation.filterMode - "hide" or "warn" + * @returns {Array} + */ +export function applyModerationFilters(items, { mutedUrls, mutedKeywords, blockedUrls, filterMode }) { + const blockedSet = new Set(blockedUrls); + const mutedSet = new Set(mutedUrls); + + if (blockedSet.size === 0 && mutedSet.size === 0 && mutedKeywords.length === 0) { + return items; + } + + return items.filter((item) => { + // Blocked actors are ALWAYS hidden + if (item.author?.url && blockedSet.has(item.author.url)) { + return false; + } + + // Check muted actor + const isMutedActor = item.author?.url && mutedSet.has(item.author.url); + + // Check muted keywords against content, title, and summary + let matchedKeyword = null; + if (mutedKeywords.length > 0) { + const searchable = [item.content?.text, item.name, item.summary] + .filter(Boolean) + .join(" ") + .toLowerCase(); + if (searchable) { + matchedKeyword = mutedKeywords.find((kw) => + searchable.includes(kw.toLowerCase()), + ); + } + } + + if (isMutedActor || matchedKeyword) { + if (filterMode === "warn") { + // Mark for content warning instead of hiding + item._moderated = true; + item._moderationReason = isMutedActor ? "muted_account" : "muted_keyword"; + if (matchedKeyword) { + item._moderationKeyword = matchedKeyword; + } + return true; + } + return false; + } + + return true; + }); +} + +/** + * Strip "RE:" quote reference paragraphs from items that have quote embeds. + * Mutates items in place. + * + * @param {Array} items + */ +export function stripQuoteReferences(items) { + for (const item of items) { + const quoteRef = item.quoteUrl || item.quote?.url || item.quote?.uid; + if (item.quote && quoteRef && item.content?.html) { + item.content.html = stripQuoteReferenceHtml(item.content.html, quoteRef); + } + } +} + +/** + * Build interaction map (likes/boosts) for template rendering. + * Returns { [uid]: { like: true, boost: true } }. + * + * @param {Array} items + * @param {object} interactionsCol - MongoDB collection + * @returns {Promise} + */ +export async function buildInteractionMap(items, interactionsCol) { + const interactionMap = {}; + const lookupUrls = new Set(); + const objectUrlToUid = new Map(); + + for (const item of items) { + const uid = item.uid; + const displayUrl = item.url || item.originalUrl; + + if (uid) { + lookupUrls.add(uid); + objectUrlToUid.set(uid, uid); + } + if (displayUrl) { + lookupUrls.add(displayUrl); + objectUrlToUid.set(displayUrl, uid || displayUrl); + } + } + + if (lookupUrls.size === 0) return interactionMap; + + const interactions = await interactionsCol + .find({ objectUrl: { $in: [...lookupUrls] } }) + .toArray(); + + for (const interaction of interactions) { + const key = objectUrlToUid.get(interaction.objectUrl) || interaction.objectUrl; + if (!interactionMap[key]) interactionMap[key] = {}; + interactionMap[key][interaction.type] = true; + } + + return interactionMap; +} + +/** + * Filter items by tab type (reader-specific). + * + * @param {Array} items + * @param {string} tab - "notes", "articles", "boosts", "replies", "media", "all" + * @returns {Array} + */ +export function applyTabFilter(items, tab) { + if (tab === "replies") { + return items.filter((item) => item.inReplyTo); + } + if (tab === "media") { + return items.filter( + (item) => + (item.photo && item.photo.length > 0) || + (item.video && item.video.length > 0) || + (item.audio && item.audio.length > 0), + ); + } + return items; +} + +/** + * Render items to HTML using ap-item-card.njk. + * Used by all API endpoints that return pre-rendered card HTML. + * + * @param {Array} items + * @param {object} request - Express request (for app.render) + * @param {object} templateData - Merged template context (locals, mountPath, csrfToken, interactionMap) + * @returns {Promise} + */ +export async function renderItemCards(items, request, templateData) { + const htmlParts = await Promise.all( + items.map( + (item) => + new Promise((resolve, reject) => { + request.app.render( + "partials/ap-item-card.njk", + { ...templateData, item }, + (err, html) => { + if (err) reject(err); + else resolve(html); + }, + ); + }), + ), + ); + return htmlParts.join(""); +} + +/** + * Load moderation data from MongoDB collections. + * Convenience wrapper to reduce boilerplate in controllers. + * + * @param {object} modCollections - { ap_muted, ap_blocked, ap_profile } + * @returns {Promise} moderation data for postProcessItems() + */ +export async function loadModerationData(modCollections) { + // Dynamic import to avoid circular dependency + const { getMutedUrls, getMutedKeywords, getBlockedUrls, getFilterMode } = + await import("./storage/moderation.js"); + + const [mutedUrls, mutedKeywords, blockedUrls, filterMode] = await Promise.all([ + getMutedUrls(modCollections), + getMutedKeywords(modCollections), + getBlockedUrls(modCollections), + getFilterMode(modCollections), + ]); + + return { mutedUrls, mutedKeywords, blockedUrls, filterMode }; +} diff --git a/lib/og-unfurl.js b/lib/og-unfurl.js index 9fb8c86..0d219eb 100644 --- a/lib/og-unfurl.js +++ b/lib/og-unfurl.js @@ -267,6 +267,22 @@ export async function fetchAndStoreQuote(collections, uid, quoteUrl, ctx, docume const quoteData = await extractObjectData(object, { documentLoader }); + // If author photo is empty, try fetching the actor directly + if (!quoteData.author.photo && quoteData.author.url) { + try { + const actor = await ctx.lookupObject(new URL(quoteData.author.url), { documentLoader }); + if (actor) { + const { extractActorInfo } = await import("./timeline-store.js"); + const actorInfo = await extractActorInfo(actor, { documentLoader }); + if (actorInfo.photo) { + quoteData.author.photo = actorInfo.photo; + } + } + } catch { + // Actor fetch failed — keep existing author data + } + } + const quote = { url: quoteData.url || quoteData.uid, uid: quoteData.uid, @@ -277,11 +293,46 @@ export async function fetchAndStoreQuote(collections, uid, quoteUrl, ctx, docume photo: quoteData.photo?.slice(0, 1) || [], }; - await collections.ap_timeline.updateOne( - { uid }, - { $set: { quote } }, - ); + // Strip the "RE: " paragraph from the parent post's content + // Mastodon adds this as:

RE: ...

+ const update = { $set: { quote } }; + const parentItem = await collections.ap_timeline.findOne({ uid }); + if (parentItem?.content?.html) { + const cleaned = stripQuoteReferenceHtml(parentItem.content.html, quoteUrl); + if (cleaned !== parentItem.content.html) { + update.$set["content.html"] = cleaned; + } + } + + await collections.ap_timeline.updateOne({ uid }, update); } catch (error) { console.error(`[og-unfurl] Failed to fetch quote for ${uid}: ${error.message}`); } } + +/** + * Strip the "RE: " paragraph that Mastodon adds for quoted posts. + * Removes

elements containing "RE:" followed by a link to the quote URL. + * @param {string} html - Content HTML + * @param {string} quoteUrl - URL of the quoted post + * @returns {string} Cleaned HTML + */ +export function stripQuoteReferenceHtml(html, quoteUrl) { + if (!html || !quoteUrl) return html; + // Match

containing "RE:" and a link whose href contains the quote domain+path + // Mastodon uses both /users/X/statuses/Y and /@X/Y URL formats + try { + const quoteUrlObj = new URL(quoteUrl); + const quoteDomain = quoteUrlObj.hostname; + // Escape special regex chars in domain + const domainEscaped = quoteDomain.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + // Match

RE: ...

(with optional whitespace) + const re = new RegExp( + `

\\s*RE:\\s*]*href="[^"]*${domainEscaped}[^"]*"[^>]*>.*?\\s*

`, + "i", + ); + return html.replace(re, "").trim(); + } catch { + return html; + } +} diff --git a/package.json b/package.json index 3173d9d..0c242b5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@rmdes/indiekit-endpoint-activitypub", - "version": "2.4.1", + "version": "2.5.0", "description": "ActivityPub federation endpoint for Indiekit via Fedify. Adds full fediverse support: actor, inbox, outbox, followers, following, syndication, and Mastodon migration.", "keywords": [ "indiekit", diff --git a/views/activitypub-explore.njk b/views/activitypub-explore.njk index 90cd224..0c89b76 100644 --- a/views/activitypub-explore.njk +++ b/views/activitypub-explore.njk @@ -247,11 +247,13 @@ {% if maxId %}
- {# Inline loading spinner for subsequent pages #} -
- {{ __("activitypub.reader.pagination.loading") }} + {# Load more button + loading spinner for subsequent pages #} +
+ + + {{ __("activitypub.reader.pagination.loading") }} +
{# Empty state — loaded successfully but no posts #} @@ -357,10 +368,19 @@ x-html="tabState[tab._id] ? tabState[tab._id].html : ''">
- {# Inline loading spinner for subsequent pages #} -
- {{ __("activitypub.reader.pagination.loading") }} + {# Load more button + loading spinner for subsequent pages #} +
+ + + {{ __("activitypub.reader.pagination.loading") }} +
{# Empty state — no instance tabs pinned yet #} diff --git a/views/activitypub-reader.njk b/views/activitypub-reader.njk index 14c4ac6..b84100a 100644 --- a/views/activitypub-reader.njk +++ b/views/activitypub-reader.njk @@ -140,12 +140,15 @@ {% if before %}
+ x-init="init()">