refactor: unify reader and explore processing pipeline (Release 0)

Extract shared item-processing.js module with postProcessItems(),
applyModerationFilters(), buildInteractionMap(), applyTabFilter(),
renderItemCards(), and loadModerationData(). All controllers (reader,
api-timeline, explore, hashtag-explore, tag-timeline) now flow through
the same pipeline.

Unify Alpine.js infinite scroll into single parameterized
apInfiniteScroll component configured via data attributes, replacing
the separate apExploreScroll component.

Also adds fetchAndStoreQuote() for quote enrichment and on-demand
quote fetching in post-detail controller.

Bump version to 2.5.0.

Confab-Link: http://localhost:8080/sessions/e9d666ac-3c90-4298-9e92-9ac9d142bc06
This commit is contained in:
Ricardo
2026-03-03 12:48:40 +01:00
parent 9fa3412875
commit af2f899073
16 changed files with 585 additions and 589 deletions

View File

@@ -1,121 +1,55 @@
/**
* Infinite scroll — AlpineJS component for AJAX load-more on the timeline
* Registers the `apInfiniteScroll` Alpine data component.
* Infinite scroll — unified AlpineJS component for AJAX load-more.
* Works for both reader timeline and explore view via data attributes.
*
* Required data attributes on the component element:
* data-cursor — initial pagination cursor value
* data-api-url — API endpoint URL (e.g., /activitypub/admin/reader/api/timeline)
* data-cursor-param — query param name for the cursor (e.g., "before" or "max_id")
* data-cursor-field — response JSON field for the next cursor (e.g., "before" or "maxId")
* data-timeline-id — DOM ID of the timeline container to append HTML into
*
* Optional:
* data-extra-params — JSON-encoded object of additional query params
* data-hide-pagination — CSS selector of no-JS pagination to hide
*/
document.addEventListener("alpine:init", () => {
// eslint-disable-next-line no-undef
Alpine.data("apExploreScroll", () => ({
loading: false,
done: false,
maxId: null,
instance: "",
scope: "local",
hashtag: "",
observer: null,
init() {
const el = this.$el;
this.maxId = el.dataset.maxId || null;
this.instance = el.dataset.instance || "";
this.scope = el.dataset.scope || "local";
this.hashtag = el.dataset.hashtag || "";
if (!this.maxId) {
this.done = true;
return;
}
this.observer = new IntersectionObserver(
(entries) => {
for (const entry of entries) {
if (entry.isIntersecting && !this.loading && !this.done) {
this.loadMore();
}
}
},
{ rootMargin: "200px" }
);
if (this.$refs.sentinel) {
this.observer.observe(this.$refs.sentinel);
}
},
async loadMore() {
if (this.loading || this.done || !this.maxId) return;
this.loading = true;
const timeline = document.getElementById("ap-explore-timeline");
const mountPath = timeline ? timeline.dataset.mountPath : "";
const params = new URLSearchParams({
instance: this.instance,
scope: this.scope,
max_id: this.maxId,
});
// Pass hashtag when in hashtag mode so infinite scroll stays on tag timeline
if (this.hashtag) {
params.set("hashtag", this.hashtag);
}
try {
const res = await fetch(
`${mountPath}/admin/reader/api/explore?${params.toString()}`,
{ headers: { Accept: "application/json" } }
);
if (!res.ok) throw new Error(`HTTP ${res.status}`);
const data = await res.json();
if (data.html && timeline) {
timeline.insertAdjacentHTML("beforeend", data.html);
}
if (data.maxId) {
this.maxId = data.maxId;
} else {
this.done = true;
if (this.observer) this.observer.disconnect();
}
} catch (err) {
console.error("[ap-explore-scroll] load failed:", err.message);
} finally {
this.loading = false;
}
},
destroy() {
if (this.observer) this.observer.disconnect();
},
}));
// eslint-disable-next-line no-undef
Alpine.data("apInfiniteScroll", () => ({
loading: false,
done: false,
before: null,
tab: "",
tag: "",
cursor: null,
apiUrl: "",
cursorParam: "before",
cursorField: "before",
timelineId: "",
extraParams: {},
observer: null,
init() {
const el = this.$el;
this.before = el.dataset.before || null;
this.tab = el.dataset.tab || "";
this.tag = el.dataset.tag || "";
this.cursor = el.dataset.cursor || null;
this.apiUrl = el.dataset.apiUrl || "";
this.cursorParam = el.dataset.cursorParam || "before";
this.cursorField = el.dataset.cursorField || "before";
this.timelineId = el.dataset.timelineId || "";
// Hide the no-JS pagination fallback now that JS is active
const paginationEl =
document.getElementById("ap-reader-pagination") ||
document.getElementById("ap-tag-pagination");
if (paginationEl) {
paginationEl.style.display = "none";
// Parse extra params from JSON data attribute
try {
this.extraParams = JSON.parse(el.dataset.extraParams || "{}");
} catch {
this.extraParams = {};
}
if (!this.before) {
// Hide the no-JS pagination fallback now that JS is active
const hideSel = el.dataset.hidePagination;
if (hideSel) {
const paginationEl = document.getElementById(hideSel);
if (paginationEl) paginationEl.style.display = "none";
}
if (!this.cursor) {
this.done = true;
return;
}
@@ -129,7 +63,7 @@ document.addEventListener("alpine:init", () => {
}
}
},
{ rootMargin: "200px" }
{ rootMargin: "200px" },
);
if (this.$refs.sentinel) {
@@ -138,36 +72,36 @@ document.addEventListener("alpine:init", () => {
},
async loadMore() {
if (this.loading || this.done || !this.before) return;
if (this.loading || this.done || !this.cursor) return;
this.loading = true;
const timeline = document.getElementById("ap-timeline");
const mountPath = timeline ? timeline.dataset.mountPath : "";
const params = new URLSearchParams({ before: this.before });
if (this.tab) params.set("tab", this.tab);
if (this.tag) params.set("tag", this.tag);
const params = new URLSearchParams({
[this.cursorParam]: this.cursor,
...this.extraParams,
});
try {
const res = await fetch(
`${mountPath}/admin/reader/api/timeline?${params.toString()}`,
{ headers: { Accept: "application/json" } }
`${this.apiUrl}?${params.toString()}`,
{ headers: { Accept: "application/json" } },
);
if (!res.ok) throw new Error(`HTTP ${res.status}`);
const data = await res.json();
const timeline = this.timelineId
? document.getElementById(this.timelineId)
: null;
if (data.html && timeline) {
// Append the returned pre-rendered HTML
timeline.insertAdjacentHTML("beforeend", data.html);
}
if (data.before) {
this.before = data.before;
if (data[this.cursorField]) {
this.cursor = data[this.cursorField];
} else {
// No more items
this.done = true;
if (this.observer) this.observer.disconnect();
}
@@ -178,10 +112,6 @@ document.addEventListener("alpine:init", () => {
}
},
appendItems(/* detail */) {
// Custom event hook — not used in this implementation but kept for extensibility
},
destroy() {
if (this.observer) this.observer.disconnect();
},
@@ -282,7 +212,9 @@ document.addEventListener("alpine:init", () => {
const card = entry.target;
const uid = card.dataset.uid;
if (uid && !card.classList.contains("ap-card--read")) {
card.classList.add("ap-card--read");
// Mark read server-side but DON'T dim visually in this session.
// Cards only appear dimmed when they arrive from the server
// with item.read=true on a subsequent page load.
this._batch.push(uid);
}
this._observer.unobserve(card);

View File

@@ -366,6 +366,16 @@ document.addEventListener("alpine:init", () => {
}
},
// ── Public load-more method (called by button click) ────────────────────
loadMoreTab(tab) {
if (tab.type === "instance") {
this._loadMoreInstanceTab(tab);
} else if (tab.type === "hashtag") {
this._loadMoreHashtagTab(tab);
}
},
// ── Infinite scroll for tab panels ───────────────────────────────────────
_setupScrollObserver(tab) {

View File

@@ -2486,15 +2486,21 @@
}
.ap-quote-embed__content {
-webkit-box-orient: vertical;
-webkit-line-clamp: 6;
color: var(--color-on-background);
display: -webkit-box;
font-size: var(--font-size-s);
line-height: 1.5;
max-height: calc(1.5em * 6);
overflow: hidden;
}
.ap-quote-embed__content a {
display: inline;
}
.ap-quote-embed__content a span {
display: inline;
}
.ap-quote-embed__content p {
margin: 0 0 var(--space-xs);
}

View File

@@ -4,12 +4,7 @@
import { getTimelineItems, countNewItems, markItemsRead } from "../storage/timeline.js";
import { getToken, validateToken } from "../csrf.js";
import {
getMutedUrls,
getMutedKeywords,
getBlockedUrls,
getFilterMode,
} from "../storage/moderation.js";
import { postProcessItems, applyTabFilter, loadModerationData, renderItemCards } from "../item-processing.js";
export function apiTimelineController(mountPath) {
return async (request, response, next) => {
@@ -25,7 +20,7 @@ export function apiTimelineController(mountPath) {
const before = request.query.before;
const limit = 20;
// Build storage query options (same logic as readerController)
// Build storage query options
const unread = request.query.unread === "1";
const options = { before, limit, unread };
@@ -44,124 +39,32 @@ export function apiTimelineController(mountPath) {
const result = await getTimelineItems(collections, options);
// Client-side tab filtering for types not supported by storage
let items = result.items;
if (!tag) {
if (tab === "replies") {
items = items.filter((item) => item.inReplyTo);
} else if (tab === "media") {
items = items.filter(
(item) =>
(item.photo && item.photo.length > 0) ||
(item.video && item.video.length > 0) ||
(item.audio && item.audio.length > 0)
);
}
}
// Tab filtering for types not supported by storage layer
const tabFiltered = tag ? result.items : applyTabFilter(result.items, tab);
// Apply moderation filters
// Shared processing pipeline: moderation, quote stripping, interactions
const modCollections = {
ap_muted: application?.collections?.get("ap_muted"),
ap_blocked: application?.collections?.get("ap_blocked"),
ap_profile: application?.collections?.get("ap_profile"),
};
const [mutedUrls, mutedKeywords, blockedUrls, filterMode] =
await Promise.all([
getMutedUrls(modCollections),
getMutedKeywords(modCollections),
getBlockedUrls(modCollections),
getFilterMode(modCollections),
]);
const blockedSet = new Set(blockedUrls);
const mutedSet = new Set(mutedUrls);
const moderation = await loadModerationData(modCollections);
if (blockedSet.size > 0 || mutedSet.size > 0 || mutedKeywords.length > 0) {
items = items.filter((item) => {
if (item.author?.url && blockedSet.has(item.author.url)) {
return false;
}
const isMutedActor = item.author?.url && mutedSet.has(item.author.url);
let matchedKeyword = null;
if (mutedKeywords.length > 0) {
const searchable = [item.content?.text, item.name, item.summary]
.filter(Boolean)
.join(" ")
.toLowerCase();
if (searchable) {
matchedKeyword = mutedKeywords.find((kw) =>
searchable.includes(kw.toLowerCase())
);
}
}
if (isMutedActor || matchedKeyword) {
if (filterMode === "warn") {
item._moderated = true;
item._moderationReason = isMutedActor ? "muted_account" : "muted_keyword";
if (matchedKeyword) item._moderationKeyword = matchedKeyword;
return true;
}
return false;
}
return true;
});
}
// Get interaction state
const interactionsCol = application?.collections?.get("ap_interactions");
const interactionMap = {};
if (interactionsCol) {
const lookupUrls = new Set();
const objectUrlToUid = new Map();
for (const item of items) {
const uid = item.uid;
const displayUrl = item.url || item.originalUrl;
if (uid) { lookupUrls.add(uid); objectUrlToUid.set(uid, uid); }
if (displayUrl) { lookupUrls.add(displayUrl); objectUrlToUid.set(displayUrl, uid || displayUrl); }
}
if (lookupUrls.size > 0) {
const interactions = await interactionsCol
.find({ objectUrl: { $in: [...lookupUrls] } })
.toArray();
for (const interaction of interactions) {
const key = objectUrlToUid.get(interaction.objectUrl) || interaction.objectUrl;
if (!interactionMap[key]) interactionMap[key] = {};
interactionMap[key][interaction.type] = true;
}
}
}
const { items, interactionMap } = await postProcessItems(tabFiltered, {
moderation,
interactionsCol: application?.collections?.get("ap_interactions"),
});
const csrfToken = getToken(request.session);
// Render each card server-side using the same Nunjucks template
// Merge response.locals so that i18n (__), mountPath, etc. are available
const templateData = {
const html = await renderItemCards(items, request, {
...response.locals,
mountPath,
csrfToken,
interactionMap,
};
const htmlParts = await Promise.all(
items.map((item) => {
return new Promise((resolve, reject) => {
request.app.render(
"partials/ap-item-card.njk",
{ ...templateData, item },
(err, html) => {
if (err) reject(err);
else resolve(html);
}
);
});
})
);
});
response.json({
html: htmlParts.join(""),
html,
before: result.before,
});
} catch (error) {

View File

@@ -92,7 +92,7 @@ export function mapMastodonStatusToItem(status, instance) {
}
}
return {
const item = {
uid: status.url || status.uri || "",
url: status.url || status.uri || "",
type: "note",
@@ -119,4 +119,45 @@ export function mapMastodonStatusToItem(status, instance) {
createdAt: new Date().toISOString(),
_explore: true,
};
// Map quoted post data if present (Mastodon 4.3+ quote support)
// Mastodon API wraps the quoted status: { state: "accepted", quoted_status: { ...fullStatus } }
const quotedStatus = status.quote?.quoted_status || null;
if (quotedStatus) {
item.quoteUrl = quotedStatus.url || quotedStatus.uri || "";
const q = quotedStatus;
const qAccount = q.account || {};
const qAcct = qAccount.acct || "";
const qHandle = qAcct.includes("@") ? `@${qAcct}` : `@${qAcct}@${instance}`;
const qPhoto = [];
for (const att of q.media_attachments || []) {
const attUrl = att.url || att.remote_url || "";
if (attUrl && (att.type === "image" || att.type === "gifv")) {
qPhoto.push(attUrl);
}
}
item.quote = {
url: q.url || q.uri || "",
uid: q.uri || q.url || "",
author: {
name: sanitizeHtml(qAccount.display_name || qAccount.username || "Unknown", { allowedTags: [], allowedAttributes: {} }),
url: qAccount.url || "",
photo: qAccount.avatar || qAccount.avatar_static || "",
handle: qHandle,
},
content: {
text: (q.content || "").replace(/<[^>]*>/g, ""),
html: sanitizeContent(q.content || ""),
},
published: q.created_at || "",
name: "",
photo: qPhoto.slice(0, 1),
};
} else {
item.quoteUrl = "";
}
return item;
}

View File

@@ -8,6 +8,7 @@
import { searchInstances, checkInstanceTimeline, getPopularAccounts } from "../fedidb.js";
import { getToken } from "../csrf.js";
import { validateInstance, validateHashtag, mapMastodonStatusToItem } from "./explore-utils.js";
import { postProcessItems, renderItemCards } from "../item-processing.js";
const FETCH_TIMEOUT_MS = 10_000;
const MAX_RESULTS = 20;
@@ -15,6 +16,57 @@ const MAX_RESULTS = 20;
// Re-export validateInstance for backward compatibility (used by tabs.js, index.js)
export { validateInstance } from "./explore-utils.js";
/**
* Fetch statuses from a remote Mastodon-compatible instance.
*
* @param {string} instance - Validated hostname
* @param {object} options
* @param {string} [options.scope] - "local" or "federated"
* @param {string} [options.hashtag] - Validated hashtag (no #)
* @param {string} [options.maxId] - Pagination cursor
* @param {number} [options.limit] - Max results
* @returns {Promise<{ items: Array, nextMaxId: string|null }>}
*/
export async function fetchMastodonTimeline(instance, { scope = "local", hashtag, maxId, limit = MAX_RESULTS } = {}) {
const isLocal = scope === "local";
let apiUrl;
if (hashtag) {
apiUrl = new URL(`https://${instance}/api/v1/timelines/tag/${encodeURIComponent(hashtag)}`);
} else {
apiUrl = new URL(`https://${instance}/api/v1/timelines/public`);
}
apiUrl.searchParams.set("local", isLocal ? "true" : "false");
apiUrl.searchParams.set("limit", String(limit));
if (maxId) apiUrl.searchParams.set("max_id", maxId);
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS);
const fetchRes = await fetch(apiUrl.toString(), {
headers: { Accept: "application/json" },
signal: controller.signal,
});
clearTimeout(timeoutId);
if (!fetchRes.ok) {
throw new Error(`Remote instance returned HTTP ${fetchRes.status}`);
}
const statuses = await fetchRes.json();
if (!Array.isArray(statuses)) {
throw new Error("Unexpected API response format");
}
const items = statuses.map((s) => mapMastodonStatusToItem(s, instance));
const nextMaxId =
statuses.length === limit && statuses.length > 0
? statuses[statuses.length - 1].id || null
: null;
return { items, nextMaxId };
}
export function exploreController(mountPath) {
return async (request, response, next) => {
try {
@@ -59,50 +111,15 @@ export function exploreController(mountPath) {
});
}
// Build API URL: hashtag timeline or public timeline
const isLocal = scope === "local";
let apiUrl;
if (hashtag) {
apiUrl = new URL(`https://${instance}/api/v1/timelines/tag/${encodeURIComponent(hashtag)}`);
apiUrl.searchParams.set("local", isLocal ? "true" : "false");
} else {
apiUrl = new URL(`https://${instance}/api/v1/timelines/public`);
apiUrl.searchParams.set("local", isLocal ? "true" : "false");
}
apiUrl.searchParams.set("limit", String(MAX_RESULTS));
if (maxId) apiUrl.searchParams.set("max_id", maxId);
let items = [];
let nextMaxId = null;
let error = null;
try {
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS);
const fetchRes = await fetch(apiUrl.toString(), {
headers: { Accept: "application/json" },
signal: controller.signal,
});
clearTimeout(timeoutId);
if (!fetchRes.ok) {
throw new Error(`Remote instance returned HTTP ${fetchRes.status}`);
}
const statuses = await fetchRes.json();
if (!Array.isArray(statuses)) {
throw new Error("Unexpected API response format");
}
items = statuses.map((s) => mapMastodonStatusToItem(s, instance));
// Get next max_id from last item for pagination
if (statuses.length === MAX_RESULTS && statuses.length > 0) {
const last = statuses[statuses.length - 1];
nextMaxId = last.id || null;
}
const result = await fetchMastodonTimeline(instance, { scope, hashtag, maxId });
const processed = await postProcessItems(result.items);
items = processed.items;
nextMaxId = result.nextMaxId;
} catch (fetchError) {
const msg = fetchError.name === "AbortError"
? response.locals.__("activitypub.reader.explore.timeout")
@@ -121,7 +138,6 @@ export function exploreController(mountPath) {
error,
mountPath,
csrfToken,
// Pass empty interactionMap — explore posts are not in our DB
interactionMap: {},
});
} catch (error) {
@@ -148,73 +164,18 @@ export function exploreApiController(mountPath) {
return response.status(400).json({ error: "Invalid instance" });
}
// Build API URL: hashtag timeline or public timeline
const isLocal = scope === "local";
let apiUrl;
if (hashtag) {
apiUrl = new URL(`https://${instance}/api/v1/timelines/tag/${encodeURIComponent(hashtag)}`);
apiUrl.searchParams.set("local", isLocal ? "true" : "false");
} else {
apiUrl = new URL(`https://${instance}/api/v1/timelines/public`);
apiUrl.searchParams.set("local", isLocal ? "true" : "false");
}
apiUrl.searchParams.set("limit", String(MAX_RESULTS));
if (maxId) apiUrl.searchParams.set("max_id", maxId);
const { items: rawItems, nextMaxId } = await fetchMastodonTimeline(instance, { scope, hashtag, maxId });
const { items } = await postProcessItems(rawItems);
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS);
const fetchRes = await fetch(apiUrl.toString(), {
headers: { Accept: "application/json" },
signal: controller.signal,
});
clearTimeout(timeoutId);
if (!fetchRes.ok) {
return response.status(502).json({ error: `Remote returned ${fetchRes.status}` });
}
const statuses = await fetchRes.json();
if (!Array.isArray(statuses)) {
return response.status(502).json({ error: "Unexpected API response" });
}
const items = statuses.map((s) => mapMastodonStatusToItem(s, instance));
let nextMaxId = null;
if (statuses.length === MAX_RESULTS && statuses.length > 0) {
const last = statuses[statuses.length - 1];
nextMaxId = last.id || null;
}
// Render each card server-side
const csrfToken = getToken(request.session);
const templateData = {
const html = await renderItemCards(items, request, {
...response.locals,
mountPath,
csrfToken,
interactionMap: {},
};
const htmlParts = await Promise.all(
items.map((item) => {
return new Promise((resolve, reject) => {
request.app.render(
"partials/ap-item-card.njk",
{ ...templateData, item },
(err, html) => {
if (err) reject(err);
else resolve(html);
}
);
});
})
);
response.json({
html: htmlParts.join(""),
maxId: nextMaxId,
});
response.json({ html, maxId: nextMaxId });
} catch (error) {
next(error);
}

View File

@@ -18,6 +18,7 @@
import { validateHashtag, mapMastodonStatusToItem } from "./explore-utils.js";
import { getToken } from "../csrf.js";
import { postProcessItems, renderItemCards } from "../item-processing.js";
const FETCH_TIMEOUT_MS = 10_000;
const PAGE_SIZE = 20;
@@ -179,39 +180,26 @@ export function hashtagExploreApiController(mountPath) {
const pageItems = dedupedItems.slice(0, PAGE_SIZE);
// Map to timeline item format
const items = pageItems.map(({ status, domain }) =>
const rawItems = pageItems.map(({ status, domain }) =>
mapMastodonStatusToItem(status, domain)
);
// Shared processing pipeline (quote stripping, etc.)
const { items } = await postProcessItems(rawItems);
// Render HTML AFTER merge/dedup/paginate (don't waste CPU on discarded items)
const csrfToken = getToken(request.session);
const templateData = {
const html = await renderItemCards(items, request, {
...response.locals,
mountPath,
csrfToken,
interactionMap: {},
};
const htmlParts = await Promise.all(
items.map(
(item) =>
new Promise((resolve, reject) => {
request.app.render(
"partials/ap-item-card.njk",
{ ...templateData, item },
(err, html) => {
if (err) reject(err);
else resolve(html);
}
);
})
)
);
});
const instanceLabels = instanceTabs.map((t) => t.domain);
response.json({
html: htmlParts.join(""),
html,
cursors: updatedCursors,
sources,
instancesQueried: instanceTabs.length,

View File

@@ -1,9 +1,9 @@
// Post detail controller — view individual AP posts/notes/articles
import { Article, Note, Person, Service, Application } from "@fedify/fedify/vocab";
import { getToken } from "../csrf.js";
import { extractObjectData } from "../timeline-store.js";
import { extractObjectData, extractActorInfo } from "../timeline-store.js";
import { getCached, setCache } from "../lookup-cache.js";
import { fetchAndStoreQuote } from "../og-unfurl.js";
import { fetchAndStoreQuote, stripQuoteReferenceHtml } from "../og-unfurl.js";
// Load parent posts (inReplyTo chain) up to maxDepth levels
async function loadParentChain(ctx, documentLoader, timelineCol, parentUrl, maxDepth = 5) {
@@ -332,6 +332,20 @@ export function postDetailController(mountPath, plugin) {
if (quoteObject) {
const quoteData = await extractObjectData(quoteObject, { documentLoader: qLoader });
// If author photo is empty, try fetching the actor directly
if (!quoteData.author.photo && quoteData.author.url) {
try {
const actor = await qCtx.lookupObject(new URL(quoteData.author.url), { documentLoader: qLoader });
if (actor) {
const actorInfo = await extractActorInfo(actor, { documentLoader: qLoader });
if (actorInfo.photo) quoteData.author.photo = actorInfo.photo;
}
} catch {
// Actor fetch failed — keep existing author data
}
}
timelineItem.quote = {
url: quoteData.url || quoteData.uid,
uid: quoteData.uid,
@@ -342,11 +356,24 @@ export function postDetailController(mountPath, plugin) {
photo: quoteData.photo?.slice(0, 1) || [],
};
// Strip RE: paragraph from parent content
const quoteRef = timelineItem.quoteUrl || timelineItem.quote.url || timelineItem.quote.uid;
if (timelineItem.content?.html && quoteRef) {
timelineItem.content.html = stripQuoteReferenceHtml(
timelineItem.content.html,
quoteRef,
);
}
// Persist for future requests (fire-and-forget)
if (timelineCol) {
const persistUpdate = { $set: { quote: timelineItem.quote } };
if (timelineItem.content?.html) {
persistUpdate.$set["content.html"] = timelineItem.content.html;
}
timelineCol.updateOne(
{ $or: [{ uid: objectUrl }, { url: objectUrl }] },
{ $set: { quote: timelineItem.quote } },
persistUpdate,
).catch(() => {});
}
}
@@ -355,6 +382,14 @@ export function postDetailController(mountPath, plugin) {
}
}
// Strip RE: paragraph for items with existing quote data (render-time cleanup)
if (timelineItem.quote && timelineItem.content?.html) {
const quoteRef = timelineItem.quoteUrl || timelineItem.quote.url || timelineItem.quote.uid;
if (quoteRef) {
timelineItem.content.html = stripQuoteReferenceHtml(timelineItem.content.html, quoteRef);
}
}
const csrfToken = getToken(request.session);
response.render("activitypub-post-detail", {

View File

@@ -12,13 +12,8 @@ import {
deleteNotification,
} from "../storage/notifications.js";
import { getToken, validateToken } from "../csrf.js";
import {
getMutedUrls,
getMutedKeywords,
getBlockedUrls,
getFilterMode,
} from "../storage/moderation.js";
import { getFollowedTags } from "../storage/followed-tags.js";
import { postProcessItems, applyTabFilter, loadModerationData } from "../item-processing.js";
// Re-export controllers from split modules for backward compatibility
export {
@@ -54,7 +49,7 @@ export function readerController(mountPath) {
// Build query options
const options = { before, after, limit, unread };
// Tab filtering
// Tab filtering at storage level
if (tab === "notes") {
options.type = "note";
options.excludeReplies = true;
@@ -67,82 +62,21 @@ export function readerController(mountPath) {
// Get timeline items
const result = await getTimelineItems(collections, options);
// Apply client-side filtering for tabs not supported by storage layer
let items = result.items;
if (tab === "replies") {
items = items.filter((item) => item.inReplyTo);
} else if (tab === "media") {
items = items.filter(
(item) =>
(item.photo && item.photo.length > 0) ||
(item.video && item.video.length > 0) ||
(item.audio && item.audio.length > 0),
);
}
// Tab filtering for types not supported by storage layer
const tabFiltered = applyTabFilter(result.items, tab);
// Apply moderation filters (muted actors, keywords, blocked actors)
// Load moderation data + interactions, apply shared pipeline
const modCollections = {
ap_muted: application?.collections?.get("ap_muted"),
ap_blocked: application?.collections?.get("ap_blocked"),
ap_profile: application?.collections?.get("ap_profile"),
};
const [mutedUrls, mutedKeywords, blockedUrls, filterMode] =
await Promise.all([
getMutedUrls(modCollections),
getMutedKeywords(modCollections),
getBlockedUrls(modCollections),
getFilterMode(modCollections),
]);
const blockedSet = new Set(blockedUrls);
const mutedSet = new Set(mutedUrls);
const moderation = await loadModerationData(modCollections);
if (blockedSet.size > 0 || mutedSet.size > 0 || mutedKeywords.length > 0) {
items = items.filter((item) => {
// Blocked actors are ALWAYS hidden
if (item.author?.url && blockedSet.has(item.author.url)) {
return false;
}
// Check muted actor
const isMutedActor =
item.author?.url && mutedSet.has(item.author.url);
// Check muted keywords against content, title, and summary
let matchedKeyword = null;
if (mutedKeywords.length > 0) {
const searchable = [
item.content?.text,
item.name,
item.summary,
]
.filter(Boolean)
.join(" ")
.toLowerCase();
if (searchable) {
matchedKeyword = mutedKeywords.find((kw) =>
searchable.includes(kw.toLowerCase()),
);
}
}
if (isMutedActor || matchedKeyword) {
if (filterMode === "warn") {
// Mark for content warning instead of hiding
item._moderated = true;
item._moderationReason = isMutedActor
? "muted_account"
: "muted_keyword";
if (matchedKeyword) {
item._moderationKeyword = matchedKeyword;
}
return true;
}
return false;
}
return true;
});
}
const { items, interactionMap } = await postProcessItems(tabFiltered, {
moderation,
interactionsCol: application?.collections?.get("ap_interactions"),
});
// Get unread notification count for badge + unread timeline count for toggle
const [unreadCount, unreadTimelineCount] = await Promise.all([
@@ -150,52 +84,6 @@ export function readerController(mountPath) {
countUnreadItems(collections),
]);
// Get interaction state for liked/boosted indicators
// Interactions are keyed by canonical AP uid (new) or display url (legacy).
// Query by both, normalize map keys to uid for template lookup.
const interactionsCol =
application?.collections?.get("ap_interactions");
const interactionMap = {};
if (interactionsCol) {
const lookupUrls = new Set();
const objectUrlToUid = new Map();
for (const item of items) {
const uid = item.uid;
const displayUrl = item.url || item.originalUrl;
if (uid) {
lookupUrls.add(uid);
objectUrlToUid.set(uid, uid);
}
if (displayUrl) {
lookupUrls.add(displayUrl);
objectUrlToUid.set(displayUrl, uid || displayUrl);
}
}
if (lookupUrls.size > 0) {
const interactions = await interactionsCol
.find({ objectUrl: { $in: [...lookupUrls] } })
.toArray();
for (const interaction of interactions) {
// Normalize to uid so template can look up by itemUid
const key =
objectUrlToUid.get(interaction.objectUrl) ||
interaction.objectUrl;
if (!interactionMap[key]) {
interactionMap[key] = {};
}
interactionMap[key][interaction.type] = true;
}
}
}
// CSRF token for interaction forms
const csrfToken = getToken(request.session);

View File

@@ -4,12 +4,7 @@
import { getTimelineItems } from "../storage/timeline.js";
import { getToken } from "../csrf.js";
import {
getMutedUrls,
getMutedKeywords,
getBlockedUrls,
getFilterMode,
} from "../storage/moderation.js";
import { postProcessItems, loadModerationData } from "../item-processing.js";
export function tagTimelineController(mountPath) {
return async (request, response, next) => {
@@ -36,88 +31,21 @@ export function tagTimelineController(mountPath) {
// Get timeline items filtered by tag
const result = await getTimelineItems(collections, { before, after, limit, tag });
let items = result.items;
// Apply moderation filters (same as main reader)
// Shared processing pipeline: moderation, quote stripping, interactions
const modCollections = {
ap_muted: application?.collections?.get("ap_muted"),
ap_blocked: application?.collections?.get("ap_blocked"),
ap_profile: application?.collections?.get("ap_profile"),
};
const [mutedUrls, mutedKeywords, blockedUrls, filterMode] =
await Promise.all([
getMutedUrls(modCollections),
getMutedKeywords(modCollections),
getBlockedUrls(modCollections),
getFilterMode(modCollections),
]);
const blockedSet = new Set(blockedUrls);
const mutedSet = new Set(mutedUrls);
const moderation = await loadModerationData(modCollections);
if (blockedSet.size > 0 || mutedSet.size > 0 || mutedKeywords.length > 0) {
items = items.filter((item) => {
if (item.author?.url && blockedSet.has(item.author.url)) {
return false;
}
const { items, interactionMap } = await postProcessItems(result.items, {
moderation,
interactionsCol: application?.collections?.get("ap_interactions"),
});
const isMutedActor = item.author?.url && mutedSet.has(item.author.url);
let matchedKeyword = null;
if (mutedKeywords.length > 0) {
const searchable = [item.content?.text, item.name, item.summary]
.filter(Boolean)
.join(" ")
.toLowerCase();
if (searchable) {
matchedKeyword = mutedKeywords.find((kw) =>
searchable.includes(kw.toLowerCase())
);
}
}
if (isMutedActor || matchedKeyword) {
if (filterMode === "warn") {
item._moderated = true;
item._moderationReason = isMutedActor ? "muted_account" : "muted_keyword";
if (matchedKeyword) item._moderationKeyword = matchedKeyword;
return true;
}
return false;
}
return true;
});
}
// Get interaction state for liked/boosted indicators
const interactionsCol = application?.collections?.get("ap_interactions");
const interactionMap = {};
if (interactionsCol) {
const lookupUrls = new Set();
const objectUrlToUid = new Map();
for (const item of items) {
const uid = item.uid;
const displayUrl = item.url || item.originalUrl;
if (uid) { lookupUrls.add(uid); objectUrlToUid.set(uid, uid); }
if (displayUrl) { lookupUrls.add(displayUrl); objectUrlToUid.set(displayUrl, uid || displayUrl); }
}
if (lookupUrls.size > 0) {
const interactions = await interactionsCol
.find({ objectUrl: { $in: [...lookupUrls] } })
.toArray();
for (const interaction of interactions) {
const key = objectUrlToUid.get(interaction.objectUrl) || interaction.objectUrl;
if (!interactionMap[key]) interactionMap[key] = {};
interactionMap[key][interaction.type] = true;
}
}
}
// Check if this hashtag is followed (Task 7 will populate ap_followed_tags)
// Check if this hashtag is followed
const followedTagsCol = application?.collections?.get("ap_followed_tags");
let isFollowed = false;
if (followedTagsCol) {

226
lib/item-processing.js Normal file
View File

@@ -0,0 +1,226 @@
/**
* Shared item processing pipeline for the ActivityPub reader.
*
* Both the reader (inbox-sourced items) and explore (Mastodon API items)
* flow through these functions. This ensures every enhancement (emoji,
* quote stripping, moderation, etc.) is implemented once.
*/
import { stripQuoteReferenceHtml } from "./og-unfurl.js";
/**
* Post-process timeline items for rendering.
* Called after items are loaded from any source (MongoDB or Mastodon API).
*
* @param {Array} items - Timeline items (from DB or Mastodon API mapping)
* @param {object} [options]
* @param {object} [options.moderation] - { mutedUrls, mutedKeywords, blockedUrls, filterMode }
* @param {object} [options.interactionsCol] - MongoDB collection for interaction state
* @returns {Promise<{ items: Array, interactionMap: object }>}
*/
export async function postProcessItems(items, options = {}) {
// 1. Moderation filters (muted actors, keywords, blocked actors)
if (options.moderation) {
items = applyModerationFilters(items, options.moderation);
}
// 2. Strip "RE:" paragraphs from items with quote embeds
stripQuoteReferences(items);
// 3. Build interaction map (likes/boosts) — empty when no collection
const interactionMap = options.interactionsCol
? await buildInteractionMap(items, options.interactionsCol)
: {};
return { items, interactionMap };
}
/**
* Apply moderation filters to items.
* Blocked actors are always hidden. Muted actors/keywords are hidden or
* marked with a content warning depending on filterMode.
*
* @param {Array} items
* @param {object} moderation
* @param {string[]} moderation.mutedUrls
* @param {string[]} moderation.mutedKeywords
* @param {string[]} moderation.blockedUrls
* @param {string} moderation.filterMode - "hide" or "warn"
* @returns {Array}
*/
export function applyModerationFilters(items, { mutedUrls, mutedKeywords, blockedUrls, filterMode }) {
const blockedSet = new Set(blockedUrls);
const mutedSet = new Set(mutedUrls);
if (blockedSet.size === 0 && mutedSet.size === 0 && mutedKeywords.length === 0) {
return items;
}
return items.filter((item) => {
// Blocked actors are ALWAYS hidden
if (item.author?.url && blockedSet.has(item.author.url)) {
return false;
}
// Check muted actor
const isMutedActor = item.author?.url && mutedSet.has(item.author.url);
// Check muted keywords against content, title, and summary
let matchedKeyword = null;
if (mutedKeywords.length > 0) {
const searchable = [item.content?.text, item.name, item.summary]
.filter(Boolean)
.join(" ")
.toLowerCase();
if (searchable) {
matchedKeyword = mutedKeywords.find((kw) =>
searchable.includes(kw.toLowerCase()),
);
}
}
if (isMutedActor || matchedKeyword) {
if (filterMode === "warn") {
// Mark for content warning instead of hiding
item._moderated = true;
item._moderationReason = isMutedActor ? "muted_account" : "muted_keyword";
if (matchedKeyword) {
item._moderationKeyword = matchedKeyword;
}
return true;
}
return false;
}
return true;
});
}
/**
* Strip "RE:" quote reference paragraphs from items that have quote embeds.
* Mutates items in place.
*
* @param {Array} items
*/
export function stripQuoteReferences(items) {
for (const item of items) {
const quoteRef = item.quoteUrl || item.quote?.url || item.quote?.uid;
if (item.quote && quoteRef && item.content?.html) {
item.content.html = stripQuoteReferenceHtml(item.content.html, quoteRef);
}
}
}
/**
* Build interaction map (likes/boosts) for template rendering.
* Returns { [uid]: { like: true, boost: true } }.
*
* @param {Array} items
* @param {object} interactionsCol - MongoDB collection
* @returns {Promise<object>}
*/
export async function buildInteractionMap(items, interactionsCol) {
const interactionMap = {};
const lookupUrls = new Set();
const objectUrlToUid = new Map();
for (const item of items) {
const uid = item.uid;
const displayUrl = item.url || item.originalUrl;
if (uid) {
lookupUrls.add(uid);
objectUrlToUid.set(uid, uid);
}
if (displayUrl) {
lookupUrls.add(displayUrl);
objectUrlToUid.set(displayUrl, uid || displayUrl);
}
}
if (lookupUrls.size === 0) return interactionMap;
const interactions = await interactionsCol
.find({ objectUrl: { $in: [...lookupUrls] } })
.toArray();
for (const interaction of interactions) {
const key = objectUrlToUid.get(interaction.objectUrl) || interaction.objectUrl;
if (!interactionMap[key]) interactionMap[key] = {};
interactionMap[key][interaction.type] = true;
}
return interactionMap;
}
/**
* Filter items by tab type (reader-specific).
*
* @param {Array} items
* @param {string} tab - "notes", "articles", "boosts", "replies", "media", "all"
* @returns {Array}
*/
export function applyTabFilter(items, tab) {
if (tab === "replies") {
return items.filter((item) => item.inReplyTo);
}
if (tab === "media") {
return items.filter(
(item) =>
(item.photo && item.photo.length > 0) ||
(item.video && item.video.length > 0) ||
(item.audio && item.audio.length > 0),
);
}
return items;
}
/**
* Render items to HTML using ap-item-card.njk.
* Used by all API endpoints that return pre-rendered card HTML.
*
* @param {Array} items
* @param {object} request - Express request (for app.render)
* @param {object} templateData - Merged template context (locals, mountPath, csrfToken, interactionMap)
* @returns {Promise<string>}
*/
export async function renderItemCards(items, request, templateData) {
const htmlParts = await Promise.all(
items.map(
(item) =>
new Promise((resolve, reject) => {
request.app.render(
"partials/ap-item-card.njk",
{ ...templateData, item },
(err, html) => {
if (err) reject(err);
else resolve(html);
},
);
}),
),
);
return htmlParts.join("");
}
/**
* Load moderation data from MongoDB collections.
* Convenience wrapper to reduce boilerplate in controllers.
*
* @param {object} modCollections - { ap_muted, ap_blocked, ap_profile }
* @returns {Promise<object>} moderation data for postProcessItems()
*/
export async function loadModerationData(modCollections) {
// Dynamic import to avoid circular dependency
const { getMutedUrls, getMutedKeywords, getBlockedUrls, getFilterMode } =
await import("./storage/moderation.js");
const [mutedUrls, mutedKeywords, blockedUrls, filterMode] = await Promise.all([
getMutedUrls(modCollections),
getMutedKeywords(modCollections),
getBlockedUrls(modCollections),
getFilterMode(modCollections),
]);
return { mutedUrls, mutedKeywords, blockedUrls, filterMode };
}

View File

@@ -267,6 +267,22 @@ export async function fetchAndStoreQuote(collections, uid, quoteUrl, ctx, docume
const quoteData = await extractObjectData(object, { documentLoader });
// If author photo is empty, try fetching the actor directly
if (!quoteData.author.photo && quoteData.author.url) {
try {
const actor = await ctx.lookupObject(new URL(quoteData.author.url), { documentLoader });
if (actor) {
const { extractActorInfo } = await import("./timeline-store.js");
const actorInfo = await extractActorInfo(actor, { documentLoader });
if (actorInfo.photo) {
quoteData.author.photo = actorInfo.photo;
}
}
} catch {
// Actor fetch failed — keep existing author data
}
}
const quote = {
url: quoteData.url || quoteData.uid,
uid: quoteData.uid,
@@ -277,11 +293,46 @@ export async function fetchAndStoreQuote(collections, uid, quoteUrl, ctx, docume
photo: quoteData.photo?.slice(0, 1) || [],
};
await collections.ap_timeline.updateOne(
{ uid },
{ $set: { quote } },
);
// Strip the "RE: <link>" paragraph from the parent post's content
// Mastodon adds this as: <p>RE: <a href="QUOTE_URL">...</a></p>
const update = { $set: { quote } };
const parentItem = await collections.ap_timeline.findOne({ uid });
if (parentItem?.content?.html) {
const cleaned = stripQuoteReferenceHtml(parentItem.content.html, quoteUrl);
if (cleaned !== parentItem.content.html) {
update.$set["content.html"] = cleaned;
}
}
await collections.ap_timeline.updateOne({ uid }, update);
} catch (error) {
console.error(`[og-unfurl] Failed to fetch quote for ${uid}: ${error.message}`);
}
}
/**
* Strip the "RE: <link>" paragraph that Mastodon adds for quoted posts.
* Removes <p> elements containing "RE:" followed by a link to the quote URL.
* @param {string} html - Content HTML
* @param {string} quoteUrl - URL of the quoted post
* @returns {string} Cleaned HTML
*/
export function stripQuoteReferenceHtml(html, quoteUrl) {
if (!html || !quoteUrl) return html;
// Match <p> containing "RE:" and a link whose href contains the quote domain+path
// Mastodon uses both /users/X/statuses/Y and /@X/Y URL formats
try {
const quoteUrlObj = new URL(quoteUrl);
const quoteDomain = quoteUrlObj.hostname;
// Escape special regex chars in domain
const domainEscaped = quoteDomain.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
// Match <p>RE: <a href="...DOMAIN...">...</a></p> (with optional whitespace)
const re = new RegExp(
`<p>\\s*RE:\\s*<a\\s[^>]*href="[^"]*${domainEscaped}[^"]*"[^>]*>.*?</a>\\s*</p>`,
"i",
);
return html.replace(re, "").trim();
} catch {
return html;
}
}

View File

@@ -1,6 +1,6 @@
{
"name": "@rmdes/indiekit-endpoint-activitypub",
"version": "2.4.1",
"version": "2.5.0",
"description": "ActivityPub federation endpoint for Indiekit via Fedify. Adds full fediverse support: actor, inbox, outbox, followers, following, syndication, and Mastodon migration.",
"keywords": [
"indiekit",

View File

@@ -247,11 +247,13 @@
{% if maxId %}
<div class="ap-load-more"
id="ap-explore-load-more"
data-max-id="{{ maxId }}"
data-instance="{{ instance }}"
data-scope="{{ scope }}"
data-hashtag="{{ hashtag }}"
x-data="apExploreScroll()"
data-cursor="{{ maxId }}"
data-api-url="{{ mountPath }}/admin/reader/api/explore"
data-cursor-param="max_id"
data-cursor-field="maxId"
data-timeline-id="ap-explore-timeline"
data-extra-params='{{ { instance: instance, scope: scope, hashtag: hashtag } | dump }}'
x-data="apInfiniteScroll()"
x-init="init()">
<div class="ap-load-more__sentinel" x-ref="sentinel"></div>
<button class="ap-load-more__btn" @click="loadMore()" :disabled="loading" x-show="!done">
@@ -302,10 +304,19 @@
x-html="tabState[tab._id] ? tabState[tab._id].html : ''">
</div>
{# Inline loading spinner for subsequent pages #}
<div class="ap-explore-tab-loading ap-explore-tab-loading--more"
x-show="tabState[tab._id] && tabState[tab._id].loading && tabState[tab._id].html">
<span class="ap-explore-tab-loading__text">{{ __("activitypub.reader.pagination.loading") }}</span>
{# Load more button + loading spinner for subsequent pages #}
<div class="ap-load-more"
x-show="tabState[tab._id] && tabState[tab._id].html && !tabState[tab._id].done">
<button class="ap-load-more__btn"
x-show="!tabState[tab._id]?.loading"
@click="loadMoreTab(tab)"
:disabled="tabState[tab._id]?.loading">
{{ __("activitypub.reader.pagination.loadMore") }}
</button>
<span class="ap-explore-tab-loading__text"
x-show="tabState[tab._id]?.loading">
{{ __("activitypub.reader.pagination.loading") }}
</span>
</div>
{# Empty state — loaded successfully but no posts #}
@@ -357,10 +368,19 @@
x-html="tabState[tab._id] ? tabState[tab._id].html : ''">
</div>
{# Inline loading spinner for subsequent pages #}
<div class="ap-explore-tab-loading ap-explore-tab-loading--more"
x-show="tabState[tab._id] && tabState[tab._id].loading && tabState[tab._id].html">
<span class="ap-explore-tab-loading__text">{{ __("activitypub.reader.pagination.loading") }}</span>
{# Load more button + loading spinner for subsequent pages #}
<div class="ap-load-more"
x-show="tabState[tab._id] && tabState[tab._id].html && !tabState[tab._id].done">
<button class="ap-load-more__btn"
x-show="!tabState[tab._id]?.loading"
@click="loadMoreTab(tab)"
:disabled="tabState[tab._id]?.loading">
{{ __("activitypub.reader.pagination.loadMore") }}
</button>
<span class="ap-explore-tab-loading__text"
x-show="tabState[tab._id]?.loading">
{{ __("activitypub.reader.pagination.loading") }}
</span>
</div>
{# Empty state — no instance tabs pinned yet #}

View File

@@ -140,12 +140,15 @@
{% if before %}
<div class="ap-load-more"
id="ap-load-more"
data-before="{{ before }}"
data-tab="{{ tab }}"
data-tag=""
data-cursor="{{ before }}"
data-api-url="{{ mountPath }}/admin/reader/api/timeline"
data-cursor-param="before"
data-cursor-field="before"
data-timeline-id="ap-timeline"
data-extra-params='{{ { tab: tab } | dump }}'
data-hide-pagination="ap-reader-pagination"
x-data="apInfiniteScroll()"
x-init="init()"
@ap-append-items.window="appendItems($event.detail)">
x-init="init()">
<div class="ap-load-more__sentinel" x-ref="sentinel"></div>
<button class="ap-load-more__btn" @click="loadMore()" :disabled="loading" x-show="!done">
<span x-show="!loading">{{ __("activitypub.reader.pagination.loadMore") }}</span>

View File

@@ -36,6 +36,7 @@
{# Timeline items #}
{% if items.length > 0 %}
<div class="ap-timeline"
id="ap-timeline"
data-mount-path="{{ mountPath }}"
data-tag="{{ hashtag }}"
data-before="{{ before if before else '' }}">
@@ -64,12 +65,15 @@
{% if before %}
<div class="ap-load-more"
id="ap-load-more"
data-before="{{ before }}"
data-tab=""
data-tag="{{ hashtag }}"
data-cursor="{{ before }}"
data-api-url="{{ mountPath }}/admin/reader/api/timeline"
data-cursor-param="before"
data-cursor-field="before"
data-timeline-id="ap-timeline"
data-extra-params='{{ { tag: hashtag } | dump }}'
data-hide-pagination="ap-tag-pagination"
x-data="apInfiniteScroll()"
x-init="init()"
@ap-append-items.window="appendItems($event.detail)">
x-init="init()">
<div class="ap-load-more__sentinel" x-ref="sentinel"></div>
<button class="ap-load-more__btn" @click="loadMore()" :disabled="loading" x-show="!done">
<span x-show="!loading">{{ __("activitypub.reader.pagination.loadMore") }}</span>