From 8ace76f8c2ed5af21c3bc1d55c80c8bc5af63edd Mon Sep 17 00:00:00 2001 From: Ricardo Date: Sun, 8 Feb 2026 12:35:52 +0100 Subject: [PATCH] feat: Add Microsub integration with reference-based data approach - Add Microsub source type to sync subscriptions from Microsub channels - Use reference-based approach to avoid data duplication: - Blogs store microsubFeedId reference instead of copying data - Items for Microsub blogs are queried from microsub_items directly - No duplicate storage or retention management needed - Add channel filter and category prefix options for Microsub sources - Add webhook endpoint for Microsub subscription change notifications - Update scheduler to skip item fetching for Microsub blogs - Update items storage to combine results from both collections - Bump version to 1.0.7 Co-Authored-By: Claude Opus 4.5 --- index.js | 4 + lib/controllers/api.js | 96 +++++++++++- lib/controllers/sources.js | 94 +++++++++-- lib/storage/items.js | 130 ++++++++++++++-- lib/storage/sources.js | 7 +- lib/sync/microsub.js | 277 +++++++++++++++++++++++++++++++++ lib/sync/scheduler.js | 25 ++- package.json | 2 +- views/blogroll-source-edit.njk | 34 +++- 9 files changed, 631 insertions(+), 38 deletions(-) create mode 100644 lib/sync/microsub.js diff --git a/index.js b/index.js index bb9d367..7d2a136 100644 --- a/index.js +++ b/index.js @@ -89,6 +89,10 @@ export default class BlogrollEndpoint { // Feed discovery (protected to prevent abuse) protectedRouter.get("/api/discover", apiController.discover); + // Microsub integration (protected - internal use) + protectedRouter.post("/api/microsub-webhook", apiController.microsubWebhook); + protectedRouter.get("/api/microsub-status", apiController.microsubStatus); + return protectedRouter; } diff --git a/lib/controllers/api.js b/lib/controllers/api.js index 165148e..3790814 100644 --- a/lib/controllers/api.js +++ b/lib/controllers/api.js @@ -9,6 +9,7 @@ import { getItems, getItemsForBlog } from "../storage/items.js"; import { getSyncStatus } from "../sync/scheduler.js"; import { generateOpml } from "../sync/opml.js"; import { discoverFeeds } from "../utils/feed-discovery.js"; +import { handleMicrosubWebhook, isMicrosubAvailable } from "../sync/microsub.js"; /** * List blogs with optional filtering @@ -57,7 +58,9 @@ async function getBlogDetail(request, response) { return response.status(404).json({ error: "Blog not found" }); } - const items = await getItemsForBlog(application, blog._id, 20); + // Pass blog to getItemsForBlog to avoid duplicate lookup + // This handles both regular and Microsub-sourced blogs transparently + const items = await getItemsForBlog(application, blog._id, 20, blog); response.json({ ...sanitizeBlog(blog), @@ -214,7 +217,7 @@ async function discover(request, response) { * @returns {object} Sanitized blog */ function sanitizeBlog(blog) { - return { + const sanitized = { id: blog._id.toString(), title: blog.title, description: blog.description, @@ -230,6 +233,14 @@ function sanitizeBlog(blog) { pinned: blog.pinned, lastFetchAt: blog.lastFetchAt, }; + + // Include Microsub metadata if applicable + if (blog.source === "microsub") { + sanitized.source = "microsub"; + sanitized.microsubChannel = blog.microsubChannelName; + } + + return sanitized; } /** @@ -250,6 +261,85 @@ function sanitizeItem(item) { }; } +/** + * Microsub webhook handler + * Receives subscription change notifications from Microsub + * POST /api/microsub-webhook + */ +async function microsubWebhook(request, response) { + const { application } = request.app.locals; + + try { + // Verify Microsub is available + if (!isMicrosubAvailable(application)) { + return response.status(503).json({ + ok: false, + error: "Microsub integration not available", + }); + } + + const { action, url, channelName, title } = request.body; + + if (!action || !url) { + return response.status(400).json({ + ok: false, + error: "Missing required fields: action and url", + }); + } + + const result = await handleMicrosubWebhook(application, { + action, + url, + channelName, + title, + }); + + response.json(result); + } catch (error) { + console.error("[Blogroll API] microsubWebhook error:", error); + response.status(500).json({ + ok: false, + error: "Webhook processing failed", + }); + } +} + +/** + * Check Microsub integration status + * GET /api/microsub-status + */ +async function microsubStatus(request, response) { + const { application } = request.app.locals; + + try { + const available = isMicrosubAvailable(application); + + if (!available) { + return response.json({ + available: false, + message: "Microsub plugin not installed or collections not available", + }); + } + + // Get count of microsub-sourced blogs + const db = application.getBlogrollDb(); + const microsubBlogCount = await db.collection("blogrollBlogs").countDocuments({ + source: { $regex: /^microsub/ }, + }); + + response.json({ + available: true, + blogs: microsubBlogCount, + }); + } catch (error) { + console.error("[Blogroll API] microsubStatus error:", error); + response.status(500).json({ + available: false, + error: error.message, + }); + } +} + export const apiController = { listBlogs, getBlog: getBlogDetail, @@ -259,4 +349,6 @@ export const apiController = { exportOpml, exportOpmlCategory, discover, + microsubWebhook, + microsubStatus, }; diff --git a/lib/controllers/sources.js b/lib/controllers/sources.js index 49a46e1..3a013f6 100644 --- a/lib/controllers/sources.js +++ b/lib/controllers/sources.js @@ -11,6 +11,11 @@ import { deleteSource, } from "../storage/sources.js"; import { syncOpmlSource } from "../sync/opml.js"; +import { + syncMicrosubSource, + getMicrosubChannels, + isMicrosubAvailable, +} from "../sync/microsub.js"; /** * List sources @@ -50,12 +55,22 @@ async function list(request, response) { * New source form * GET /sources/new */ -function newForm(request, response) { +async function newForm(request, response) { + const { application } = request.app.locals; + + // Check if Microsub is available and get channels + const microsubAvailable = isMicrosubAvailable(application); + const microsubChannels = microsubAvailable + ? await getMicrosubChannels(application) + : []; + response.render("blogroll-source-edit", { title: request.__("blogroll.sources.new"), source: null, isNew: true, baseUrl: request.baseUrl, + microsubAvailable, + microsubChannels, }); } @@ -65,7 +80,16 @@ function newForm(request, response) { */ async function create(request, response) { const { application } = request.app.locals; - const { name, type, url, opmlContent, syncInterval, enabled } = request.body; + const { + name, + type, + url, + opmlContent, + syncInterval, + enabled, + channelFilter, + categoryPrefix, + } = request.body; try { // Validate required fields @@ -83,18 +107,37 @@ async function create(request, response) { return response.redirect(`${request.baseUrl}/sources/new`); } - const source = await createSource(application, { + if (type === "microsub" && !isMicrosubAvailable(application)) { + request.session.messages = [ + { type: "error", content: "Microsub plugin is not available" }, + ]; + return response.redirect(`${request.baseUrl}/sources/new`); + } + + const sourceData = { name, type, url: url || null, opmlContent: opmlContent || null, syncInterval: Number(syncInterval) || 60, enabled: enabled === "on" || enabled === true, - }); + }; - // Trigger initial sync for OPML sources + // Add microsub-specific fields + if (type === "microsub") { + sourceData.channelFilter = channelFilter || null; + sourceData.categoryPrefix = categoryPrefix || ""; + } + + const source = await createSource(application, sourceData); + + // Trigger initial sync based on source type try { - await syncOpmlSource(application, source); + if (type === "microsub") { + await syncMicrosubSource(application, source); + } else { + await syncOpmlSource(application, source); + } request.session.messages = [ { type: "success", content: request.__("blogroll.sources.created_synced") }, ]; @@ -134,11 +177,19 @@ async function edit(request, response) { return response.status(404).render("404"); } + // Check if Microsub is available and get channels + const microsubAvailable = isMicrosubAvailable(application); + const microsubChannels = microsubAvailable + ? await getMicrosubChannels(application) + : []; + response.render("blogroll-source-edit", { title: request.__("blogroll.sources.edit"), source, isNew: false, baseUrl: request.baseUrl, + microsubAvailable, + microsubChannels, }); } catch (error) { console.error("[Blogroll] Edit source error:", error); @@ -156,7 +207,16 @@ async function edit(request, response) { async function update(request, response) { const { application } = request.app.locals; const { id } = request.params; - const { name, type, url, opmlContent, syncInterval, enabled } = request.body; + const { + name, + type, + url, + opmlContent, + syncInterval, + enabled, + channelFilter, + categoryPrefix, + } = request.body; try { const source = await getSource(application, id); @@ -165,14 +225,22 @@ async function update(request, response) { return response.status(404).render("404"); } - await updateSource(application, id, { + const updateData = { name, type, url: url || null, opmlContent: opmlContent || null, syncInterval: Number(syncInterval) || 60, enabled: enabled === "on" || enabled === true, - }); + }; + + // Add microsub-specific fields + if (type === "microsub") { + updateData.channelFilter = channelFilter || null; + updateData.categoryPrefix = categoryPrefix || ""; + } + + await updateSource(application, id, updateData); request.session.messages = [ { type: "success", content: request.__("blogroll.sources.updated") }, @@ -234,7 +302,13 @@ async function sync(request, response) { return response.status(404).render("404"); } - const result = await syncOpmlSource(application, source); + // Use appropriate sync function based on source type + let result; + if (source.type === "microsub") { + result = await syncMicrosubSource(application, source); + } else { + result = await syncOpmlSource(application, source); + } if (result.success) { request.session.messages = [ diff --git a/lib/storage/items.js b/lib/storage/items.js index 403ef6f..574dd6c 100644 --- a/lib/storage/items.js +++ b/lib/storage/items.js @@ -1,9 +1,14 @@ /** * Item storage operations * @module storage/items + * + * IMPORTANT: This module handles items from TWO sources: + * - Regular blogs: items stored in blogrollItems collection + * - Microsub blogs: items queried from microsub_items collection (no duplication) */ import { ObjectId } from "mongodb"; +import { getMicrosubItemsForBlog } from "../sync/microsub.js"; /** * Get collection reference @@ -17,6 +22,7 @@ function getCollection(application) { /** * Get items with optional filtering + * Combines items from blogrollItems (regular blogs) and microsub_items (Microsub blogs) * @param {object} application - Application instance * @param {object} options - Query options * @returns {Promise} Items with blog info @@ -25,10 +31,21 @@ export async function getItems(application, options = {}) { const db = application.getBlogrollDb(); const { blogId, category, limit = 50, offset = 0 } = options; - const pipeline = [ + // If requesting items for a specific blog, check if it's a Microsub blog + if (blogId) { + const blog = await db.collection("blogrollBlogs").findOne({ _id: new ObjectId(blogId) }); + if (blog?.source === "microsub" && blog.microsubFeedId) { + const microsubItems = await getMicrosubItemsForBlog(application, blog, limit + 1); + const itemsWithBlog = microsubItems.map((item) => ({ ...item, blog })); + const hasMore = itemsWithBlog.length > limit; + if (hasMore) itemsWithBlog.pop(); + return { items: itemsWithBlog, hasMore }; + } + } + + // Get regular items from blogrollItems + const regularPipeline = [ { $sort: { published: -1 } }, - { $skip: offset }, - { $limit: limit + 1 }, // Fetch one extra to check hasMore { $lookup: { from: "blogrollBlogs", @@ -38,36 +55,80 @@ export async function getItems(application, options = {}) { }, }, { $unwind: "$blog" }, - { $match: { "blog.hidden": { $ne: true } } }, + // Exclude hidden blogs and Microsub blogs (their items come from microsub_items) + { $match: { "blog.hidden": { $ne: true }, "blog.source": { $ne: "microsub" } } }, ]; if (blogId) { - pipeline.unshift({ $match: { blogId: new ObjectId(blogId) } }); + regularPipeline.unshift({ $match: { blogId: new ObjectId(blogId) } }); } if (category) { - pipeline.push({ $match: { "blog.category": category } }); + regularPipeline.push({ $match: { "blog.category": category } }); } - const items = await db.collection("blogrollItems").aggregate(pipeline).toArray(); + const regularItems = await db.collection("blogrollItems").aggregate(regularPipeline).toArray(); - const hasMore = items.length > limit; - if (hasMore) items.pop(); + // Get items from Microsub-sourced blogs + const microsubBlogsQuery = { + source: "microsub", + hidden: { $ne: true }, + }; + if (category) { + microsubBlogsQuery.category = category; + } - return { items, hasMore }; + const microsubBlogs = await db.collection("blogrollBlogs").find(microsubBlogsQuery).toArray(); + + let microsubItems = []; + for (const blog of microsubBlogs) { + if (blog.microsubFeedId) { + const items = await getMicrosubItemsForBlog(application, blog, 100); + microsubItems.push(...items.map((item) => ({ ...item, blog }))); + } + } + + // Combine and sort all items by published date + const allItems = [...regularItems, ...microsubItems]; + allItems.sort((a, b) => { + const dateA = a.published ? new Date(a.published) : new Date(0); + const dateB = b.published ? new Date(b.published) : new Date(0); + return dateB - dateA; + }); + + // Apply pagination + const paginatedItems = allItems.slice(offset, offset + limit + 1); + const hasMore = paginatedItems.length > limit; + if (hasMore) paginatedItems.pop(); + + return { items: paginatedItems, hasMore }; } /** * Get items for a specific blog + * Handles both regular blogs (blogrollItems) and Microsub blogs (microsub_items) * @param {object} application - Application instance * @param {string|ObjectId} blogId - Blog ID * @param {number} limit - Max items + * @param {object} blog - Optional blog document (to avoid extra lookup) * @returns {Promise} Items */ -export async function getItemsForBlog(application, blogId, limit = 20) { - const collection = getCollection(application); +export async function getItemsForBlog(application, blogId, limit = 20, blog = null) { + const db = application.getBlogrollDb(); const objectId = typeof blogId === "string" ? new ObjectId(blogId) : blogId; + // Get blog if not provided + if (!blog) { + blog = await db.collection("blogrollBlogs").findOne({ _id: objectId }); + } + + // For Microsub-sourced blogs, query microsub_items directly + if (blog?.source === "microsub" && blog.microsubFeedId) { + return getMicrosubItemsForBlog(application, blog, limit); + } + + // For regular blogs, query blogrollItems + const collection = getCollection(application); return collection .find({ blogId: objectId }) .sort({ published: -1 }) @@ -76,20 +137,55 @@ export async function getItemsForBlog(application, blogId, limit = 20) { } /** - * Count items + * Count items (including Microsub items) * @param {object} application - Application instance * @param {object} options - Query options * @returns {Promise} Count */ export async function countItems(application, options = {}) { - const collection = getCollection(application); - const query = {}; + const db = application.getBlogrollDb(); + // Count regular items + const regularQuery = {}; if (options.blogId) { - query.blogId = new ObjectId(options.blogId); + regularQuery.blogId = new ObjectId(options.blogId); + } + const regularCount = await db.collection("blogrollItems").countDocuments(regularQuery); + + // Count Microsub items for microsub-sourced blogs + let microsubCount = 0; + const itemsCollection = application.collections?.get("microsub_items"); + + if (itemsCollection) { + if (options.blogId) { + // Count for specific blog + const blog = await db.collection("blogrollBlogs").findOne({ _id: new ObjectId(options.blogId) }); + if (blog?.source === "microsub" && blog.microsubFeedId) { + microsubCount = await itemsCollection.countDocuments({ + feedId: new ObjectId(blog.microsubFeedId), + }); + } + } else { + // Count all Microsub items from blogroll-associated feeds + const microsubBlogs = await db + .collection("blogrollBlogs") + .find({ source: "microsub", microsubFeedId: { $exists: true } }) + .toArray(); + + const feedIds = microsubBlogs + .map((b) => b.microsubFeedId) + .filter(Boolean) + .map((id) => new ObjectId(id)); + + if (feedIds.length > 0) { + microsubCount = await itemsCollection.countDocuments({ + feedId: { $in: feedIds }, + }); + } + } } - return collection.countDocuments(query); + return regularCount + microsubCount; } /** diff --git a/lib/storage/sources.js b/lib/storage/sources.js index d3517ac..6a01572 100644 --- a/lib/storage/sources.js +++ b/lib/storage/sources.js @@ -48,10 +48,13 @@ export async function createSource(application, data) { const now = new Date(); const source = { - type: data.type, // "opml_url" | "opml_file" | "manual" | "json_feed" + type: data.type, // "opml_url" | "opml_file" | "manual" | "json_feed" | "microsub" name: data.name, url: data.url || null, opmlContent: data.opmlContent || null, + // Microsub-specific fields + channelFilter: data.channelFilter || null, + categoryPrefix: data.categoryPrefix || "", enabled: data.enabled !== false, syncInterval: data.syncInterval || 60, // minutes lastSyncAt: null, @@ -157,7 +160,7 @@ export async function getSourcesDueForSync(application) { return collection .find({ enabled: true, - type: { $in: ["opml_url", "json_feed"] }, + type: { $in: ["opml_url", "json_feed", "microsub"] }, $or: [ { lastSyncAt: null }, { diff --git a/lib/sync/microsub.js b/lib/sync/microsub.js new file mode 100644 index 0000000..cbf67f0 --- /dev/null +++ b/lib/sync/microsub.js @@ -0,0 +1,277 @@ +/** + * Microsub integration - sync subscriptions from Microsub channels + * @module sync/microsub + * + * IMPORTANT: This module uses a REFERENCE-BASED approach to avoid data duplication. + * - Blogs from Microsub are stored with `source: "microsub"` and `microsubFeedId` + * - Items are NOT copied to blogrollItems - we query microsub_items directly + * - The blogroll API joins data from both collections as needed + */ + +import { upsertBlog, getBlogByFeedUrl } from "../storage/blogs.js"; +import { updateSourceSyncStatus } from "../storage/sources.js"; + +/** + * Sync blogs from Microsub subscriptions + * Creates references to Microsub feeds, NOT copies of the data + * @param {object} application - Application instance + * @param {object} source - Source document with microsub config + * @returns {Promise} Sync result + */ +export async function syncMicrosubSource(application, source) { + try { + // Get Microsub collections via Indiekit's collection system + const channelsCollection = application.collections?.get("microsub_channels"); + const feedsCollection = application.collections?.get("microsub_feeds"); + + if (!channelsCollection || !feedsCollection) { + throw new Error("Microsub collections not available. Is the Microsub plugin installed?"); + } + + // Get channels (optionally filter by specific channel) + const channelQuery = source.channelFilter + ? { uid: source.channelFilter } + : {}; + const channels = await channelsCollection.find(channelQuery).toArray(); + + if (channels.length === 0) { + console.log("[Blogroll] No Microsub channels found"); + await updateSourceSyncStatus(application, source._id, { success: true }); + return { success: true, added: 0, updated: 0, total: 0 }; + } + + let added = 0; + let updated = 0; + let total = 0; + + for (const channel of channels) { + // Get all feeds subscribed in this channel + const feeds = await feedsCollection.find({ channelId: channel._id }).toArray(); + + for (const feed of feeds) { + total++; + + // Store REFERENCE to Microsub feed, not a copy + // Items will be queried from microsub_items directly + const blogData = { + title: feed.title || extractDomainFromUrl(feed.url), + feedUrl: feed.url, + siteUrl: extractSiteUrl(feed.url), + feedType: "rss", + category: source.categoryPrefix + ? `${source.categoryPrefix}${channel.name}` + : channel.name, + // Mark as microsub source - items come from microsub_items, not blogrollItems + source: "microsub", + sourceId: source._id, + // Store reference IDs for joining with Microsub data + microsubFeedId: feed._id.toString(), + microsubChannelId: channel._id.toString(), + microsubChannelName: channel.name, + // Mirror status from Microsub (don't duplicate, just reference) + status: feed.status === "error" ? "error" : "active", + lastFetchAt: feed.lastFetchedAt || null, + photo: feed.photo || null, + // Flag to skip item fetching - Microsub handles this + skipItemFetch: true, + }; + + const result = await upsertBlog(application, blogData); + + if (result.upserted) added++; + else if (result.modified) updated++; + } + } + + // Update source sync status + await updateSourceSyncStatus(application, source._id, { success: true }); + + console.log( + `[Blogroll] Synced Microsub source "${source.name}": ${added} added, ${updated} updated, ${total} total from ${channels.length} channels (items served from Microsub)` + ); + + return { success: true, added, updated, total }; + } catch (error) { + // Update source with error status + await updateSourceSyncStatus(application, source._id, { + success: false, + error: error.message, + }); + + console.error(`[Blogroll] Microsub sync failed for "${source.name}":`, error.message); + return { success: false, error: error.message }; + } +} + +/** + * Get items for a Microsub-sourced blog + * Queries microsub_items directly instead of blogrollItems + * @param {object} application - Application instance + * @param {object} blog - Blog with microsubFeedId + * @param {number} limit - Max items to return + * @returns {Promise} Items from Microsub + */ +export async function getMicrosubItemsForBlog(application, blog, limit = 20) { + if (!blog.microsubFeedId) { + return []; + } + + const itemsCollection = application.collections?.get("microsub_items"); + if (!itemsCollection) { + return []; + } + + const { ObjectId } = await import("mongodb"); + const feedId = new ObjectId(blog.microsubFeedId); + + const items = await itemsCollection + .find({ feedId }) + .sort({ published: -1 }) + .limit(limit) + .toArray(); + + // Transform Microsub item format to Blogroll format + return items.map((item) => ({ + _id: item._id, + blogId: blog._id, + url: item.url, + title: item.name || item.url, + summary: item.summary || item.content?.text?.substring(0, 300), + published: item.published, + author: item.author?.name, + photo: item.photo?.[0] || item.featured, + categories: item.category || [], + })); +} + +/** + * Handle Microsub subscription webhook + * Called when a feed is subscribed/unsubscribed in Microsub + * @param {object} application - Application instance + * @param {object} data - Webhook data + * @param {string} data.action - "subscribe" or "unsubscribe" + * @param {string} data.url - Feed URL + * @param {string} data.channelName - Channel name + * @param {string} [data.title] - Feed title + * @returns {Promise} Result + */ +export async function handleMicrosubWebhook(application, data) { + const { action, url, channelName, title } = data; + + if (action === "subscribe") { + // Check if blog already exists + const existing = await getBlogByFeedUrl(application, url); + + if (existing) { + // Update category if it's from microsub + if (existing.source === "microsub") { + console.log(`[Blogroll] Webhook: Feed ${url} already exists, skipping`); + return { ok: true, action: "skipped", reason: "already_exists" }; + } + // Don't overwrite manually added blogs + return { ok: true, action: "skipped", reason: "manual_entry" }; + } + + // Add new blog + await upsertBlog(application, { + title: title || extractDomainFromUrl(url), + feedUrl: url, + siteUrl: extractSiteUrl(url), + feedType: "rss", + category: channelName || "Microsub", + source: "microsub-webhook", + status: "pending", + }); + + console.log(`[Blogroll] Webhook: Added feed ${url} from Microsub`); + return { ok: true, action: "added" }; + } + + if (action === "unsubscribe") { + // Mark as inactive rather than delete (preserve history) + const existing = await getBlogByFeedUrl(application, url); + + if (existing && existing.source?.startsWith("microsub")) { + // Update status to inactive + const db = application.getBlogrollDb(); + await db.collection("blogrollBlogs").updateOne( + { _id: existing._id }, + { + $set: { + status: "inactive", + unsubscribedAt: new Date(), + updatedAt: new Date(), + }, + } + ); + + console.log(`[Blogroll] Webhook: Marked feed ${url} as inactive`); + return { ok: true, action: "deactivated" }; + } + + return { ok: true, action: "skipped", reason: "not_found_or_not_microsub" }; + } + + return { ok: false, error: `Unknown action: ${action}` }; +} + +/** + * Get all Microsub channels for source configuration UI + * @param {object} application - Application instance + * @returns {Promise} Array of channels + */ +export async function getMicrosubChannels(application) { + const channelsCollection = application.collections?.get("microsub_channels"); + + if (!channelsCollection) { + return []; + } + + const channels = await channelsCollection.find({}).sort({ order: 1 }).toArray(); + + return channels.map((ch) => ({ + uid: ch.uid, + name: ch.name, + _id: ch._id.toString(), + })); +} + +/** + * Check if Microsub plugin is available + * @param {object} application - Application instance + * @returns {boolean} True if Microsub is available + */ +export function isMicrosubAvailable(application) { + return !!( + application.collections?.get("microsub_channels") && + application.collections?.get("microsub_feeds") + ); +} + +/** + * Extract domain from URL for fallback title + * @param {string} url - Feed URL + * @returns {string} Domain name + */ +function extractDomainFromUrl(url) { + try { + const parsed = new URL(url); + return parsed.hostname.replace(/^www\./, ""); + } catch { + return url; + } +} + +/** + * Extract site URL from feed URL + * @param {string} feedUrl - Feed URL + * @returns {string} Site URL + */ +function extractSiteUrl(feedUrl) { + try { + const parsed = new URL(feedUrl); + return `${parsed.protocol}//${parsed.host}`; + } catch { + return ""; + } +} diff --git a/lib/sync/scheduler.js b/lib/sync/scheduler.js index 5b222d8..b50e93c 100644 --- a/lib/sync/scheduler.js +++ b/lib/sync/scheduler.js @@ -7,6 +7,7 @@ import { getSources } from "../storage/sources.js"; import { getBlogs, countBlogs } from "../storage/blogs.js"; import { countItems, deleteOldItems } from "../storage/items.js"; import { syncOpmlSource } from "./opml.js"; +import { syncMicrosubSource } from "./microsub.js"; import { syncBlogItems } from "./feed.js"; let syncInterval = null; @@ -38,10 +39,10 @@ export async function runFullSync(application, options = {}) { // First, clean up old items to encourage discovery const deletedItems = await deleteOldItems(application, maxItemAge); - // Sync all enabled OPML/JSON sources + // Sync all enabled sources (OPML, JSON, Microsub) const sources = await getSources(application); const enabledSources = sources.filter( - (s) => s.enabled && ["opml_url", "opml_file", "json_feed"].includes(s.type) + (s) => s.enabled && ["opml_url", "opml_file", "json_feed", "microsub"].includes(s.type) ); let sourcesSuccess = 0; @@ -49,7 +50,12 @@ export async function runFullSync(application, options = {}) { for (const source of enabledSources) { try { - const result = await syncOpmlSource(application, source); + let result; + if (source.type === "microsub") { + result = await syncMicrosubSource(application, source); + } else { + result = await syncOpmlSource(application, source); + } if (result.success) sourcesSuccess++; else sourcesFailed++; } catch (error) { @@ -58,14 +64,21 @@ export async function runFullSync(application, options = {}) { } } - // Sync all non-hidden blogs + // Sync all non-hidden blogs (skip microsub blogs - their items come from Microsub) const blogs = await getBlogs(application, { includeHidden: false, limit: 1000 }); let blogsSuccess = 0; let blogsFailed = 0; + let blogsSkipped = 0; let newItems = 0; for (const blog of blogs) { + // Skip microsub blogs - items are served directly from microsub_items + if (blog.source === "microsub" || blog.skipItemFetch) { + blogsSkipped++; + continue; + } + try { const result = await syncBlogItems(application, blog, { maxItems: maxItemsPerBlog, @@ -84,6 +97,10 @@ export async function runFullSync(application, options = {}) { } } + if (blogsSkipped > 0) { + console.log(`[Blogroll] Skipped ${blogsSkipped} Microsub blogs (items served from Microsub)`); + } + const duration = Date.now() - startTime; // Update sync stats in meta collection diff --git a/package.json b/package.json index 94c4eba..647ae96 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@rmdes/indiekit-endpoint-blogroll", - "version": "1.0.6", + "version": "1.0.7", "description": "Blogroll endpoint for Indiekit. Aggregates blog feeds from OPML, JSON feeds, or manual entry.", "keywords": [ "indiekit", diff --git a/views/blogroll-source-edit.njk b/views/blogroll-source-edit.njk index 1b940d5..f57cafd 100644 --- a/views/blogroll-source-edit.njk +++ b/views/blogroll-source-edit.njk @@ -82,6 +82,9 @@ {{ __("blogroll.sources.form.typeHint") }} @@ -98,6 +101,23 @@ {{ __("blogroll.sources.form.opmlContentHint") }} + + + +