From 4819c229cddd3b94cb6a05e72b462a22b23ec598 Mon Sep 17 00:00:00 2001 From: Ricardo Date: Fri, 6 Feb 2026 20:20:25 +0100 Subject: [PATCH] feat: restore full microsub implementation with reader UI Restores complete implementation from feat/endpoint-microsub branch: - Reader UI with views (reader.njk, channel.njk, feeds.njk, etc.) - Feed polling, parsing, and normalization - WebSub subscriber - SSE realtime updates - Redis caching - Search indexing - Media proxy - Webmention processing --- assets/styles.css | 764 +++++++++++++++++++++++++++++++++++ index.js | 113 +++++- lib/cache/redis.js | 181 +++++++++ lib/controllers/block.js | 86 ++++ lib/controllers/channels.js | 34 +- lib/controllers/events.js | 57 +++ lib/controllers/follow.js | 128 ++++++ lib/controllers/microsub.js | 72 +++- lib/controllers/mute.js | 125 ++++++ lib/controllers/preview.js | 67 +++ lib/controllers/reader.js | 586 +++++++++++++++++++++++++++ lib/controllers/search.js | 143 +++++++ lib/controllers/timeline.js | 14 +- lib/feeds/atom.js | 61 +++ lib/feeds/fetcher.js | 316 +++++++++++++++ lib/feeds/hfeed.js | 177 ++++++++ lib/feeds/jsonfeed.js | 43 ++ lib/feeds/normalizer.js | 697 ++++++++++++++++++++++++++++++++ lib/feeds/parser.js | 135 +++++++ lib/feeds/rss.js | 61 +++ lib/media/proxy.js | 219 ++++++++++ lib/polling/processor.js | 234 +++++++++++ lib/polling/scheduler.js | 128 ++++++ lib/polling/tier.js | 139 +++++++ lib/realtime/broker.js | 241 +++++++++++ lib/search/indexer.js | 90 +++++ lib/search/query.js | 198 +++++++++ lib/storage/channels.js | 68 +++- lib/storage/feeds.js | 299 ++++++++++++++ lib/storage/filters.js | 265 ++++++++++++ lib/storage/items.js | 306 +++++++++++++- lib/storage/read-state.js | 109 +++++ lib/utils/auth.js | 3 +- lib/utils/jf2.js | 170 ++++++++ lib/utils/pagination.js | 31 +- lib/utils/uid.js | 17 - lib/utils/validation.js | 94 ++++- lib/webmention/processor.js | 214 ++++++++++ lib/webmention/receiver.js | 56 +++ lib/webmention/verifier.js | 308 ++++++++++++++ lib/websub/discovery.js | 129 ++++++ lib/websub/handler.js | 163 ++++++++ lib/websub/subscriber.js | 181 +++++++++ locales/en.json | 80 +++- package.json | 22 +- views/404.njk | 17 + views/channel-new.njk | 31 ++ views/channel.njk | 102 +++++ views/compose.njk | 98 +++++ views/feeds.njk | 69 ++++ views/item.njk | 151 +++++++ views/layouts/reader.njk | 10 + views/partials/actions.njk | 15 + views/partials/author.njk | 17 + views/partials/item-card.njk | 179 ++++++++ views/partials/timeline.njk | 10 + views/reader.njk | 41 ++ views/search.njk | 61 +++ views/settings.njk | 75 ++++ 59 files changed, 8418 insertions(+), 82 deletions(-) create mode 100644 assets/styles.css create mode 100644 lib/cache/redis.js create mode 100644 lib/controllers/block.js create mode 100644 lib/controllers/events.js create mode 100644 lib/controllers/follow.js create mode 100644 lib/controllers/mute.js create mode 100644 lib/controllers/preview.js create mode 100644 lib/controllers/reader.js create mode 100644 lib/controllers/search.js create mode 100644 lib/feeds/atom.js create mode 100644 lib/feeds/fetcher.js create mode 100644 lib/feeds/hfeed.js create mode 100644 lib/feeds/jsonfeed.js create mode 100644 lib/feeds/normalizer.js create mode 100644 lib/feeds/parser.js create mode 100644 lib/feeds/rss.js create mode 100644 lib/media/proxy.js create mode 100644 lib/polling/processor.js create mode 100644 lib/polling/scheduler.js create mode 100644 lib/polling/tier.js create mode 100644 lib/realtime/broker.js create mode 100644 lib/search/indexer.js create mode 100644 lib/search/query.js create mode 100644 lib/storage/feeds.js create mode 100644 lib/storage/filters.js create mode 100644 lib/storage/read-state.js create mode 100644 lib/utils/jf2.js delete mode 100644 lib/utils/uid.js create mode 100644 lib/webmention/processor.js create mode 100644 lib/webmention/receiver.js create mode 100644 lib/webmention/verifier.js create mode 100644 lib/websub/discovery.js create mode 100644 lib/websub/handler.js create mode 100644 lib/websub/subscriber.js create mode 100644 views/404.njk create mode 100644 views/channel-new.njk create mode 100644 views/channel.njk create mode 100644 views/compose.njk create mode 100644 views/feeds.njk create mode 100644 views/item.njk create mode 100644 views/layouts/reader.njk create mode 100644 views/partials/actions.njk create mode 100644 views/partials/author.njk create mode 100644 views/partials/item-card.njk create mode 100644 views/partials/timeline.njk create mode 100644 views/reader.njk create mode 100644 views/search.njk create mode 100644 views/settings.njk diff --git a/assets/styles.css b/assets/styles.css new file mode 100644 index 0000000..6921f92 --- /dev/null +++ b/assets/styles.css @@ -0,0 +1,764 @@ +/** + * Microsub Reader Styles + * Inspired by Aperture/Monocle + */ + +/* ========================================================================== + Reader Layout + ========================================================================== */ + +.reader { + display: flex; + flex-direction: column; + gap: var(--space-m); +} + +.reader__header { + align-items: center; + display: flex; + flex-wrap: wrap; + gap: var(--space-s); + justify-content: space-between; +} + +.reader__actions { + display: flex; + flex-wrap: wrap; + gap: var(--space-xs); +} + +/* ========================================================================== + Channel List + ========================================================================== */ + +.reader__channels { + display: flex; + flex-direction: column; + gap: var(--space-xs); +} + +.reader__channel { + align-items: center; + background: var(--color-offset); + border-radius: var(--border-radius); + color: inherit; + display: flex; + gap: var(--space-s); + padding: var(--space-s) var(--space-m); + text-decoration: none; + transition: + background-color 0.2s ease, + box-shadow 0.2s ease; +} + +.reader__channel:hover { + background: var(--color-offset-active); +} + +.reader__channel--active { + background: var(--color-primary); + color: var(--color-background); +} + +.reader__channel-name { + flex: 1; + font-weight: 500; +} + +.reader__channel-badge { + align-items: center; + background: var(--color-primary); + border-radius: 0.75rem; + color: var(--color-background); + display: inline-flex; + font-size: var(--font-size-small); + font-weight: 600; + height: 1.5rem; + justify-content: center; + min-width: 1.5rem; + padding: 0 var(--space-xs); +} + +.reader__channel--active .reader__channel-badge { + background: var(--color-background); + color: var(--color-primary); +} + +/* Dot indicator for boolean unread state */ +.reader__channel-badge--dot { + height: 0.75rem; + min-width: 0.75rem; + padding: 0; + width: 0.75rem; +} + +/* ========================================================================== + Timeline + ========================================================================== */ + +.timeline { + display: flex; + flex-direction: column; + gap: var(--space-m); +} + +.timeline__paging { + border-top: 1px solid var(--color-offset); + display: flex; + gap: var(--space-m); + justify-content: space-between; + padding-top: var(--space-m); +} + +/* ========================================================================== + Item Card + ========================================================================== */ + +.item-card { + background: var(--color-background); + border: 1px solid var(--color-offset); + border-radius: var(--border-radius); + display: block; + overflow: hidden; + transition: + box-shadow 0.2s ease, + transform 0.1s ease; +} + +.item-card:hover { + border-color: var(--color-offset-active); +} + +/* Unread state - yellow glow (Aperture pattern) */ +.item-card:not(.item-card--read) { + border-color: rgba(255, 204, 0, 0.5); + box-shadow: 0 0 10px 0 rgba(255, 204, 0, 0.8); +} + +.item-card--read { + opacity: 0.85; +} + +.item-card__link { + color: inherit; + display: block; + padding: var(--space-m); + text-decoration: none; +} + +/* Author */ +.item-card__author { + align-items: center; + display: flex; + gap: var(--space-s); + margin-bottom: var(--space-s); +} + +.item-card__author-photo { + border: 1px solid var(--color-offset); + border-radius: 50%; + flex-shrink: 0; + height: 40px; + object-fit: cover; + width: 40px; +} + +.item-card__author-info { + display: flex; + flex-direction: column; + min-width: 0; +} + +.item-card__author-name { + font-size: var(--font-size-body); + font-weight: 600; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.item-card__source { + color: var(--color-text-muted); + font-size: var(--font-size-small); + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +/* Post type indicator */ +.item-card__type { + align-items: center; + background: var(--color-offset); + border-radius: var(--border-radius); + color: var(--color-text-muted); + display: inline-flex; + font-size: var(--font-size-small); + gap: var(--space-xs); + margin-bottom: var(--space-s); + padding: var(--space-xs) var(--space-s); +} + +.item-card__type svg { + height: 1em; + width: 1em; +} + +/* Context bar for interactions (Aperture pattern) */ +.item-card__context { + align-items: center; + background: var(--color-offset); + display: flex; + font-size: var(--font-size-small); + gap: var(--space-xs); + margin: calc(-1 * var(--space-m)); + margin-bottom: var(--space-s); + padding: var(--space-s) var(--space-m); +} + +.item-card__context a { + color: var(--color-primary); + text-decoration: none; +} + +.item-card__context a:hover { + text-decoration: underline; +} + +.item-card__context svg { + flex-shrink: 0; + height: 1em; + width: 1em; +} + +/* Title */ +.item-card__title { + font-size: var(--font-size-heading-4); + font-weight: 600; + line-height: 1.3; + margin-bottom: var(--space-xs); +} + +/* Content with expandable overflow (Aperture pattern) */ +.item-card__content { + color: var(--color-text); + line-height: 1.5; + margin-bottom: var(--space-s); + max-height: 200px; + overflow: hidden; + position: relative; +} + +.item-card__content--expanded { + max-height: none; +} + +.item-card__content--truncated::after { + background: linear-gradient(to bottom, transparent, var(--color-background)); + bottom: 0; + content: ""; + height: 60px; + left: 0; + pointer-events: none; + position: absolute; + right: 0; +} + +.item-card__read-more { + color: var(--color-primary); + cursor: pointer; + display: block; + font-size: var(--font-size-small); + padding: var(--space-xs); + text-align: center; +} + +/* Photo grid (Aperture multi-photo pattern) */ +.item-card__photos { + border-radius: var(--border-radius); + display: grid; + gap: 2px; + margin-bottom: var(--space-s); + overflow: hidden; +} + +/* Single photo */ +.item-card__photos--1 { + grid-template-columns: 1fr; +} + +/* 2 photos - side by side */ +.item-card__photos--2 { + grid-template-columns: 1fr 1fr; +} + +/* 3 photos - one large, two small */ +.item-card__photos--3 { + grid-template-columns: 2fr 1fr; + grid-template-rows: 1fr 1fr; +} + +/* 4+ photos - grid */ +.item-card__photos--4 { + grid-template-columns: 1fr 1fr; + grid-template-rows: 1fr 1fr; +} + +/* Base photo styles - must come before specific overrides */ +.item-card__photo { + background: var(--color-offset); + height: 150px; + object-fit: cover; + width: 100%; +} + +.item-card__photos--1 .item-card__photo { + height: auto; + max-height: 400px; +} + +.item-card__photos--3 .item-card__photo:first-child { + grid-row: 1 / 3; + height: 302px; +} + +/* Video/Audio */ +.item-card__video, +.item-card__audio { + border-radius: var(--border-radius); + margin-bottom: var(--space-s); + width: 100%; +} + +/* Footer */ +.item-card__footer { + align-items: center; + border-top: 1px solid var(--color-offset); + color: var(--color-text-muted); + display: flex; + font-size: var(--font-size-small); + justify-content: space-between; + padding-top: var(--space-s); +} + +.item-card__date { + color: inherit; +} + +.item-card__unread { + color: var(--color-warning, #ffcc00); + font-size: 0.75rem; +} + +/* Categories/Tags */ +.item-card__categories { + display: flex; + flex-wrap: wrap; + gap: var(--space-xs); + margin-bottom: var(--space-s); +} + +.item-card__category { + background: var(--color-offset); + border-radius: var(--border-radius); + color: var(--color-text-muted); + display: inline-block; + font-size: var(--font-size-small); + padding: 2px var(--space-xs); +} + +/* ========================================================================== + Item Actions (inline on cards) + ========================================================================== */ + +.item-actions { + border-top: 1px solid var(--color-offset); + display: flex; + gap: var(--space-s); + padding-top: var(--space-s); +} + +.item-actions__button { + align-items: center; + background: transparent; + border: 1px solid var(--color-offset); + border-radius: var(--border-radius); + color: var(--color-text-muted); + cursor: pointer; + display: inline-flex; + font-size: var(--font-size-small); + gap: var(--space-xs); + padding: var(--space-xs) var(--space-s); + text-decoration: none; + transition: all 0.2s ease; +} + +.item-actions__button:hover { + background: var(--color-offset); + border-color: var(--color-offset-active); + color: var(--color-text); +} + +.item-actions__button svg { + height: 1em; + width: 1em; +} + +.item-actions__button--primary { + background: var(--color-primary); + border-color: var(--color-primary); + color: var(--color-background); +} + +.item-actions__button--primary:hover { + background: var(--color-primary-dark, var(--color-primary)); + border-color: var(--color-primary-dark, var(--color-primary)); + color: var(--color-background); +} + +/* Mark as read button */ +.item-actions__mark-read { + margin-left: auto; +} + +/* ========================================================================== + Single Item View + ========================================================================== */ + +.item { + max-width: 40rem; +} + +.item__header { + margin-bottom: var(--space-m); +} + +.item__author { + align-items: center; + display: flex; + gap: var(--space-s); + margin-bottom: var(--space-m); +} + +.item__author-photo { + border-radius: 50%; + height: 48px; + object-fit: cover; + width: 48px; +} + +.item__author-info { + display: flex; + flex-direction: column; +} + +.item__author-name { + font-weight: 600; +} + +.item__date { + color: var(--color-text-muted); + font-size: var(--font-size-small); +} + +.item__title { + font-size: var(--font-size-heading-2); + margin-bottom: var(--space-m); +} + +.item__content { + line-height: 1.6; + margin-bottom: var(--space-m); +} + +.item__content img { + border-radius: var(--border-radius); + height: auto; + max-width: 100%; +} + +.item__photos { + display: grid; + gap: var(--space-s); + margin-bottom: var(--space-m); +} + +.item__photo { + border-radius: var(--border-radius); + width: 100%; +} + +.item__context { + background: var(--color-offset); + border-radius: var(--border-radius); + margin-bottom: var(--space-m); + padding: var(--space-m); +} + +.item__context-label { + color: var(--color-text-muted); + font-size: var(--font-size-small); + margin-bottom: var(--space-xs); +} + +.item__actions { + border-top: 1px solid var(--color-offset); + display: flex; + flex-wrap: wrap; + gap: var(--space-s); + padding-top: var(--space-m); +} + +/* ========================================================================== + Channel Header + ========================================================================== */ + +.channel__header { + align-items: center; + display: flex; + flex-wrap: wrap; + gap: var(--space-s); + justify-content: space-between; + margin-bottom: var(--space-m); +} + +.channel__actions { + display: flex; + gap: var(--space-xs); +} + +/* ========================================================================== + Feeds Management + ========================================================================== */ + +.feeds { + display: flex; + flex-direction: column; + gap: var(--space-m); +} + +.feeds__header { + align-items: center; + display: flex; + flex-wrap: wrap; + gap: var(--space-s); + justify-content: space-between; +} + +.feeds__list { + display: flex; + flex-direction: column; + gap: var(--space-s); +} + +.feeds__item { + align-items: center; + background: var(--color-offset); + border-radius: var(--border-radius); + display: flex; + gap: var(--space-m); + padding: var(--space-m); +} + +.feeds__photo { + border-radius: var(--border-radius); + flex-shrink: 0; + height: 48px; + object-fit: cover; + width: 48px; +} + +.feeds__info { + flex: 1; + min-width: 0; +} + +.feeds__name { + font-weight: 600; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.feeds__url { + color: var(--color-text-muted); + font-size: var(--font-size-small); + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.feeds__actions { + flex-shrink: 0; +} + +.feeds__add { + background: var(--color-offset); + border-radius: var(--border-radius); + padding: var(--space-m); +} + +.feeds__form { + display: flex; + gap: var(--space-s); +} + +.feeds__form input { + flex: 1; +} + +/* ========================================================================== + Search + ========================================================================== */ + +.search { + display: flex; + flex-direction: column; + gap: var(--space-m); +} + +.search__form { + display: flex; + gap: var(--space-s); +} + +.search__form input { + flex: 1; +} + +.search__results { + margin-top: var(--space-m); +} + +.search__list { + display: flex; + flex-direction: column; + gap: var(--space-s); +} + +.search__item { + align-items: center; + background: var(--color-offset); + border-radius: var(--border-radius); + display: flex; + justify-content: space-between; + padding: var(--space-m); +} + +.search__feed { + flex: 1; + min-width: 0; +} + +.search__url { + color: var(--color-text-muted); + font-size: var(--font-size-small); + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +/* ========================================================================== + Compose + ========================================================================== */ + +.compose { + max-width: 40rem; +} + +.compose__context { + background: var(--color-offset); + border-radius: var(--border-radius); + margin-bottom: var(--space-m); + padding: var(--space-m); +} + +.compose__counter { + color: var(--color-text-muted); + font-size: var(--font-size-small); + margin-top: var(--space-xs); + text-align: right; +} + +/* ========================================================================== + Settings + ========================================================================== */ + +.settings { + max-width: 40rem; +} + +.settings .divider { + border-top: 1px solid var(--color-offset); + margin: var(--space-l) 0; +} + +.settings .danger-zone { + background: rgba(var(--color-error-rgb, 255, 0, 0), 0.1); + border: 1px solid var(--color-error); + border-radius: var(--border-radius); + padding: var(--space-m); +} + +/* ========================================================================== + Keyboard Navigation Focus + ========================================================================== */ + +.item-card:focus-within, +.item-card.item-card--focused { + outline: 2px solid var(--color-primary); + outline-offset: 2px; +} + +/* ========================================================================== + Empty States + ========================================================================== */ + +.reader__empty { + color: var(--color-text-muted); + padding: var(--space-xl); + text-align: center; +} + +.reader__empty svg { + height: 4rem; + margin-bottom: var(--space-m); + opacity: 0.5; + width: 4rem; +} + +/* ========================================================================== + Responsive + ========================================================================== */ + +@media (max-width: 640px) { + .item-card__photos--3 { + grid-template-columns: 1fr 1fr; + grid-template-rows: auto auto; + } + + .item-card__photos--3 .item-card__photo:first-child { + grid-column: 1 / 3; + grid-row: 1; + height: 200px; + } + + .item-card__photos--3 .item-card__photo:nth-child(2), + .item-card__photos--3 .item-card__photo:nth-child(3) { + height: 100px; + } + + .feeds__item { + flex-wrap: wrap; + } + + .feeds__info { + order: 1; + width: calc(100% - 64px); + } + + .feeds__actions { + margin-top: var(--space-s); + order: 2; + width: 100%; + } +} diff --git a/index.js b/index.js index a4e1d02..a69503a 100644 --- a/index.js +++ b/index.js @@ -1,12 +1,20 @@ +import path from "node:path"; + import express from "express"; import { microsubController } from "./lib/controllers/microsub.js"; +import { readerController } from "./lib/controllers/reader.js"; +import { handleMediaProxy } from "./lib/media/proxy.js"; +import { startScheduler, stopScheduler } from "./lib/polling/scheduler.js"; import { createIndexes } from "./lib/storage/items.js"; +import { webmentionReceiver } from "./lib/webmention/receiver.js"; +import { websubHandler } from "./lib/websub/handler.js"; const defaults = { mountPath: "/microsub", }; const router = express.Router(); +const readerRouter = express.Router(); export default class MicrosubEndpoint { name = "Microsub endpoint"; @@ -21,7 +29,32 @@ export default class MicrosubEndpoint { } /** - * Microsub API routes (authenticated) + * Navigation items for Indiekit admin + * @returns {object} Navigation item configuration + */ + get navigationItems() { + return { + href: path.join(this.options.mountPath, "reader"), + text: "microsub.reader.title", + requiresDatabase: true, + }; + } + + /** + * Shortcut items for quick actions + * @returns {object} Shortcut item configuration + */ + get shortcutItems() { + return { + url: path.join(this.options.mountPath, "reader", "channels"), + name: "microsub.channels.title", + iconName: "feed", + requiresDatabase: true, + }; + } + + /** + * Microsub API and reader UI routes (authenticated) * @returns {import("express").Router} Express router */ get routes() { @@ -29,9 +62,65 @@ export default class MicrosubEndpoint { router.get("/", microsubController.get); router.post("/", microsubController.post); + // WebSub callback endpoint + router.get("/websub/:id", websubHandler.verify); + router.post("/websub/:id", websubHandler.receive); + + // Webmention receiving endpoint + router.post("/webmention", webmentionReceiver.receive); + + // Media proxy endpoint + router.get("/media/:hash", handleMediaProxy); + + // Reader UI routes (mounted as sub-router for correct baseUrl) + readerRouter.get("/", readerController.index); + readerRouter.get("/channels", readerController.channels); + readerRouter.get("/channels/new", readerController.newChannel); + readerRouter.post("/channels/new", readerController.createChannel); + readerRouter.get("/channels/:uid", readerController.channel); + readerRouter.get("/channels/:uid/settings", readerController.settings); + readerRouter.post( + "/channels/:uid/settings", + readerController.updateSettings, + ); + readerRouter.post("/channels/:uid/delete", readerController.deleteChannel); + readerRouter.get("/channels/:uid/feeds", readerController.feeds); + readerRouter.post("/channels/:uid/feeds", readerController.addFeed); + readerRouter.post( + "/channels/:uid/feeds/remove", + readerController.removeFeed, + ); + readerRouter.get("/item/:id", readerController.item); + readerRouter.get("/compose", readerController.compose); + readerRouter.post("/compose", readerController.submitCompose); + readerRouter.get("/search", readerController.searchPage); + readerRouter.post("/search", readerController.searchFeeds); + readerRouter.post("/subscribe", readerController.subscribe); + router.use("/reader", readerRouter); + return router; } + /** + * Public routes (no authentication required) + * @returns {import("express").Router} Express router + */ + get routesPublic() { + const publicRouter = express.Router(); + + // WebSub verification must be public for hubs to verify + publicRouter.get("/websub/:id", websubHandler.verify); + publicRouter.post("/websub/:id", websubHandler.receive); + + // Webmention endpoint must be public + publicRouter.post("/webmention", webmentionReceiver.receive); + + // Media proxy must be public for images to load + publicRouter.get("/media/:hash", handleMediaProxy); + + return publicRouter; + } + /** * Initialize plugin * @param {object} indiekit - Indiekit instance @@ -41,7 +130,11 @@ export default class MicrosubEndpoint { // Register MongoDB collections indiekit.addCollection("microsub_channels"); + indiekit.addCollection("microsub_feeds"); indiekit.addCollection("microsub_items"); + indiekit.addCollection("microsub_notifications"); + indiekit.addCollection("microsub_muted"); + indiekit.addCollection("microsub_blocked"); console.info("[Microsub] Registered MongoDB collections"); @@ -53,11 +146,27 @@ export default class MicrosubEndpoint { indiekit.config.application.microsubEndpoint = this.mountPath; } - // Create indexes for optimal performance (runs in background) + // Start feed polling scheduler when server starts + // This will be called after the server is ready if (indiekit.database) { + console.info("[Microsub] Database available, starting scheduler"); + startScheduler(indiekit); + + // Create indexes for optimal performance (runs in background) createIndexes(indiekit).catch((error) => { console.warn("[Microsub] Index creation failed:", error.message); }); + } else { + console.warn( + "[Microsub] Database not available at init, scheduler not started", + ); } } + + /** + * Cleanup on shutdown + */ + destroy() { + stopScheduler(); + } } diff --git a/lib/cache/redis.js b/lib/cache/redis.js new file mode 100644 index 0000000..419a7d9 --- /dev/null +++ b/lib/cache/redis.js @@ -0,0 +1,181 @@ +/** + * Redis caching utilities + * @module cache/redis + */ + +import Redis from "ioredis"; + +let redisClient; + +/** + * Get Redis client from application + * @param {object} application - Indiekit application + * @returns {object|undefined} Redis client or undefined + */ +export function getRedisClient(application) { + // Check if Redis is already initialized on the application + if (application.redis) { + return application.redis; + } + + // Check if we already created a client + if (redisClient) { + return redisClient; + } + + // Check for Redis URL in config + const redisUrl = application.config?.application?.redisUrl; + if (redisUrl) { + try { + redisClient = new Redis(redisUrl, { + maxRetriesPerRequest: 3, + retryStrategy(times) { + const delay = Math.min(times * 50, 2000); + return delay; + }, + lazyConnect: true, + }); + + redisClient.on("error", (error) => { + console.error("[Microsub] Redis error:", error.message); + }); + + redisClient.on("connect", () => { + console.info("[Microsub] Redis connected"); + }); + + // Connect asynchronously + redisClient.connect().catch((error) => { + console.warn("[Microsub] Redis connection failed:", error.message); + }); + + return redisClient; + } catch (error) { + console.warn("[Microsub] Failed to initialize Redis:", error.message); + } + } +} + +/** + * Get value from cache + * @param {object} redis - Redis client + * @param {string} key - Cache key + * @returns {Promise} Cached value or undefined + */ +export async function getCache(redis, key) { + if (!redis) { + return; + } + + try { + const value = await redis.get(key); + if (value) { + return JSON.parse(value); + } + } catch { + // Ignore cache errors + } +} + +/** + * Set value in cache + * @param {object} redis - Redis client + * @param {string} key - Cache key + * @param {object} value - Value to cache + * @param {number} [ttl] - Time to live in seconds + * @returns {Promise} + */ +export async function setCache(redis, key, value, ttl = 300) { + if (!redis) { + return; + } + + try { + const serialized = JSON.stringify(value); + await (ttl + ? redis.set(key, serialized, "EX", ttl) + : redis.set(key, serialized)); + } catch { + // Ignore cache errors + } +} + +/** + * Delete value from cache + * @param {object} redis - Redis client + * @param {string} key - Cache key + * @returns {Promise} + */ +export async function deleteCache(redis, key) { + if (!redis) { + return; + } + + try { + await redis.del(key); + } catch { + // Ignore cache errors + } +} + +/** + * Publish event to channel + * @param {object} redis - Redis client + * @param {string} channel - Channel name + * @param {object} data - Event data + * @returns {Promise} + */ +export async function publishEvent(redis, channel, data) { + if (!redis) { + return; + } + + try { + await redis.publish(channel, JSON.stringify(data)); + } catch { + // Ignore pub/sub errors + } +} + +/** + * Subscribe to channel + * @param {object} redis - Redis client (must be separate connection for pub/sub) + * @param {string} channel - Channel name + * @param {(data: object) => void} callback - Callback function for messages + * @returns {Promise} + */ +export async function subscribeToChannel(redis, channel, callback) { + if (!redis) { + return; + } + + try { + await redis.subscribe(channel); + redis.on("message", (receivedChannel, message) => { + if (receivedChannel === channel) { + try { + const data = JSON.parse(message); + callback(data); + } catch { + callback(message); + } + } + }); + } catch { + // Ignore subscription errors + } +} + +/** + * Cleanup Redis connection on shutdown + */ +export async function closeRedis() { + if (redisClient) { + try { + await redisClient.quit(); + redisClient = undefined; + } catch { + // Ignore cleanup errors + } + } +} diff --git a/lib/controllers/block.js b/lib/controllers/block.js new file mode 100644 index 0000000..13fd09e --- /dev/null +++ b/lib/controllers/block.js @@ -0,0 +1,86 @@ +/** + * Block controller + * @module controllers/block + */ + +import { deleteItemsByAuthorUrl } from "../storage/items.js"; +import { getUserId } from "../utils/auth.js"; +import { validateUrl } from "../utils/validation.js"; + +/** + * Get blocked collection + * @param {object} application - Indiekit application + * @returns {object} MongoDB collection + */ +function getCollection(application) { + return application.collections.get("microsub_blocked"); +} + +/** + * List blocked URLs + * GET ?action=block + * @param {object} request - Express request + * @param {object} response - Express response + */ +export async function list(request, response) { + const { application } = request.app.locals; + const userId = getUserId(request); + + const collection = getCollection(application); + const blocked = await collection.find({ userId }).toArray(); + const items = blocked.map((b) => ({ url: b.url })); + + response.json({ items }); +} + +/** + * Block a URL + * POST ?action=block + * @param {object} request - Express request + * @param {object} response - Express response + */ +export async function block(request, response) { + const { application } = request.app.locals; + const userId = getUserId(request); + const { url } = request.body; + + validateUrl(url); + + const collection = getCollection(application); + + // Check if already blocked + const existing = await collection.findOne({ userId, url }); + if (!existing) { + await collection.insertOne({ + userId, + url, + createdAt: new Date(), + }); + } + + // Remove past items from blocked URL + await deleteItemsByAuthorUrl(application, userId, url); + + response.json({ result: "ok" }); +} + +/** + * Unblock a URL + * POST ?action=unblock + * @param {object} request - Express request + * @param {object} response - Express response + */ +export async function unblock(request, response) { + const { application } = request.app.locals; + const userId = getUserId(request); + const { url } = request.body; + + validateUrl(url); + + const collection = getCollection(application); + await collection.deleteOne({ userId, url }); + + response.json({ result: "ok" }); +} + +export const blockController = { list, block, unblock }; diff --git a/lib/controllers/channels.js b/lib/controllers/channels.js index be861b0..4d957cd 100644 --- a/lib/controllers/channels.js +++ b/lib/controllers/channels.js @@ -7,6 +7,7 @@ import { IndiekitError } from "@indiekit/error"; import { getChannels, + getChannel, createChannel, updateChannel, deleteChannel, @@ -16,7 +17,7 @@ import { getUserId } from "../utils/auth.js"; import { validateChannel, validateChannelName, - parseArrayParameter, + parseArrayParameter as parseArrayParametereter, } from "../utils/validation.js"; /** @@ -39,7 +40,6 @@ export async function list(request, response) { * POST ?action=channels * @param {object} request - Express request * @param {object} response - Express response - * @returns {Promise} */ export async function action(request, response) { const { application } = request.app.locals; @@ -62,7 +62,7 @@ export async function action(request, response) { // Reorder channels if (method === "order") { - const channelUids = parseArrayParameter(request.body, "channels"); + const channelUids = parseArrayParametereter(request.body, "channels"); if (channelUids.length === 0) { throw new IndiekitError("Missing channels[] parameter", { status: 400, @@ -107,4 +107,30 @@ export async function action(request, response) { }); } -export const channelsController = { list, action }; +/** + * Get a single channel by UID + * @param {object} request - Express request + * @param {object} response - Express response + */ +export async function get(request, response) { + const { application } = request.app.locals; + const userId = getUserId(request); + const { uid } = request.params; + + validateChannel(uid); + + const channel = await getChannel(application, uid, userId); + if (!channel) { + throw new IndiekitError("Channel not found", { + status: 404, + }); + } + + response.json({ + uid: channel.uid, + name: channel.name, + settings: channel.settings, + }); +} + +export const channelsController = { list, action, get }; diff --git a/lib/controllers/events.js b/lib/controllers/events.js new file mode 100644 index 0000000..0f0ddbe --- /dev/null +++ b/lib/controllers/events.js @@ -0,0 +1,57 @@ +/** + * Server-Sent Events controller + * @module controllers/events + */ + +import { + addClient, + removeClient, + sendEvent, + subscribeClient, +} from "../realtime/broker.js"; +import { getUserId } from "../utils/auth.js"; + +/** + * SSE stream endpoint + * GET ?action=events + * @param {object} request - Express request + * @param {object} response - Express response + */ +export async function stream(request, response) { + const { application } = request.app.locals; + const userId = getUserId(request); + + // Set SSE headers + response.setHeader("Content-Type", "text/event-stream"); + response.setHeader("Cache-Control", "no-cache"); + response.setHeader("Connection", "keep-alive"); + response.setHeader("X-Accel-Buffering", "no"); // Disable nginx buffering + + // Flush headers immediately + response.flushHeaders(); + + // Add client to broker (handles ping internally) + const client = addClient(response, userId, application); + + // Subscribe to channels from query parameter + const { channels } = request.query; + if (channels) { + const channelList = Array.isArray(channels) ? channels : [channels]; + for (const channelId of channelList) { + subscribeClient(response, channelId); + } + } + + // Send initial event + sendEvent(response, "started", { + version: "1.0.0", + channels: [...client.channels], + }); + + // Handle client disconnect + request.on("close", () => { + removeClient(response); + }); +} + +export const eventsController = { stream }; diff --git a/lib/controllers/follow.js b/lib/controllers/follow.js new file mode 100644 index 0000000..7493e1f --- /dev/null +++ b/lib/controllers/follow.js @@ -0,0 +1,128 @@ +/** + * Follow/unfollow controller + * @module controllers/follow + */ + +import { IndiekitError } from "@indiekit/error"; + +import { refreshFeedNow } from "../polling/scheduler.js"; +import { getChannel } from "../storage/channels.js"; +import { + createFeed, + deleteFeed, + getFeedByUrl, + getFeedsForChannel, +} from "../storage/feeds.js"; +import { getUserId } from "../utils/auth.js"; +import { createFeedResponse } from "../utils/jf2.js"; +import { validateChannel, validateUrl } from "../utils/validation.js"; +import { + unsubscribe as websubUnsubscribe, + getCallbackUrl, +} from "../websub/subscriber.js"; + +/** + * List followed feeds for a channel + * GET ?action=follow&channel= + * @param {object} request - Express request + * @param {object} response - Express response + */ +export async function list(request, response) { + const { application } = request.app.locals; + const userId = getUserId(request); + const { channel } = request.query; + + validateChannel(channel); + + const channelDocument = await getChannel(application, channel, userId); + if (!channelDocument) { + throw new IndiekitError("Channel not found", { status: 404 }); + } + + const feeds = await getFeedsForChannel(application, channelDocument._id); + const items = feeds.map((feed) => createFeedResponse(feed)); + + response.json({ items }); +} + +/** + * Follow a feed URL + * POST ?action=follow + * @param {object} request - Express request + * @param {object} response - Express response + */ +export async function follow(request, response) { + const { application } = request.app.locals; + const userId = getUserId(request); + const { channel, url } = request.body; + + validateChannel(channel); + validateUrl(url); + + const channelDocument = await getChannel(application, channel, userId); + if (!channelDocument) { + throw new IndiekitError("Channel not found", { status: 404 }); + } + + // Create feed subscription + const feed = await createFeed(application, { + channelId: channelDocument._id, + url, + title: undefined, // Will be populated on first fetch + photo: undefined, + }); + + // Trigger immediate fetch in background (don't await) + // This will also discover and subscribe to WebSub hubs + refreshFeedNow(application, feed._id).catch((error) => { + console.error(`[Microsub] Error fetching new feed ${url}:`, error.message); + }); + + response.status(201).json(createFeedResponse(feed)); +} + +/** + * Unfollow a feed URL + * POST ?action=unfollow + * @param {object} request - Express request + * @param {object} response - Express response + */ +export async function unfollow(request, response) { + const { application } = request.app.locals; + const userId = getUserId(request); + const { channel, url } = request.body; + + validateChannel(channel); + validateUrl(url); + + const channelDocument = await getChannel(application, channel, userId); + if (!channelDocument) { + throw new IndiekitError("Channel not found", { status: 404 }); + } + + // Get feed before deletion to check for WebSub subscription + const feed = await getFeedByUrl(application, channelDocument._id, url); + + // Unsubscribe from WebSub hub if active + if (feed?.websub?.hub) { + const baseUrl = application.url; + if (baseUrl) { + const callbackUrl = getCallbackUrl(baseUrl, feed._id.toString()); + websubUnsubscribe(application, feed, callbackUrl).catch((error) => { + console.error( + `[Microsub] WebSub unsubscribe error for ${url}:`, + error.message, + ); + }); + } + } + + const deleted = await deleteFeed(application, channelDocument._id, url); + if (!deleted) { + throw new IndiekitError("Feed not found", { status: 404 }); + } + + response.json({ result: "ok" }); +} + +export const followController = { list, follow, unfollow }; diff --git a/lib/controllers/microsub.js b/lib/controllers/microsub.js index a25fd67..c007b6d 100644 --- a/lib/controllers/microsub.js +++ b/lib/controllers/microsub.js @@ -7,7 +7,13 @@ import { IndiekitError } from "@indiekit/error"; import { validateAction } from "../utils/validation.js"; +import { list as listBlocked, block, unblock } from "./block.js"; import { list as listChannels, action as channelAction } from "./channels.js"; +import { stream as eventsStream } from "./events.js"; +import { list as listFollows, follow, unfollow } from "./follow.js"; +import { list as listMuted, mute, unmute } from "./mute.js"; +import { get as getPreview, preview } from "./preview.js"; +import { discover, search } from "./search.js"; import { get as getTimeline, action as timelineAction } from "./timeline.js"; /** @@ -15,18 +21,14 @@ import { get as getTimeline, action as timelineAction } from "./timeline.js"; * @param {object} request - Express request * @param {object} response - Express response * @param {Function} next - Express next function - * @returns {Promise} */ export async function get(request, response, next) { try { const { action } = request.query; + // If no action provided, redirect to reader UI if (!action) { - // Return basic endpoint info - return response.json({ - type: "microsub", - actions: ["channels", "timeline"], - }); + return response.redirect(request.baseUrl + "/reader"); } validateAction(action); @@ -40,6 +42,31 @@ export async function get(request, response, next) { return getTimeline(request, response); } + case "follow": { + return listFollows(request, response); + } + + case "preview": { + return getPreview(request, response); + } + + case "mute": { + return listMuted(request, response); + } + + case "block": { + return listBlocked(request, response); + } + + case "events": { + return eventsStream(request, response); + } + + case "search": { + // Search is typically POST, but GET is allowed for feed discovery + return discover(request, response); + } + default: { throw new IndiekitError(`Unsupported GET action: ${action}`, { status: 400, @@ -56,7 +83,6 @@ export async function get(request, response, next) { * @param {object} request - Express request * @param {object} response - Express response * @param {Function} next - Express next function - * @returns {Promise} */ export async function post(request, response, next) { try { @@ -72,6 +98,38 @@ export async function post(request, response, next) { return timelineAction(request, response); } + case "follow": { + return follow(request, response); + } + + case "unfollow": { + return unfollow(request, response); + } + + case "search": { + return search(request, response); + } + + case "preview": { + return preview(request, response); + } + + case "mute": { + return mute(request, response); + } + + case "unmute": { + return unmute(request, response); + } + + case "block": { + return block(request, response); + } + + case "unblock": { + return unblock(request, response); + } + default: { throw new IndiekitError(`Unsupported POST action: ${action}`, { status: 400, diff --git a/lib/controllers/mute.js b/lib/controllers/mute.js new file mode 100644 index 0000000..ea9efd4 --- /dev/null +++ b/lib/controllers/mute.js @@ -0,0 +1,125 @@ +/** + * Mute controller + * @module controllers/mute + */ + +import { IndiekitError } from "@indiekit/error"; + +import { getUserId } from "../utils/auth.js"; +import { validateChannel, validateUrl } from "../utils/validation.js"; + +/** + * Get muted collection + * @param {object} application - Indiekit application + * @returns {object} MongoDB collection + */ +function getCollection(application) { + return application.collections.get("microsub_muted"); +} + +/** + * List muted URLs for a channel + * GET ?action=mute&channel= + * @param {object} request - Express request + * @param {object} response - Express response + */ +export async function list(request, response) { + const { application } = request.app.locals; + const userId = getUserId(request); + const { channel } = request.query; + + // Channel can be "global" or a specific channel UID + const isGlobal = channel === "global"; + + const collection = getCollection(application); + const filter = { userId }; + + if (!isGlobal && channel) { + // Get channel-specific mutes + const channelsCollection = application.collections.get("microsub_channels"); + const channelDocument = await channelsCollection.findOne({ uid: channel }); + if (channelDocument) { + filter.channelId = channelDocument._id; + } + } + // For global mutes, we query without channelId (matches all channels) + + // eslint-disable-next-line unicorn/no-array-callback-reference -- filter is MongoDB query object + const muted = await collection.find(filter).toArray(); + const items = muted.map((m) => ({ url: m.url })); + + response.json({ items }); +} + +/** + * Mute a URL + * POST ?action=mute + * @param {object} request - Express request + * @param {object} response - Express response + */ +export async function mute(request, response) { + const { application } = request.app.locals; + const userId = getUserId(request); + const { channel, url } = request.body; + + validateUrl(url); + + const collection = getCollection(application); + const isGlobal = channel === "global" || !channel; + + let channelId; + if (!isGlobal) { + validateChannel(channel); + const channelsCollection = application.collections.get("microsub_channels"); + const channelDocument = await channelsCollection.findOne({ uid: channel }); + if (!channelDocument) { + throw new IndiekitError("Channel not found", { status: 404 }); + } + channelId = channelDocument._id; + } + + // Check if already muted + const existing = await collection.findOne({ userId, channelId, url }); + if (!existing) { + await collection.insertOne({ + userId, + channelId, + url, + createdAt: new Date(), + }); + } + + response.json({ result: "ok" }); +} + +/** + * Unmute a URL + * POST ?action=unmute + * @param {object} request - Express request + * @param {object} response - Express response + */ +export async function unmute(request, response) { + const { application } = request.app.locals; + const userId = getUserId(request); + const { channel, url } = request.body; + + validateUrl(url); + + const collection = getCollection(application); + const isGlobal = channel === "global" || !channel; + + let channelId; + if (!isGlobal) { + const channelsCollection = application.collections.get("microsub_channels"); + const channelDocument = await channelsCollection.findOne({ uid: channel }); + if (channelDocument) { + channelId = channelDocument._id; + } + } + + await collection.deleteOne({ userId, channelId, url }); + + response.json({ result: "ok" }); +} + +export const muteController = { list, mute, unmute }; diff --git a/lib/controllers/preview.js b/lib/controllers/preview.js new file mode 100644 index 0000000..89636c9 --- /dev/null +++ b/lib/controllers/preview.js @@ -0,0 +1,67 @@ +/** + * Preview controller + * @module controllers/preview + */ + +import { IndiekitError } from "@indiekit/error"; + +import { fetchAndParseFeed } from "../feeds/fetcher.js"; +import { validateUrl } from "../utils/validation.js"; + +const MAX_PREVIEW_ITEMS = 10; + +/** + * Fetch and preview a feed + * @param {string} url - Feed URL + * @returns {Promise} Preview response + */ +async function fetchPreview(url) { + try { + const parsed = await fetchAndParseFeed(url); + + // Return feed metadata and sample items + return { + type: "feed", + url: parsed.url, + name: parsed.name, + photo: parsed.photo, + items: parsed.items.slice(0, MAX_PREVIEW_ITEMS), + }; + } catch (error) { + throw new IndiekitError(`Failed to preview feed: ${error.message}`, { + status: 502, + }); + } +} + +/** + * Preview a feed URL (GET) + * GET ?action=preview&url= + * @param {object} request - Express request + * @param {object} response - Express response + */ +export async function get(request, response) { + const { url } = request.query; + + validateUrl(url); + + const preview = await fetchPreview(url); + response.json(preview); +} + +/** + * Preview a feed URL (POST) + * POST ?action=preview + * @param {object} request - Express request + * @param {object} response - Express response + */ +export async function preview(request, response) { + const { url } = request.body; + + validateUrl(url); + + const previewData = await fetchPreview(url); + response.json(previewData); +} + +export const previewController = { get, preview }; diff --git a/lib/controllers/reader.js b/lib/controllers/reader.js new file mode 100644 index 0000000..7fcda50 --- /dev/null +++ b/lib/controllers/reader.js @@ -0,0 +1,586 @@ +/** + * Reader UI controller + * @module controllers/reader + */ + +import { discoverFeedsFromUrl } from "../feeds/fetcher.js"; +import { refreshFeedNow } from "../polling/scheduler.js"; +import { + getChannels, + getChannel, + createChannel, + updateChannelSettings, + deleteChannel, +} from "../storage/channels.js"; +import { + getFeedsForChannel, + createFeed, + deleteFeed, +} from "../storage/feeds.js"; +import { getTimelineItems, getItemById } from "../storage/items.js"; +import { getUserId } from "../utils/auth.js"; +import { + validateChannelName, + validateExcludeTypes, + validateExcludeRegex, +} from "../utils/validation.js"; + +/** + * Reader index - redirect to channels + * @param {object} request - Express request + * @param {object} response - Express response + */ +export async function index(request, response) { + response.redirect(`${request.baseUrl}/channels`); +} + +/** + * List channels + * @param {object} request - Express request + * @param {object} response - Express response + */ +export async function channels(request, response) { + const { application } = request.app.locals; + const userId = getUserId(request); + + const channelList = await getChannels(application, userId); + + response.render("reader", { + title: request.__("microsub.reader.title"), + channels: channelList, + baseUrl: request.baseUrl, + }); +} + +/** + * New channel form + * @param {object} request - Express request + * @param {object} response - Express response + */ +export async function newChannel(request, response) { + response.render("channel-new", { + title: request.__("microsub.channels.new"), + baseUrl: request.baseUrl, + }); +} + +/** + * Create channel + * @param {object} request - Express request + * @param {object} response - Express response + */ +export async function createChannelAction(request, response) { + const { application } = request.app.locals; + const userId = getUserId(request); + const { name } = request.body; + + validateChannelName(name); + + await createChannel(application, { name, userId }); + + response.redirect(`${request.baseUrl}/channels`); +} + +/** + * View channel timeline + * @param {object} request - Express request + * @param {object} response - Express response + * @returns {Promise} + */ +export async function channel(request, response) { + const { application } = request.app.locals; + const userId = getUserId(request); + const { uid } = request.params; + const { before, after } = request.query; + + const channelDocument = await getChannel(application, uid, userId); + if (!channelDocument) { + return response.status(404).render("404"); + } + + const timeline = await getTimelineItems(application, channelDocument._id, { + before, + after, + userId, + }); + + response.render("channel", { + title: channelDocument.name, + channel: channelDocument, + items: timeline.items, + paging: timeline.paging, + baseUrl: request.baseUrl, + }); +} + +/** + * Channel settings form + * @param {object} request - Express request + * @param {object} response - Express response + * @returns {Promise} + */ +export async function settings(request, response) { + const { application } = request.app.locals; + const userId = getUserId(request); + const { uid } = request.params; + + const channelDocument = await getChannel(application, uid, userId); + if (!channelDocument) { + return response.status(404).render("404"); + } + + response.render("settings", { + title: request.__("microsub.settings.title", { + channel: channelDocument.name, + }), + channel: channelDocument, + baseUrl: request.baseUrl, + }); +} + +/** + * Update channel settings + * @param {object} request - Express request + * @param {object} response - Express response + * @returns {Promise} + */ +export async function updateSettings(request, response) { + const { application } = request.app.locals; + const userId = getUserId(request); + const { uid } = request.params; + const { excludeTypes, excludeRegex } = request.body; + + const channelDocument = await getChannel(application, uid, userId); + if (!channelDocument) { + return response.status(404).render("404"); + } + + const validatedTypes = validateExcludeTypes( + Array.isArray(excludeTypes) ? excludeTypes : [excludeTypes].filter(Boolean), + ); + const validatedRegex = validateExcludeRegex(excludeRegex); + + await updateChannelSettings( + application, + uid, + { + excludeTypes: validatedTypes, + excludeRegex: validatedRegex, + }, + userId, + ); + + response.redirect(`${request.baseUrl}/channels/${uid}`); +} + +/** + * Delete channel + * @param {object} request - Express request + * @param {object} response - Express response + * @returns {Promise} + */ +export async function deleteChannelAction(request, response) { + const { application } = request.app.locals; + const userId = getUserId(request); + const { uid } = request.params; + + // Don't allow deleting notifications channel + if (uid === "notifications") { + return response.redirect(`${request.baseUrl}/channels`); + } + + const channelDocument = await getChannel(application, uid, userId); + if (!channelDocument) { + return response.status(404).render("404"); + } + + await deleteChannel(application, uid, userId); + + response.redirect(`${request.baseUrl}/channels`); +} + +/** + * View feeds for a channel + * @param {object} request - Express request + * @param {object} response - Express response + * @returns {Promise} + */ +export async function feeds(request, response) { + const { application } = request.app.locals; + const userId = getUserId(request); + const { uid } = request.params; + + const channelDocument = await getChannel(application, uid, userId); + if (!channelDocument) { + return response.status(404).render("404"); + } + + const feedList = await getFeedsForChannel(application, channelDocument._id); + + response.render("feeds", { + title: request.__("microsub.feeds.title"), + channel: channelDocument, + feeds: feedList, + baseUrl: request.baseUrl, + }); +} + +/** + * Add feed to channel + * @param {object} request - Express request + * @param {object} response - Express response + * @returns {Promise} + */ +export async function addFeed(request, response) { + const { application } = request.app.locals; + const userId = getUserId(request); + const { uid } = request.params; + const { url } = request.body; + + const channelDocument = await getChannel(application, uid, userId); + if (!channelDocument) { + return response.status(404).render("404"); + } + + // Create feed subscription + const feed = await createFeed(application, { + channelId: channelDocument._id, + url, + title: undefined, + photo: undefined, + }); + + // Trigger immediate fetch in background + refreshFeedNow(application, feed._id).catch((error) => { + console.error(`[Microsub] Error fetching new feed ${url}:`, error.message); + }); + + response.redirect(`${request.baseUrl}/channels/${uid}/feeds`); +} + +/** + * Remove feed from channel + * @param {object} request - Express request + * @param {object} response - Express response + * @returns {Promise} + */ +export async function removeFeed(request, response) { + const { application } = request.app.locals; + const userId = getUserId(request); + const { uid } = request.params; + const { url } = request.body; + + const channelDocument = await getChannel(application, uid, userId); + if (!channelDocument) { + return response.status(404).render("404"); + } + + await deleteFeed(application, channelDocument._id, url); + + response.redirect(`${request.baseUrl}/channels/${uid}/feeds`); +} + +/** + * View single item + * @param {object} request - Express request + * @param {object} response - Express response + * @returns {Promise} + */ +export async function item(request, response) { + const { application } = request.app.locals; + const userId = getUserId(request); + const { id } = request.params; + + const itemDocument = await getItemById(application, id, userId); + if (!itemDocument) { + return response.status(404).render("404"); + } + + response.render("item", { + title: itemDocument.name || "Item", + item: itemDocument, + baseUrl: request.baseUrl, + }); +} + +/** + * Ensure value is a string URL + * @param {string|object|undefined} value - Value to check + * @returns {string|undefined} String value or undefined + */ +function ensureString(value) { + if (!value) return; + if (typeof value === "string") return value; + if (typeof value === "object" && value.url) return value.url; + return String(value); +} + +/** + * Compose response form + * @param {object} request - Express request + * @param {object} response - Express response + * @returns {Promise} + */ +export async function compose(request, response) { + // Support both long-form (replyTo) and short-form (reply) query params + const { + replyTo, + reply, + likeOf, + like, + repostOf, + repost, + bookmarkOf, + bookmark, + } = request.query; + + response.render("compose", { + title: request.__("microsub.compose.title"), + replyTo: ensureString(replyTo || reply), + likeOf: ensureString(likeOf || like), + repostOf: ensureString(repostOf || repost), + bookmarkOf: ensureString(bookmarkOf || bookmark), + baseUrl: request.baseUrl, + }); +} + +/** + * Submit composed response via Micropub + * @param {object} request - Express request + * @param {object} response - Express response + * @returns {Promise} + */ +export async function submitCompose(request, response) { + const { application } = request.app.locals; + const { content } = request.body; + const inReplyTo = request.body["in-reply-to"]; + const likeOf = request.body["like-of"]; + const repostOf = request.body["repost-of"]; + const bookmarkOf = request.body["bookmark-of"]; + + // Debug logging + console.info( + "[Microsub] submitCompose request.body:", + JSON.stringify(request.body), + ); + console.info("[Microsub] Extracted values:", { + content, + inReplyTo, + likeOf, + repostOf, + bookmarkOf, + }); + + // Get Micropub endpoint + const micropubEndpoint = application.micropubEndpoint; + if (!micropubEndpoint) { + return response.status(500).render("error", { + title: "Error", + content: "Micropub endpoint not configured", + }); + } + + // Build absolute Micropub URL + const micropubUrl = micropubEndpoint.startsWith("http") + ? micropubEndpoint + : new URL(micropubEndpoint, application.url).href; + + // Get auth token from session + const token = request.session?.access_token; + if (!token) { + return response.redirect("/session/login?redirect=" + request.originalUrl); + } + + // Build Micropub request body + const micropubData = new URLSearchParams(); + micropubData.append("h", "entry"); + + if (likeOf) { + // Like post (no content needed) + micropubData.append("like-of", likeOf); + } else if (repostOf) { + // Repost (no content needed) + micropubData.append("repost-of", repostOf); + } else if (bookmarkOf) { + // Bookmark (content optional) + micropubData.append("bookmark-of", bookmarkOf); + if (content) { + micropubData.append("content", content); + } + } else if (inReplyTo) { + // Reply + micropubData.append("in-reply-to", inReplyTo); + micropubData.append("content", content || ""); + } else { + // Regular note + micropubData.append("content", content || ""); + } + + // Debug: log what we're sending + console.info("[Microsub] Sending to Micropub:", { + url: micropubUrl, + body: micropubData.toString(), + }); + + try { + const micropubResponse = await fetch(micropubUrl, { + method: "POST", + headers: { + Authorization: `Bearer ${token}`, + "Content-Type": "application/x-www-form-urlencoded", + Accept: "application/json", + }, + body: micropubData.toString(), + }); + + if ( + micropubResponse.ok || + micropubResponse.status === 201 || + micropubResponse.status === 202 + ) { + // Success - get the Location header for the new post URL + const location = micropubResponse.headers.get("Location"); + console.info( + `[Microsub] Created post via Micropub: ${location || "success"}`, + ); + + // Redirect back to reader with success message + return response.redirect(`${request.baseUrl}/channels`); + } + + // Handle error + const errorBody = await micropubResponse.text(); + const statusText = micropubResponse.statusText || "Unknown error"; + console.error( + `[Microsub] Micropub error: ${micropubResponse.status} ${errorBody}`, + ); + + // Parse error message from response body if JSON + let errorMessage = `Micropub error: ${statusText}`; + try { + const errorJson = JSON.parse(errorBody); + if (errorJson.error_description) { + errorMessage = String(errorJson.error_description); + } else if (errorJson.error) { + errorMessage = String(errorJson.error); + } + } catch { + // Not JSON, use status text + } + + return response.status(micropubResponse.status).render("error", { + title: "Error", + content: errorMessage, + }); + } catch (error) { + console.error(`[Microsub] Micropub request failed: ${error.message}`); + + return response.status(500).render("error", { + title: "Error", + content: `Failed to create post: ${error.message}`, + }); + } +} + +/** + * Search/discover feeds page + * @param {object} request - Express request + * @param {object} response - Express response + * @returns {Promise} + */ +export async function searchPage(request, response) { + const { application } = request.app.locals; + const userId = getUserId(request); + + const channelList = await getChannels(application, userId); + + response.render("search", { + title: request.__("microsub.search.title"), + channels: channelList, + baseUrl: request.baseUrl, + }); +} + +/** + * Search for feeds from URL + * @param {object} request - Express request + * @param {object} response - Express response + * @returns {Promise} + */ +export async function searchFeeds(request, response) { + const { application } = request.app.locals; + const userId = getUserId(request); + const { query } = request.body; + + const channelList = await getChannels(application, userId); + + let results = []; + if (query) { + try { + results = await discoverFeedsFromUrl(query); + } catch { + // Ignore discovery errors + } + } + + response.render("search", { + title: request.__("microsub.search.title"), + channels: channelList, + query, + results, + searched: true, + baseUrl: request.baseUrl, + }); +} + +/** + * Subscribe to a feed from search results + * @param {object} request - Express request + * @param {object} response - Express response + * @returns {Promise} + */ +export async function subscribe(request, response) { + const { application } = request.app.locals; + const userId = getUserId(request); + const { url, channel: channelUid } = request.body; + + const channelDocument = await getChannel(application, channelUid, userId); + if (!channelDocument) { + return response.status(404).render("404"); + } + + // Create feed subscription + const feed = await createFeed(application, { + channelId: channelDocument._id, + url, + title: undefined, + photo: undefined, + }); + + // Trigger immediate fetch in background + refreshFeedNow(application, feed._id).catch((error) => { + console.error(`[Microsub] Error fetching new feed ${url}:`, error.message); + }); + + response.redirect(`${request.baseUrl}/channels/${channelUid}/feeds`); +} + +export const readerController = { + index, + channels, + newChannel, + createChannel: createChannelAction, + channel, + settings, + updateSettings, + deleteChannel: deleteChannelAction, + feeds, + addFeed, + removeFeed, + item, + compose, + submitCompose, + searchPage, + searchFeeds, + subscribe, +}; diff --git a/lib/controllers/search.js b/lib/controllers/search.js new file mode 100644 index 0000000..79e7bc9 --- /dev/null +++ b/lib/controllers/search.js @@ -0,0 +1,143 @@ +/** + * Search controller + * @module controllers/search + */ + +import { IndiekitError } from "@indiekit/error"; + +import { discoverFeeds } from "../feeds/hfeed.js"; +import { searchWithFallback } from "../search/query.js"; +import { getChannel } from "../storage/channels.js"; +import { getUserId } from "../utils/auth.js"; +import { validateChannel, validateUrl } from "../utils/validation.js"; + +/** + * Discover feeds from a URL + * GET ?action=search&query= + * @param {object} request - Express request + * @param {object} response - Express response + */ +export async function discover(request, response) { + const { query } = request.query; + + if (!query) { + throw new IndiekitError("Missing required parameter: query", { + status: 400, + }); + } + + // Check if query is a URL + let url; + try { + url = new URL(query); + } catch { + // Not a URL, return empty results + return response.json({ results: [] }); + } + + try { + // Fetch the URL content + const fetchResponse = await fetch(url.href, { + headers: { + Accept: "text/html, application/xhtml+xml, */*", + "User-Agent": "Indiekit Microsub/1.0 (+https://getindiekit.com)", + }, + }); + + if (!fetchResponse.ok) { + throw new IndiekitError(`Failed to fetch URL: ${fetchResponse.status}`, { + status: 502, + }); + } + + const content = await fetchResponse.text(); + const feeds = await discoverFeeds(content, url.href); + + // Transform to Microsub search result format + const results = feeds.map((feed) => ({ + type: "feed", + url: feed.url, + })); + + response.json({ results }); + } catch (error) { + if (error instanceof IndiekitError) { + throw error; + } + throw new IndiekitError(`Feed discovery failed: ${error.message}`, { + status: 502, + }); + } +} + +/** + * Search feeds or items + * POST ?action=search + * @param {object} request - Express request + * @param {object} response - Express response + */ +export async function search(request, response) { + const { application } = request.app.locals; + const userId = getUserId(request); + const { query, channel } = request.body; + + if (!query) { + throw new IndiekitError("Missing required parameter: query", { + status: 400, + }); + } + + // If channel is provided, search within channel items + if (channel) { + validateChannel(channel); + + const channelDocument = await getChannel(application, channel, userId); + if (!channelDocument) { + throw new IndiekitError("Channel not found", { status: 404 }); + } + + const items = await searchWithFallback( + application, + channelDocument._id, + query, + ); + return response.json({ items }); + } + + // Check if query is a URL (feed discovery) + try { + validateUrl(query, "query"); + + // Use the discover function for URL queries + const fetchResponse = await fetch(query, { + headers: { + Accept: "text/html, application/xhtml+xml, */*", + "User-Agent": "Indiekit Microsub/1.0 (+https://getindiekit.com)", + }, + }); + + if (!fetchResponse.ok) { + throw new IndiekitError(`Failed to fetch URL: ${fetchResponse.status}`, { + status: 502, + }); + } + + const content = await fetchResponse.text(); + const feeds = await discoverFeeds(content, query); + + const results = feeds.map((feed) => ({ + type: "feed", + url: feed.url, + })); + + return response.json({ results }); + } catch (error) { + // Not a URL or fetch failed, return empty results + if (error instanceof IndiekitError) { + throw error; + } + return response.json({ results: [] }); + } +} + +export const searchController = { discover, search }; diff --git a/lib/controllers/timeline.js b/lib/controllers/timeline.js index 8419d9a..d22fa52 100644 --- a/lib/controllers/timeline.js +++ b/lib/controllers/timeline.js @@ -5,6 +5,7 @@ import { IndiekitError } from "@indiekit/error"; +import { proxyItemImages } from "../media/proxy.js"; import { getChannel } from "../storage/channels.js"; import { getTimelineItems, @@ -16,7 +17,7 @@ import { getUserId } from "../utils/auth.js"; import { validateChannel, validateEntries, - parseArrayParameter, + parseArrayParameter as parseArrayParametereter, } from "../utils/validation.js"; /** @@ -47,6 +48,14 @@ export async function get(request, response) { userId, }); + // Proxy images if application URL is available + const baseUrl = application.url; + if (baseUrl && timeline.items) { + timeline.items = timeline.items.map((item) => + proxyItemImages(item, baseUrl), + ); + } + response.json(timeline); } @@ -55,7 +64,6 @@ export async function get(request, response) { * POST ?action=timeline * @param {object} request - Express request * @param {object} response - Express response - * @returns {Promise} */ export async function action(request, response) { const { application } = request.app.locals; @@ -73,7 +81,7 @@ export async function action(request, response) { } // Get entry IDs from request - const entries = parseArrayParameter(request.body, "entry"); + const entries = parseArrayParametereter(request.body, "entry"); switch (method) { case "mark_read": { diff --git a/lib/feeds/atom.js b/lib/feeds/atom.js new file mode 100644 index 0000000..fcef777 --- /dev/null +++ b/lib/feeds/atom.js @@ -0,0 +1,61 @@ +/** + * Atom feed parser + * @module feeds/atom + */ + +import { Readable } from "node:stream"; + +import FeedParser from "feedparser"; + +import { normalizeItem, normalizeFeedMeta } from "./normalizer.js"; + +/** + * Parse Atom feed content + * @param {string} content - Atom XML content + * @param {string} feedUrl - URL of the feed + * @returns {Promise} Parsed feed with metadata and items + */ +export async function parseAtom(content, feedUrl) { + return new Promise((resolve, reject) => { + const feedparser = new FeedParser({ feedurl: feedUrl }); + const items = []; + let meta; + + feedparser.on("error", (error) => { + reject(new Error(`Atom parse error: ${error.message}`)); + }); + + feedparser.on("meta", (feedMeta) => { + meta = feedMeta; + }); + + feedparser.on("readable", function () { + let item; + while ((item = this.read())) { + items.push(item); + } + }); + + feedparser.on("end", () => { + try { + const normalizedMeta = normalizeFeedMeta(meta, feedUrl); + const normalizedItems = items.map((item) => + normalizeItem(item, feedUrl, "atom"), + ); + + resolve({ + type: "feed", + url: feedUrl, + ...normalizedMeta, + items: normalizedItems, + }); + } catch (error) { + reject(error); + } + }); + + // Create readable stream from string and pipe to feedparser + const stream = Readable.from([content]); + stream.pipe(feedparser); + }); +} diff --git a/lib/feeds/fetcher.js b/lib/feeds/fetcher.js new file mode 100644 index 0000000..9b14338 --- /dev/null +++ b/lib/feeds/fetcher.js @@ -0,0 +1,316 @@ +/** + * Feed fetcher with HTTP caching + * @module feeds/fetcher + */ + +import { getCache, setCache } from "../cache/redis.js"; + +const DEFAULT_TIMEOUT = 30_000; // 30 seconds +const DEFAULT_USER_AGENT = "Indiekit Microsub/1.0 (+https://getindiekit.com)"; + +/** + * Fetch feed content with caching + * @param {string} url - Feed URL + * @param {object} options - Fetch options + * @param {string} [options.etag] - Previous ETag for conditional request + * @param {string} [options.lastModified] - Previous Last-Modified for conditional request + * @param {number} [options.timeout] - Request timeout in ms + * @param {object} [options.redis] - Redis client for caching + * @returns {Promise} Fetch result with content and headers + */ +export async function fetchFeed(url, options = {}) { + const { etag, lastModified, timeout = DEFAULT_TIMEOUT, redis } = options; + + // Check cache first + if (redis) { + const cached = await getCache(redis, `feed:${url}`); + if (cached) { + return { + content: cached.content, + contentType: cached.contentType, + etag: cached.etag, + lastModified: cached.lastModified, + fromCache: true, + status: 200, + }; + } + } + + const headers = { + Accept: + "application/atom+xml, application/rss+xml, application/json, application/feed+json, text/xml, text/html;q=0.9, */*;q=0.8", + "User-Agent": DEFAULT_USER_AGENT, + }; + + // Add conditional request headers + if (etag) { + headers["If-None-Match"] = etag; + } + if (lastModified) { + headers["If-Modified-Since"] = lastModified; + } + + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), timeout); + + try { + const response = await fetch(url, { + headers, + signal: controller.signal, + redirect: "follow", + }); + + clearTimeout(timeoutId); + + // Not modified - use cached version + if (response.status === 304) { + return { + content: undefined, + contentType: undefined, + etag, + lastModified, + notModified: true, + status: 304, + }; + } + + if (!response.ok) { + throw new Error(`HTTP ${response.status}: ${response.statusText}`); + } + + const content = await response.text(); + const responseEtag = response.headers.get("ETag"); + const responseLastModified = response.headers.get("Last-Modified"); + const contentType = response.headers.get("Content-Type") || ""; + + const result = { + content, + contentType, + etag: responseEtag, + lastModified: responseLastModified, + fromCache: false, + status: response.status, + }; + + // Extract hub URL from Link header for WebSub + const linkHeader = response.headers.get("Link"); + if (linkHeader) { + result.hub = extractHubFromLinkHeader(linkHeader); + result.self = extractSelfFromLinkHeader(linkHeader); + } + + // Cache the result + if (redis) { + const cacheData = { + content, + contentType, + etag: responseEtag, + lastModified: responseLastModified, + }; + // Cache for 5 minutes by default + await setCache(redis, `feed:${url}`, cacheData, 300); + } + + return result; + } catch (error) { + clearTimeout(timeoutId); + + if (error.name === "AbortError") { + throw new Error(`Request timeout after ${timeout}ms`); + } + + throw error; + } +} + +/** + * Extract hub URL from Link header + * @param {string} linkHeader - Link header value + * @returns {string|undefined} Hub URL + */ +function extractHubFromLinkHeader(linkHeader) { + const hubMatch = linkHeader.match(/<([^>]+)>;\s*rel=["']?hub["']?/i); + return hubMatch ? hubMatch[1] : undefined; +} + +/** + * Extract self URL from Link header + * @param {string} linkHeader - Link header value + * @returns {string|undefined} Self URL + */ +function extractSelfFromLinkHeader(linkHeader) { + const selfMatch = linkHeader.match(/<([^>]+)>;\s*rel=["']?self["']?/i); + return selfMatch ? selfMatch[1] : undefined; +} + +/** + * Fetch feed and parse it + * @param {string} url - Feed URL + * @param {object} options - Options + * @returns {Promise} Parsed feed + */ +export async function fetchAndParseFeed(url, options = {}) { + const { parseFeed, detectFeedType } = await import("./parser.js"); + + const result = await fetchFeed(url, options); + + if (result.notModified) { + return { + ...result, + items: [], + }; + } + + // Check if we got a parseable feed + const feedType = detectFeedType(result.content, result.contentType); + + // If we got ActivityPub or unknown, try common feed paths + if (feedType === "activitypub" || feedType === "unknown") { + const fallbackFeed = await tryCommonFeedPaths(url, options); + if (fallbackFeed) { + // Fetch and parse the discovered feed + const feedResult = await fetchFeed(fallbackFeed.url, options); + if (!feedResult.notModified) { + const parsed = await parseFeed(feedResult.content, fallbackFeed.url, { + contentType: feedResult.contentType, + }); + return { + ...feedResult, + ...parsed, + hub: feedResult.hub || parsed._hub, + discoveredFrom: url, + }; + } + } + throw new Error( + `Unable to find a feed at ${url}. Try the direct feed URL.`, + ); + } + + const parsed = await parseFeed(result.content, url, { + contentType: result.contentType, + }); + + return { + ...result, + ...parsed, + hub: result.hub || parsed._hub, + }; +} + +/** + * Common feed paths to try when discovery fails + */ +const COMMON_FEED_PATHS = ["/feed/", "/feed", "/rss", "/rss.xml", "/atom.xml"]; + +/** + * Try to fetch a feed from common paths + * @param {string} baseUrl - Base URL of the site + * @param {object} options - Fetch options + * @returns {Promise} Feed result or undefined + */ +async function tryCommonFeedPaths(baseUrl, options = {}) { + const base = new URL(baseUrl); + + for (const feedPath of COMMON_FEED_PATHS) { + const feedUrl = new URL(feedPath, base).href; + try { + const result = await fetchFeed(feedUrl, { ...options, timeout: 10_000 }); + const contentType = result.contentType?.toLowerCase() || ""; + + // Check if we got a feed + if ( + contentType.includes("xml") || + contentType.includes("rss") || + contentType.includes("atom") || + (contentType.includes("json") && + result.content?.includes("jsonfeed.org")) + ) { + return { + url: feedUrl, + type: contentType.includes("json") ? "jsonfeed" : "xml", + rel: "alternate", + }; + } + } catch { + // Try next path + } + } + + return; +} + +/** + * Discover feeds from a URL + * @param {string} url - Page URL + * @param {object} options - Options + * @returns {Promise} Array of discovered feeds + */ +export async function discoverFeedsFromUrl(url, options = {}) { + const result = await fetchFeed(url, options); + const { discoverFeeds } = await import("./hfeed.js"); + + // If it's already a feed, return it + const contentType = result.contentType?.toLowerCase() || ""; + if ( + contentType.includes("xml") || + contentType.includes("rss") || + contentType.includes("atom") + ) { + return [ + { + url, + type: "xml", + rel: "self", + }, + ]; + } + + // Check for JSON Feed specifically + if ( + contentType.includes("json") && + result.content?.includes("jsonfeed.org") + ) { + return [ + { + url, + type: "jsonfeed", + rel: "self", + }, + ]; + } + + // Check if we got ActivityPub JSON or other non-feed JSON + // This happens with WordPress sites using ActivityPub plugin + if ( + contentType.includes("json") || + (result.content?.trim().startsWith("{") && + result.content?.includes("@context")) + ) { + // Try common feed paths as fallback + const fallbackFeed = await tryCommonFeedPaths(url, options); + if (fallbackFeed) { + return [fallbackFeed]; + } + } + + // If content looks like HTML, discover feeds from it + if ( + contentType.includes("html") || + result.content?.includes(" 0) { + return feeds; + } + } + + // Last resort: try common feed paths + const fallbackFeed = await tryCommonFeedPaths(url, options); + if (fallbackFeed) { + return [fallbackFeed]; + } + + return []; +} diff --git a/lib/feeds/hfeed.js b/lib/feeds/hfeed.js new file mode 100644 index 0000000..878cfdb --- /dev/null +++ b/lib/feeds/hfeed.js @@ -0,0 +1,177 @@ +/** + * h-feed (Microformats2) parser + * @module feeds/hfeed + */ + +import { mf2 } from "microformats-parser"; + +import { normalizeHfeedItem, normalizeHfeedMeta } from "./normalizer.js"; + +/** + * Parse h-feed content from HTML + * @param {string} content - HTML content with h-feed + * @param {string} feedUrl - URL of the page + * @returns {Promise} Parsed feed with metadata and items + */ +export async function parseHfeed(content, feedUrl) { + let parsed; + + try { + parsed = mf2(content, { baseUrl: feedUrl }); + } catch (error) { + throw new Error(`h-feed parse error: ${error.message}`); + } + + // Find h-feed in the parsed microformats + const hfeed = findHfeed(parsed); + + if (!hfeed) { + // If no h-feed, look for h-entry items at the root + const entries = parsed.items.filter( + (item) => item.type && item.type.includes("h-entry"), + ); + + if (entries.length === 0) { + throw new Error("No h-feed or h-entry found on page"); + } + + // Create synthetic feed from entries + return { + type: "feed", + url: feedUrl, + name: parsed.rels?.canonical?.[0] || feedUrl, + items: entries.map((entry) => normalizeHfeedItem(entry, feedUrl)), + }; + } + + const normalizedMeta = normalizeHfeedMeta(hfeed, feedUrl); + + // Get children entries from h-feed + const entries = hfeed.children || []; + const normalizedItems = entries + .filter((child) => child.type && child.type.includes("h-entry")) + .map((entry) => normalizeHfeedItem(entry, feedUrl)); + + return { + type: "feed", + url: feedUrl, + ...normalizedMeta, + items: normalizedItems, + }; +} + +/** + * Find h-feed in parsed microformats + * @param {object} parsed - Parsed microformats object + * @returns {object|undefined} h-feed object or undefined + */ +function findHfeed(parsed) { + // Look for h-feed at top level + for (const item of parsed.items) { + if (item.type && item.type.includes("h-feed")) { + return item; + } + + // Check nested children + if (item.children) { + for (const child of item.children) { + if (child.type && child.type.includes("h-feed")) { + return child; + } + } + } + } + + return; +} + +/** + * Discover feeds from HTML page + * @param {string} content - HTML content + * @param {string} pageUrl - URL of the page + * @returns {Promise} Array of discovered feed URLs with types + */ +export async function discoverFeeds(content, pageUrl) { + const feeds = []; + const parsed = mf2(content, { baseUrl: pageUrl }); + + // Check for rel="alternate" feed links + const alternates = parsed.rels?.alternate || []; + for (const url of alternates) { + // Try to determine feed type from URL + if (url.includes("feed") || url.endsWith(".xml") || url.endsWith(".json")) { + feeds.push({ + url, + type: "unknown", + rel: "alternate", + }); + } + } + + // Check for rel="feed" links (Microsub discovery) + const feedLinks = parsed.rels?.feed || []; + for (const url of feedLinks) { + feeds.push({ + url, + type: "hfeed", + rel: "feed", + }); + } + + // Check if page itself has h-feed + const hfeed = findHfeed(parsed); + if (hfeed) { + feeds.push({ + url: pageUrl, + type: "hfeed", + rel: "self", + }); + } + + // Parse elements for feed discovery + const linkFeeds = extractLinkFeeds(content, pageUrl); + feeds.push(...linkFeeds); + + return feeds; +} + +/** + * Extract feed links from HTML elements + * @param {string} content - HTML content + * @param {string} baseUrl - Base URL for resolving relative URLs + * @returns {Array} Array of discovered feeds + */ +function extractLinkFeeds(content, baseUrl) { + const feeds = []; + const linkRegex = /]+rel=["'](?:alternate|feed)["'][^>]*>/gi; + const matches = content.match(linkRegex) || []; + + for (const link of matches) { + const hrefMatch = link.match(/href=["']([^"']+)["']/i); + const typeMatch = link.match(/type=["']([^"']+)["']/i); + + if (hrefMatch) { + const href = hrefMatch[1]; + const type = typeMatch ? typeMatch[1] : "unknown"; + const url = new URL(href, baseUrl).href; + + let feedType = "unknown"; + if (type.includes("rss")) { + feedType = "rss"; + } else if (type.includes("atom")) { + feedType = "atom"; + } else if (type.includes("json")) { + feedType = "jsonfeed"; + } + + feeds.push({ + url, + type: feedType, + contentType: type, + rel: "link", + }); + } + } + + return feeds; +} diff --git a/lib/feeds/jsonfeed.js b/lib/feeds/jsonfeed.js new file mode 100644 index 0000000..61e7e25 --- /dev/null +++ b/lib/feeds/jsonfeed.js @@ -0,0 +1,43 @@ +/** + * JSON Feed parser + * @module feeds/jsonfeed + */ + +import { normalizeJsonFeedItem, normalizeJsonFeedMeta } from "./normalizer.js"; + +/** + * Parse JSON Feed content + * @param {string} content - JSON Feed content + * @param {string} feedUrl - URL of the feed + * @returns {Promise} Parsed feed with metadata and items + */ +export async function parseJsonFeed(content, feedUrl) { + let feed; + + try { + feed = typeof content === "string" ? JSON.parse(content) : content; + } catch (error) { + throw new Error(`JSON Feed parse error: ${error.message}`); + } + + // Validate JSON Feed structure + if (!feed.version || !feed.version.includes("jsonfeed.org")) { + throw new Error("Invalid JSON Feed: missing or invalid version"); + } + + if (!Array.isArray(feed.items)) { + throw new TypeError("Invalid JSON Feed: items must be an array"); + } + + const normalizedMeta = normalizeJsonFeedMeta(feed, feedUrl); + const normalizedItems = feed.items.map((item) => + normalizeJsonFeedItem(item, feedUrl), + ); + + return { + type: "feed", + url: feedUrl, + ...normalizedMeta, + items: normalizedItems, + }; +} diff --git a/lib/feeds/normalizer.js b/lib/feeds/normalizer.js new file mode 100644 index 0000000..596bf74 --- /dev/null +++ b/lib/feeds/normalizer.js @@ -0,0 +1,697 @@ +/** + * Feed normalizer - converts all feed formats to jf2 + * @module feeds/normalizer + */ + +import crypto from "node:crypto"; + +import sanitizeHtml from "sanitize-html"; + +/** + * Parse a date string with fallback for non-standard formats + * @param {string|Date} dateInput - Date string or Date object + * @returns {Date|undefined} Parsed Date or undefined if invalid + */ +function parseDate(dateInput) { + if (!dateInput) { + return; + } + + // Already a valid Date + if (dateInput instanceof Date && !Number.isNaN(dateInput.getTime())) { + return dateInput; + } + + const dateString = String(dateInput).trim(); + + // Try standard parsing first + let date = new Date(dateString); + if (!Number.isNaN(date.getTime())) { + return date; + } + + // Handle "YYYY-MM-DD HH:MM" format (missing seconds and timezone) + // e.g., "2026-01-28 08:40" + const shortDateTime = dateString.match( + /^(\d{4}-\d{2}-\d{2})\s+(\d{2}:\d{2})$/, + ); + if (shortDateTime) { + date = new Date(`${shortDateTime[1]}T${shortDateTime[2]}:00Z`); + if (!Number.isNaN(date.getTime())) { + return date; + } + } + + // Handle "YYYY-MM-DD HH:MM:SS" without timezone + const dateTimeNoTz = dateString.match( + /^(\d{4}-\d{2}-\d{2})\s+(\d{2}:\d{2}:\d{2})$/, + ); + if (dateTimeNoTz) { + date = new Date(`${dateTimeNoTz[1]}T${dateTimeNoTz[2]}Z`); + if (!Number.isNaN(date.getTime())) { + return date; + } + } + + // If all else fails, return undefined + return; +} + +/** + * Safely convert date to ISO string + * @param {string|Date} dateInput - Date input + * @returns {string|undefined} ISO string or undefined + */ +function toISOStringSafe(dateInput) { + const date = parseDate(dateInput); + return date ? date.toISOString() : undefined; +} + +/** + * Sanitize HTML options + */ +const SANITIZE_OPTIONS = { + allowedTags: [ + "a", + "abbr", + "b", + "blockquote", + "br", + "code", + "em", + "figcaption", + "figure", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "img", + "li", + "ol", + "p", + "pre", + "s", + "span", + "strike", + "strong", + "sub", + "sup", + "table", + "tbody", + "td", + "th", + "thead", + "tr", + "u", + "ul", + "video", + "audio", + "source", + ], + allowedAttributes: { + a: ["href", "title", "rel"], + img: ["src", "alt", "title", "width", "height"], + video: ["src", "poster", "controls", "width", "height"], + audio: ["src", "controls"], + source: ["src", "type"], + "*": ["class"], + }, + allowedSchemes: ["http", "https", "mailto"], +}; + +/** + * Generate unique ID for an item + * @param {string} feedUrl - Feed URL + * @param {string} itemId - Item identifier (URL or ID) + * @returns {string} Unique ID hash + */ +export function generateItemUid(feedUrl, itemId) { + const hash = crypto.createHash("sha256"); + hash.update(`${feedUrl}::${itemId}`); + return hash.digest("hex").slice(0, 24); +} + +/** + * Normalize RSS/Atom item from feedparser + * @param {object} item - Feedparser item + * @param {string} feedUrl - Feed URL + * @param {string} feedType - 'rss' or 'atom' + * @returns {object} Normalized jf2 item + */ +export function normalizeItem(item, feedUrl, feedType) { + const url = item.link || item.origlink || item.guid; + const uid = generateItemUid(feedUrl, item.guid || url || item.title); + + const normalized = { + type: "entry", + uid, + url, + name: item.title || undefined, + published: toISOStringSafe(item.pubdate), + updated: toISOStringSafe(item.date), + _source: { + url: feedUrl, + feedUrl, + feedType, + originalId: item.guid, + }, + }; + + // Content + if (item.description || item.summary) { + const html = item.description || item.summary; + normalized.content = { + html: sanitizeHtml(html, SANITIZE_OPTIONS), + text: sanitizeHtml(html, { allowedTags: [] }).trim(), + }; + } + + // Summary (prefer explicit summary over truncated content) + if (item.summary && item.description && item.summary !== item.description) { + normalized.summary = sanitizeHtml(item.summary, { allowedTags: [] }).trim(); + } + + // Author + if (item.author || item["dc:creator"]) { + const authorName = item.author || item["dc:creator"]; + normalized.author = { + type: "card", + name: authorName, + }; + } + + // Categories/tags + if (item.categories && item.categories.length > 0) { + normalized.category = item.categories; + } + + // Enclosures (media) + if (item.enclosures && item.enclosures.length > 0) { + for (const enclosure of item.enclosures) { + const mediaUrl = enclosure.url; + const mediaType = enclosure.type || ""; + + if (mediaType.startsWith("image/")) { + normalized.photo = normalized.photo || []; + normalized.photo.push(mediaUrl); + } else if (mediaType.startsWith("video/")) { + normalized.video = normalized.video || []; + normalized.video.push(mediaUrl); + } else if (mediaType.startsWith("audio/")) { + normalized.audio = normalized.audio || []; + normalized.audio.push(mediaUrl); + } + } + } + + // Featured image from media content + if (item["media:content"] && item["media:content"].url) { + const mediaType = item["media:content"].type || ""; + if ( + mediaType.startsWith("image/") || + item["media:content"].medium === "image" + ) { + normalized.photo = normalized.photo || []; + if (!normalized.photo.includes(item["media:content"].url)) { + normalized.photo.push(item["media:content"].url); + } + } + } + + // Image from item.image + if (item.image && item.image.url) { + normalized.photo = normalized.photo || []; + if (!normalized.photo.includes(item.image.url)) { + normalized.photo.push(item.image.url); + } + } + + return normalized; +} + +/** + * Normalize feed metadata from feedparser + * @param {object} meta - Feedparser meta object + * @param {string} feedUrl - Feed URL + * @returns {object} Normalized feed metadata + */ +export function normalizeFeedMeta(meta, feedUrl) { + const normalized = { + name: meta.title || feedUrl, + }; + + if (meta.description) { + normalized.summary = meta.description; + } + + if (meta.link) { + normalized.url = meta.link; + } + + if (meta.image && meta.image.url) { + normalized.photo = meta.image.url; + } + + if (meta.favicon) { + normalized.photo = normalized.photo || meta.favicon; + } + + // Author/publisher + if (meta.author) { + normalized.author = { + type: "card", + name: meta.author, + }; + } + + // Hub for WebSub + if (meta.cloud && meta.cloud.href) { + normalized._hub = meta.cloud.href; + } + + // Look for hub in links + if (meta.link && meta["atom:link"]) { + const links = Array.isArray(meta["atom:link"]) + ? meta["atom:link"] + : [meta["atom:link"]]; + for (const link of links) { + if (link["@"] && link["@"].rel === "hub") { + normalized._hub = link["@"].href; + break; + } + } + } + + return normalized; +} + +/** + * Normalize JSON Feed item + * @param {object} item - JSON Feed item + * @param {string} feedUrl - Feed URL + * @returns {object} Normalized jf2 item + */ +export function normalizeJsonFeedItem(item, feedUrl) { + const url = item.url || item.external_url; + const uid = generateItemUid(feedUrl, item.id || url); + + const normalized = { + type: "entry", + uid, + url, + name: item.title || undefined, + published: item.date_published + ? new Date(item.date_published).toISOString() + : undefined, + updated: item.date_modified + ? new Date(item.date_modified).toISOString() + : undefined, + _source: { + url: feedUrl, + feedUrl, + feedType: "jsonfeed", + originalId: item.id, + }, + }; + + // Content + if (item.content_html || item.content_text) { + normalized.content = {}; + if (item.content_html) { + normalized.content.html = sanitizeHtml( + item.content_html, + SANITIZE_OPTIONS, + ); + normalized.content.text = sanitizeHtml(item.content_html, { + allowedTags: [], + }).trim(); + } else if (item.content_text) { + normalized.content.text = item.content_text; + } + } + + // Summary + if (item.summary) { + normalized.summary = item.summary; + } + + // Author + if (item.author || item.authors) { + const author = item.author || (item.authors && item.authors[0]); + if (author) { + normalized.author = { + type: "card", + name: author.name, + url: author.url, + photo: author.avatar, + }; + } + } + + // Tags + if (item.tags && item.tags.length > 0) { + normalized.category = item.tags; + } + + // Featured image + if (item.image) { + normalized.photo = [item.image]; + } + + if (item.banner_image && !normalized.photo) { + normalized.photo = [item.banner_image]; + } + + // Attachments + if (item.attachments && item.attachments.length > 0) { + for (const attachment of item.attachments) { + const mediaType = attachment.mime_type || ""; + + if (mediaType.startsWith("image/")) { + normalized.photo = normalized.photo || []; + normalized.photo.push(attachment.url); + } else if (mediaType.startsWith("video/")) { + normalized.video = normalized.video || []; + normalized.video.push(attachment.url); + } else if (mediaType.startsWith("audio/")) { + normalized.audio = normalized.audio || []; + normalized.audio.push(attachment.url); + } + } + } + + // External URL + if (item.external_url && item.url !== item.external_url) { + normalized["bookmark-of"] = [item.external_url]; + } + + return normalized; +} + +/** + * Normalize JSON Feed metadata + * @param {object} feed - JSON Feed object + * @param {string} feedUrl - Feed URL + * @returns {object} Normalized feed metadata + */ +export function normalizeJsonFeedMeta(feed, feedUrl) { + const normalized = { + name: feed.title || feedUrl, + }; + + if (feed.description) { + normalized.summary = feed.description; + } + + if (feed.home_page_url) { + normalized.url = feed.home_page_url; + } + + if (feed.icon) { + normalized.photo = feed.icon; + } else if (feed.favicon) { + normalized.photo = feed.favicon; + } + + if (feed.author || feed.authors) { + const author = feed.author || (feed.authors && feed.authors[0]); + if (author) { + normalized.author = { + type: "card", + name: author.name, + url: author.url, + photo: author.avatar, + }; + } + } + + // Hub for WebSub + if (feed.hubs && feed.hubs.length > 0) { + normalized._hub = feed.hubs[0].url; + } + + return normalized; +} + +/** + * Normalize h-feed entry + * @param {object} entry - Microformats h-entry + * @param {string} feedUrl - Feed URL + * @returns {object} Normalized jf2 item + */ +export function normalizeHfeedItem(entry, feedUrl) { + const properties = entry.properties || {}; + const url = getFirst(properties.url) || getFirst(properties.uid); + const uid = generateItemUid(feedUrl, getFirst(properties.uid) || url); + + const normalized = { + type: "entry", + uid, + url, + _source: { + url: feedUrl, + feedUrl, + feedType: "hfeed", + originalId: getFirst(properties.uid), + }, + }; + + // Name/title + if (properties.name) { + const name = getFirst(properties.name); + // Only include name if it's not just the content + if ( + name && + (!properties.content || name !== getContentText(properties.content)) + ) { + normalized.name = name; + } + } + + // Published + if (properties.published) { + const published = getFirst(properties.published); + normalized.published = new Date(published).toISOString(); + } + + // Updated + if (properties.updated) { + const updated = getFirst(properties.updated); + normalized.updated = new Date(updated).toISOString(); + } + + // Content + if (properties.content) { + const content = getFirst(properties.content); + if (typeof content === "object") { + normalized.content = { + html: content.html + ? sanitizeHtml(content.html, SANITIZE_OPTIONS) + : undefined, + text: content.value || undefined, + }; + } else if (typeof content === "string") { + normalized.content = { text: content }; + } + } + + // Summary + if (properties.summary) { + normalized.summary = getFirst(properties.summary); + } + + // Author + if (properties.author) { + const author = getFirst(properties.author); + normalized.author = normalizeHcard(author); + } + + // Categories + if (properties.category) { + normalized.category = properties.category; + } + + // Photos + if (properties.photo) { + normalized.photo = properties.photo.map((p) => + typeof p === "object" ? p.value || p.url : p, + ); + } + + // Videos + if (properties.video) { + normalized.video = properties.video.map((v) => + typeof v === "object" ? v.value || v.url : v, + ); + } + + // Audio + if (properties.audio) { + normalized.audio = properties.audio.map((a) => + typeof a === "object" ? a.value || a.url : a, + ); + } + + // Interaction types - normalize to string URLs + if (properties["like-of"]) { + normalized["like-of"] = normalizeUrlArray(properties["like-of"]); + } + if (properties["repost-of"]) { + normalized["repost-of"] = normalizeUrlArray(properties["repost-of"]); + } + if (properties["bookmark-of"]) { + normalized["bookmark-of"] = normalizeUrlArray(properties["bookmark-of"]); + } + if (properties["in-reply-to"]) { + normalized["in-reply-to"] = normalizeUrlArray(properties["in-reply-to"]); + } + + // RSVP + if (properties.rsvp) { + normalized.rsvp = getFirst(properties.rsvp); + } + + // Syndication + if (properties.syndication) { + normalized.syndication = properties.syndication; + } + + return normalized; +} + +/** + * Normalize h-feed metadata + * @param {object} hfeed - h-feed microformat object + * @param {string} feedUrl - Feed URL + * @returns {object} Normalized feed metadata + */ +export function normalizeHfeedMeta(hfeed, feedUrl) { + const properties = hfeed.properties || {}; + + const normalized = { + name: getFirst(properties.name) || feedUrl, + }; + + if (properties.summary) { + normalized.summary = getFirst(properties.summary); + } + + if (properties.url) { + normalized.url = getFirst(properties.url); + } + + if (properties.photo) { + normalized.photo = getFirst(properties.photo); + if (typeof normalized.photo === "object") { + normalized.photo = normalized.photo.value || normalized.photo.url; + } + } + + if (properties.author) { + const author = getFirst(properties.author); + normalized.author = normalizeHcard(author); + } + + return normalized; +} + +/** + * Extract URL string from a photo value + * @param {object|string} photo - Photo value (can be string URL or object with value/url) + * @returns {string|undefined} Photo URL string + */ +function extractPhotoUrl(photo) { + if (!photo) { + return; + } + if (typeof photo === "string") { + return photo; + } + if (typeof photo === "object") { + return photo.value || photo.url || photo.src; + } + return; +} + +/** + * Extract URL string from a value that may be string or object + * @param {object|string} value - URL string or object with url/value property + * @returns {string|undefined} URL string + */ +function extractUrl(value) { + if (!value) { + return; + } + if (typeof value === "string") { + return value; + } + if (typeof value === "object") { + return value.value || value.url || value.href; + } + return; +} + +/** + * Normalize an array of URLs that may contain strings or objects + * @param {Array} urls - Array of URL strings or objects + * @returns {Array} Array of URL strings + */ +function normalizeUrlArray(urls) { + if (!urls || !Array.isArray(urls)) { + return []; + } + return urls.map((u) => extractUrl(u)).filter(Boolean); +} + +/** + * Normalize h-card author + * @param {object|string} hcard - h-card or author name string + * @returns {object} Normalized author object + */ +function normalizeHcard(hcard) { + if (typeof hcard === "string") { + return { type: "card", name: hcard }; + } + + if (!hcard || !hcard.properties) { + return; + } + + const properties = hcard.properties; + + return { + type: "card", + name: getFirst(properties.name), + url: getFirst(properties.url), + photo: extractPhotoUrl(getFirst(properties.photo)), + }; +} + +/** + * Get first item from array or return the value itself + * @param {Array|*} value - Value or array of values + * @returns {*} First value or the value itself + */ +function getFirst(value) { + if (Array.isArray(value)) { + return value[0]; + } + return value; +} + +/** + * Get text content from content property + * @param {Array} content - Content property array + * @returns {string} Text content + */ +function getContentText(content) { + const first = getFirst(content); + if (typeof first === "object") { + return first.value || first.text || ""; + } + return first || ""; +} diff --git a/lib/feeds/parser.js b/lib/feeds/parser.js new file mode 100644 index 0000000..0ac62da --- /dev/null +++ b/lib/feeds/parser.js @@ -0,0 +1,135 @@ +/** + * Feed parser dispatcher + * @module feeds/parser + */ + +import { parseAtom } from "./atom.js"; +import { parseHfeed } from "./hfeed.js"; +import { parseJsonFeed } from "./jsonfeed.js"; +import { parseRss } from "./rss.js"; + +/** + * Detect feed type from content + * @param {string} content - Feed content + * @param {string} contentType - HTTP Content-Type header + * @returns {string} Feed type: 'rss' | 'atom' | 'jsonfeed' | 'hfeed' | 'unknown' + */ +export function detectFeedType(content, contentType = "") { + const ct = contentType.toLowerCase(); + + // Check Content-Type header first + if (ct.includes("application/json") || ct.includes("application/feed+json")) { + return "jsonfeed"; + } + + if (ct.includes("application/atom+xml")) { + return "atom"; + } + + if ( + ct.includes("application/rss+xml") || + ct.includes("application/xml") || + ct.includes("text/xml") + ) { + // Need to check content to distinguish RSS from Atom + const trimmed = content.trim(); + if ( + trimmed.includes("} Parsed feed with metadata and items + */ +export async function parseFeed(content, feedUrl, options = {}) { + const feedType = detectFeedType(content, options.contentType); + + switch (feedType) { + case "rss": { + return parseRss(content, feedUrl); + } + + case "atom": { + return parseAtom(content, feedUrl); + } + + case "jsonfeed": { + return parseJsonFeed(content, feedUrl); + } + + case "hfeed": { + return parseHfeed(content, feedUrl); + } + + case "activitypub": { + throw new Error( + `URL returns ActivityPub JSON instead of a feed. Try the direct feed URL (e.g., ${feedUrl}feed/)`, + ); + } + + default: { + throw new Error(`Unable to detect feed type for ${feedUrl}`); + } + } +} + +export { parseAtom } from "./atom.js"; +export { parseHfeed } from "./hfeed.js"; +export { parseJsonFeed } from "./jsonfeed.js"; +export { parseRss } from "./rss.js"; diff --git a/lib/feeds/rss.js b/lib/feeds/rss.js new file mode 100644 index 0000000..fd112d5 --- /dev/null +++ b/lib/feeds/rss.js @@ -0,0 +1,61 @@ +/** + * RSS 1.0/2.0 feed parser + * @module feeds/rss + */ + +import { Readable } from "node:stream"; + +import FeedParser from "feedparser"; + +import { normalizeItem, normalizeFeedMeta } from "./normalizer.js"; + +/** + * Parse RSS feed content + * @param {string} content - RSS XML content + * @param {string} feedUrl - URL of the feed + * @returns {Promise} Parsed feed with metadata and items + */ +export async function parseRss(content, feedUrl) { + return new Promise((resolve, reject) => { + const feedparser = new FeedParser({ feedurl: feedUrl }); + const items = []; + let meta; + + feedparser.on("error", (error) => { + reject(new Error(`RSS parse error: ${error.message}`)); + }); + + feedparser.on("meta", (feedMeta) => { + meta = feedMeta; + }); + + feedparser.on("readable", function () { + let item; + while ((item = this.read())) { + items.push(item); + } + }); + + feedparser.on("end", () => { + try { + const normalizedMeta = normalizeFeedMeta(meta, feedUrl); + const normalizedItems = items.map((item) => + normalizeItem(item, feedUrl, "rss"), + ); + + resolve({ + type: "feed", + url: feedUrl, + ...normalizedMeta, + items: normalizedItems, + }); + } catch (error) { + reject(error); + } + }); + + // Create readable stream from string and pipe to feedparser + const stream = Readable.from([content]); + stream.pipe(feedparser); + }); +} diff --git a/lib/media/proxy.js b/lib/media/proxy.js new file mode 100644 index 0000000..3814fd9 --- /dev/null +++ b/lib/media/proxy.js @@ -0,0 +1,219 @@ +/** + * Media proxy with caching + * @module media/proxy + */ + +import crypto from "node:crypto"; + +import { getCache, setCache } from "../cache/redis.js"; + +const MAX_SIZE = 2 * 1024 * 1024; // 2MB max image size +const CACHE_TTL = 4 * 60 * 60; // 4 hours +const ALLOWED_TYPES = new Set([ + "image/jpeg", + "image/png", + "image/gif", + "image/webp", + "image/svg+xml", +]); + +/** + * Generate a hash for a URL to use as cache key + * @param {string} url - Original image URL + * @returns {string} URL-safe hash + */ +export function hashUrl(url) { + return crypto.createHash("sha256").update(url).digest("hex").slice(0, 16); +} + +/** + * Get the proxied URL for an image + * @param {string} baseUrl - Base URL of the Microsub endpoint + * @param {string} originalUrl - Original image URL + * @returns {string} Proxied URL + */ +export function getProxiedUrl(baseUrl, originalUrl) { + if (!originalUrl || !baseUrl) { + return originalUrl; + } + + // Skip data URLs + if (originalUrl.startsWith("data:")) { + return originalUrl; + } + + // Skip already-proxied URLs + if (originalUrl.includes("/microsub/media/")) { + return originalUrl; + } + + const hash = hashUrl(originalUrl); + return `${baseUrl}/microsub/media/${hash}?url=${encodeURIComponent(originalUrl)}`; +} + +/** + * Rewrite image URLs in an item to use the proxy + * @param {object} item - JF2 item + * @param {string} baseUrl - Base URL for proxy + * @returns {object} Item with proxied URLs + */ +export function proxyItemImages(item, baseUrl) { + if (!baseUrl || !item) { + return item; + } + + const proxied = { ...item }; + + // Proxy photo URLs + if (proxied.photo) { + if (Array.isArray(proxied.photo)) { + proxied.photo = proxied.photo.map((p) => { + if (typeof p === "string") { + return getProxiedUrl(baseUrl, p); + } + if (p?.value) { + return { ...p, value: getProxiedUrl(baseUrl, p.value) }; + } + return p; + }); + } else if (typeof proxied.photo === "string") { + proxied.photo = getProxiedUrl(baseUrl, proxied.photo); + } + } + + // Proxy author photo + if (proxied.author?.photo) { + proxied.author = { + ...proxied.author, + photo: getProxiedUrl(baseUrl, proxied.author.photo), + }; + } + + return proxied; +} + +/** + * Fetch and cache an image + * @param {object} redis - Redis client + * @param {string} url - Image URL to fetch + * @returns {Promise} Cached image data or null + */ +export async function fetchImage(redis, url) { + const cacheKey = `media:${hashUrl(url)}`; + + // Try cache first + if (redis) { + const cached = await getCache(redis, cacheKey); + if (cached) { + return cached; + } + } + + try { + // Fetch the image + const response = await fetch(url, { + headers: { + "User-Agent": "Indiekit Microsub/1.0 (+https://getindiekit.com)", + Accept: "image/*", + }, + signal: AbortSignal.timeout(10_000), // 10 second timeout + }); + + if (!response.ok) { + console.error( + `[Microsub] Media proxy fetch failed: ${response.status} for ${url}`, + ); + return; + } + + // Check content type + const contentType = response.headers.get("content-type")?.split(";")[0]; + if (!ALLOWED_TYPES.has(contentType)) { + console.error( + `[Microsub] Media proxy rejected type: ${contentType} for ${url}`, + ); + return; + } + + // Check content length + const contentLength = Number.parseInt( + response.headers.get("content-length") || "0", + 10, + ); + if (contentLength > MAX_SIZE) { + console.error( + `[Microsub] Media proxy rejected size: ${contentLength} for ${url}`, + ); + return; + } + + // Read the body + const buffer = await response.arrayBuffer(); + if (buffer.byteLength > MAX_SIZE) { + return; + } + + const imageData = { + contentType, + data: Buffer.from(buffer).toString("base64"), + size: buffer.byteLength, + }; + + // Cache in Redis + if (redis) { + await setCache(redis, cacheKey, imageData, CACHE_TTL); + } + + return imageData; + } catch (error) { + console.error(`[Microsub] Media proxy error: ${error.message} for ${url}`); + return; + } +} + +/** + * Express route handler for media proxy + * @param {object} request - Express request + * @param {object} response - Express response + * @returns {Promise} + */ +export async function handleMediaProxy(request, response) { + const { url } = request.query; + + if (!url) { + return response.status(400).send("Missing url parameter"); + } + + // Validate URL + try { + const parsed = new URL(url); + if (!["http:", "https:"].includes(parsed.protocol)) { + return response.status(400).send("Invalid URL protocol"); + } + } catch { + return response.status(400).send("Invalid URL"); + } + + // Get Redis client from application + const { application } = request.app.locals; + const redis = application.redis; + + // Fetch or get from cache + const imageData = await fetchImage(redis, url); + + if (!imageData) { + // Redirect to original URL as fallback + return response.redirect(url); + } + + // Set cache headers + response.set({ + "Content-Type": imageData.contentType, + "Content-Length": imageData.size, + "Cache-Control": "public, max-age=14400", // 4 hours + "X-Proxied-From": url, + }); + + // Send the image + response.send(Buffer.from(imageData.data, "base64")); +} diff --git a/lib/polling/processor.js b/lib/polling/processor.js new file mode 100644 index 0000000..ade25de --- /dev/null +++ b/lib/polling/processor.js @@ -0,0 +1,234 @@ +/** + * Feed processing pipeline + * @module polling/processor + */ + +import { getRedisClient, publishEvent } from "../cache/redis.js"; +import { fetchAndParseFeed } from "../feeds/fetcher.js"; +import { getChannel } from "../storage/channels.js"; +import { updateFeedAfterFetch, updateFeedWebsub } from "../storage/feeds.js"; +import { passesRegexFilter, passesTypeFilter } from "../storage/filters.js"; +import { addItem } from "../storage/items.js"; +import { + subscribe as websubSubscribe, + getCallbackUrl, +} from "../websub/subscriber.js"; + +import { calculateNewTier } from "./tier.js"; + +/** + * Process a single feed + * @param {object} application - Indiekit application + * @param {object} feed - Feed document from database + * @returns {Promise} Processing result + */ +export async function processFeed(application, feed) { + const startTime = Date.now(); + const result = { + feedId: feed._id, + url: feed.url, + success: false, + itemsAdded: 0, + error: undefined, + }; + + try { + // Get Redis client for caching + const redis = getRedisClient(application); + + // Fetch and parse the feed + const parsed = await fetchAndParseFeed(feed.url, { + etag: feed.etag, + lastModified: feed.lastModified, + redis, + }); + + // Handle 304 Not Modified + if (parsed.notModified) { + const tierResult = calculateNewTier({ + currentTier: feed.tier, + hasNewItems: false, + consecutiveUnchanged: feed.unmodified || 0, + }); + + await updateFeedAfterFetch(application, feed._id, false, { + tier: tierResult.tier, + unmodified: tierResult.consecutiveUnchanged, + nextFetchAt: tierResult.nextFetchAt, + }); + + result.success = true; + result.notModified = true; + return result; + } + + // Get channel for filtering + const channel = await getChannel(application, feed.channelId); + + // Process items + let newItemCount = 0; + for (const item of parsed.items) { + // Apply channel filters + if (channel?.settings && !passesFilters(item, channel.settings)) { + continue; + } + + // Enrich item source with feed metadata + if (item._source) { + item._source.name = feed.title || parsed.name; + } + + // Store the item + const stored = await addItem(application, { + channelId: feed.channelId, + feedId: feed._id, + uid: item.uid, + item, + }); + if (stored) { + newItemCount++; + + // Publish real-time event + if (redis) { + await publishEvent(redis, `microsub:${feed.channelId}`, { + type: "new-item", + channelId: feed.channelId.toString(), + item: stored, + }); + } + } + } + + result.itemsAdded = newItemCount; + + // Update tier based on whether we found new items + const tierResult = calculateNewTier({ + currentTier: feed.tier, + hasNewItems: newItemCount > 0, + consecutiveUnchanged: newItemCount > 0 ? 0 : feed.unmodified || 0, + }); + + // Update feed metadata + const updateData = { + tier: tierResult.tier, + unmodified: tierResult.consecutiveUnchanged, + nextFetchAt: tierResult.nextFetchAt, + etag: parsed.etag, + lastModified: parsed.lastModified, + }; + + // Update feed title/photo if discovered + if (parsed.name && !feed.title) { + updateData.title = parsed.name; + } + if (parsed.photo && !feed.photo) { + updateData.photo = parsed.photo; + } + + await updateFeedAfterFetch( + application, + feed._id, + newItemCount > 0, + updateData, + ); + + // Handle WebSub hub discovery and auto-subscription + if (parsed.hub && (!feed.websub || feed.websub.hub !== parsed.hub)) { + await updateFeedWebsub(application, feed._id, { + hub: parsed.hub, + topic: parsed.self || feed.url, + }); + + // Auto-subscribe to WebSub hub if we have a callback URL + const baseUrl = application.url; + if (baseUrl) { + const callbackUrl = getCallbackUrl(baseUrl, feed._id.toString()); + const updatedFeed = { + ...feed, + websub: { hub: parsed.hub, topic: parsed.self || feed.url }, + }; + + websubSubscribe(application, updatedFeed, callbackUrl) + .then((subscribed) => { + if (subscribed) { + console.info( + `[Microsub] WebSub subscription initiated for ${feed.url}`, + ); + } + }) + .catch((error) => { + console.error( + `[Microsub] WebSub subscription error for ${feed.url}:`, + error.message, + ); + }); + } + } + + result.success = true; + result.tier = tierResult.tier; + } catch (error) { + result.error = error.message; + + // Still update the feed to prevent retry storms + try { + const tierResult = calculateNewTier({ + currentTier: feed.tier, + hasNewItems: false, + consecutiveUnchanged: (feed.unmodified || 0) + 1, + }); + + await updateFeedAfterFetch(application, feed._id, false, { + tier: Math.min(tierResult.tier + 1, 10), // Increase tier on error + unmodified: tierResult.consecutiveUnchanged, + nextFetchAt: tierResult.nextFetchAt, + lastError: error.message, + lastErrorAt: new Date(), + }); + } catch { + // Ignore update errors + } + } + + result.duration = Date.now() - startTime; + return result; +} + +/** + * Check if an item passes channel filters + * @param {object} item - Feed item + * @param {object} settings - Channel settings + * @returns {boolean} Whether the item passes filters + */ +function passesFilters(item, settings) { + return passesTypeFilter(item, settings) && passesRegexFilter(item, settings); +} + +/** + * Process multiple feeds in batch + * @param {object} application - Indiekit application + * @param {Array} feeds - Array of feed documents + * @param {object} options - Processing options + * @returns {Promise} Batch processing result + */ +export async function processFeedBatch(application, feeds, options = {}) { + const { concurrency = 5 } = options; + const results = []; + + // Process in batches with limited concurrency + for (let index = 0; index < feeds.length; index += concurrency) { + const batch = feeds.slice(index, index + concurrency); + const batchResults = await Promise.all( + batch.map((feed) => processFeed(application, feed)), + ); + results.push(...batchResults); + } + + return { + total: feeds.length, + successful: results.filter((r) => r.success).length, + failed: results.filter((r) => !r.success).length, + itemsAdded: results.reduce((sum, r) => sum + r.itemsAdded, 0), + results, + }; +} diff --git a/lib/polling/scheduler.js b/lib/polling/scheduler.js new file mode 100644 index 0000000..bd494ad --- /dev/null +++ b/lib/polling/scheduler.js @@ -0,0 +1,128 @@ +/** + * Feed polling scheduler + * @module polling/scheduler + */ + +import { getFeedsToFetch } from "../storage/feeds.js"; + +import { processFeedBatch } from "./processor.js"; + +let schedulerInterval; +let indiekitInstance; +let isRunning = false; + +const POLL_INTERVAL = 60 * 1000; // Run scheduler every minute +const BATCH_CONCURRENCY = 5; // Process 5 feeds at a time + +/** + * Start the feed polling scheduler + * @param {object} indiekit - Indiekit instance + */ +export function startScheduler(indiekit) { + if (schedulerInterval) { + return; // Already running + } + + indiekitInstance = indiekit; + + // Run every minute + schedulerInterval = setInterval(async () => { + await runSchedulerCycle(); + }, POLL_INTERVAL); + + // Run immediately on start + runSchedulerCycle(); + + console.log("[Microsub] Feed polling scheduler started"); +} + +/** + * Stop the feed polling scheduler + */ +export function stopScheduler() { + if (schedulerInterval) { + clearInterval(schedulerInterval); + schedulerInterval = undefined; + } + indiekitInstance = undefined; + console.log("[Microsub] Feed polling scheduler stopped"); +} + +/** + * Run a single scheduler cycle + */ +async function runSchedulerCycle() { + if (!indiekitInstance) { + return; + } + + // Prevent overlapping runs + if (isRunning) { + return; + } + + isRunning = true; + + try { + const application = indiekitInstance; + const feeds = await getFeedsToFetch(application); + + if (feeds.length === 0) { + isRunning = false; + return; + } + + console.log(`[Microsub] Processing ${feeds.length} feeds due for refresh`); + + const result = await processFeedBatch(application, feeds, { + concurrency: BATCH_CONCURRENCY, + }); + + console.log( + `[Microsub] Processed ${result.total} feeds: ${result.successful} successful, ` + + `${result.failed} failed, ${result.itemsAdded} new items`, + ); + + // Log any errors + for (const feedResult of result.results) { + if (feedResult.error) { + console.error( + `[Microsub] Error processing ${feedResult.url}: ${feedResult.error}`, + ); + } + } + } catch (error) { + console.error("[Microsub] Error in scheduler cycle:", error.message); + } finally { + isRunning = false; + } +} + +/** + * Manually trigger a feed refresh + * @param {object} application - Indiekit application + * @param {string} feedId - Feed ID to refresh + * @returns {Promise} Processing result + */ +export async function refreshFeedNow(application, feedId) { + const { getFeedById } = await import("../storage/feeds.js"); + const { processFeed } = await import("./processor.js"); + + const feed = await getFeedById(application, feedId); + if (!feed) { + throw new Error("Feed not found"); + } + + return processFeed(application, feed); +} + +/** + * Get scheduler status + * @returns {object} Scheduler status + */ +export function getSchedulerStatus() { + return { + running: !!schedulerInterval, + processing: isRunning, + }; +} diff --git a/lib/polling/tier.js b/lib/polling/tier.js new file mode 100644 index 0000000..8191c10 --- /dev/null +++ b/lib/polling/tier.js @@ -0,0 +1,139 @@ +/** + * Adaptive tier-based polling algorithm + * Based on Ekster's approach: https://github.com/pstuifzand/ekster + * + * Tier determines poll interval: interval = 2^tier minutes + * - Tier 0: Every minute (active/new feeds) + * - Tier 1: Every 2 minutes + * - Tier 2: Every 4 minutes + * - Tier 3: Every 8 minutes + * - Tier 4: Every 16 minutes + * - Tier 5: Every 32 minutes + * - Tier 6: Every 64 minutes (~1 hour) + * - Tier 7: Every 128 minutes (~2 hours) + * - Tier 8: Every 256 minutes (~4 hours) + * - Tier 9: Every 512 minutes (~8 hours) + * - Tier 10: Every 1024 minutes (~17 hours) + * + * @module polling/tier + */ + +const MIN_TIER = 0; +const MAX_TIER = 10; +const DEFAULT_TIER = 1; + +/** + * Get polling interval for a tier in milliseconds + * @param {number} tier - Polling tier (0-10) + * @returns {number} Interval in milliseconds + */ +export function getIntervalForTier(tier) { + const clampedTier = Math.max(MIN_TIER, Math.min(MAX_TIER, tier)); + const minutes = Math.pow(2, clampedTier); + return minutes * 60 * 1000; +} + +/** + * Get next fetch time based on tier + * @param {number} tier - Polling tier + * @returns {Date} Next fetch time + */ +export function getNextFetchTime(tier) { + const interval = getIntervalForTier(tier); + return new Date(Date.now() + interval); +} + +/** + * Calculate new tier after a fetch + * @param {object} options - Options + * @param {number} options.currentTier - Current tier + * @param {boolean} options.hasNewItems - Whether new items were found + * @param {number} options.consecutiveUnchanged - Consecutive fetches with no changes + * @returns {object} New tier and metadata + */ +export function calculateNewTier(options) { + const { + currentTier = DEFAULT_TIER, + hasNewItems, + consecutiveUnchanged = 0, + } = options; + + let newTier = currentTier; + let newConsecutiveUnchanged = consecutiveUnchanged; + + if (hasNewItems) { + // Reset unchanged counter + newConsecutiveUnchanged = 0; + + // Decrease tier (more frequent) if we found new items + if (currentTier > MIN_TIER) { + newTier = currentTier - 1; + } + } else { + // Increment unchanged counter + newConsecutiveUnchanged = consecutiveUnchanged + 1; + + // Increase tier (less frequent) after consecutive unchanged fetches + // The threshold increases with tier to prevent thrashing + const threshold = Math.max(2, currentTier); + if (newConsecutiveUnchanged >= threshold && currentTier < MAX_TIER) { + newTier = currentTier + 1; + // Reset counter after tier change + newConsecutiveUnchanged = 0; + } + } + + return { + tier: newTier, + consecutiveUnchanged: newConsecutiveUnchanged, + nextFetchAt: getNextFetchTime(newTier), + }; +} + +/** + * Get initial tier for a new feed subscription + * @returns {object} Initial tier settings + */ +export function getInitialTier() { + return { + tier: MIN_TIER, // Start at tier 0 for immediate first fetch + consecutiveUnchanged: 0, + nextFetchAt: new Date(), // Fetch immediately + }; +} + +/** + * Determine if a feed is due for fetching + * @param {object} feed - Feed document + * @returns {boolean} Whether the feed should be fetched + */ +export function isDueForFetch(feed) { + if (!feed.nextFetchAt) { + return true; + } + + return new Date(feed.nextFetchAt) <= new Date(); +} + +/** + * Get human-readable description of polling interval + * @param {number} tier - Polling tier + * @returns {string} Description + */ +export function getTierDescription(tier) { + const minutes = Math.pow(2, tier); + + if (minutes < 60) { + return `every ${minutes} minute${minutes === 1 ? "" : "s"}`; + } + + const hours = minutes / 60; + if (hours < 24) { + return `every ${hours.toFixed(1)} hour${hours === 1 ? "" : "s"}`; + } + + const days = hours / 24; + return `every ${days.toFixed(1)} day${days === 1 ? "" : "s"}`; +} + +export { MIN_TIER, MAX_TIER, DEFAULT_TIER }; diff --git a/lib/realtime/broker.js b/lib/realtime/broker.js new file mode 100644 index 0000000..a9979f9 --- /dev/null +++ b/lib/realtime/broker.js @@ -0,0 +1,241 @@ +/** + * Server-Sent Events broker + * Manages SSE connections and event distribution + * @module realtime/broker + */ + +import { subscribeToChannel } from "../cache/redis.js"; + +/** + * SSE Client connection + * @typedef {object} SseClient + * @property {object} response - Express response object + * @property {string} userId - User ID + * @property {Set} channels - Subscribed channel IDs + */ + +/** @type {Map} */ +const clients = new Map(); + +/** @type {Map} Map of userId to Redis subscriber */ +const userSubscribers = new Map(); + +const PING_INTERVAL = 10_000; // 10 seconds + +/** + * Add a client to the broker + * @param {object} response - Express response object + * @param {string} userId - User ID + * @param {object} application - Indiekit application + * @returns {object} Client object + */ +export function addClient(response, userId, application) { + const client = { + response, + userId, + channels: new Set(), + pingInterval: setInterval(() => { + sendEvent(response, "ping", { timestamp: new Date().toISOString() }); + }, PING_INTERVAL), + }; + + clients.set(response, client); + + // Set up Redis subscription for this user if not already done + setupUserSubscription(userId, application); + + return client; +} + +/** + * Remove a client from the broker + * @param {object} response - Express response object + */ +export function removeClient(response) { + const client = clients.get(response); + if (client) { + clearInterval(client.pingInterval); + clients.delete(response); + + // Check if any other clients for this user + const hasOtherClients = [...clients.values()].some( + (c) => c.userId === client.userId, + ); + if (!hasOtherClients) { + // Could clean up Redis subscription here if needed + } + } +} + +/** + * Subscribe a client to a channel + * @param {object} response - Express response object + * @param {string} channelId - Channel ID + */ +export function subscribeClient(response, channelId) { + const client = clients.get(response); + if (client) { + client.channels.add(channelId); + } +} + +/** + * Unsubscribe a client from a channel + * @param {object} response - Express response object + * @param {string} channelId - Channel ID + */ +export function unsubscribeClient(response, channelId) { + const client = clients.get(response); + if (client) { + client.channels.delete(channelId); + } +} + +/** + * Send an event to a specific client + * @param {object} response - Express response object + * @param {string} event - Event name + * @param {object} data - Event data + */ +export function sendEvent(response, event, data) { + try { + response.write(`event: ${event}\n`); + response.write(`data: ${JSON.stringify(data)}\n\n`); + } catch { + // Client disconnected + removeClient(response); + } +} + +/** + * Broadcast an event to all clients subscribed to a channel + * @param {string} channelId - Channel ID + * @param {string} event - Event name + * @param {object} data - Event data + */ +export function broadcastToChannel(channelId, event, data) { + for (const client of clients.values()) { + if (client.channels.has(channelId)) { + sendEvent(client.response, event, data); + } + } +} + +/** + * Broadcast an event to all clients for a user + * @param {string} userId - User ID + * @param {string} event - Event name + * @param {object} data - Event data + */ +export function broadcastToUser(userId, event, data) { + for (const client of clients.values()) { + if (client.userId === userId) { + sendEvent(client.response, event, data); + } + } +} + +/** + * Broadcast an event to all connected clients + * @param {string} event - Event name + * @param {object} data - Event data + */ +export function broadcastToAll(event, data) { + for (const client of clients.values()) { + sendEvent(client.response, event, data); + } +} + +/** + * Set up Redis subscription for a user + * @param {string} userId - User ID + * @param {object} application - Indiekit application + */ +async function setupUserSubscription(userId, application) { + if (userSubscribers.has(userId)) { + return; // Already subscribed + } + + const redis = application.redis; + if (!redis) { + return; // No Redis, skip real-time + } + + // Create a duplicate connection for pub/sub + const subscriber = redis.duplicate(); + userSubscribers.set(userId, subscriber); + + try { + await subscribeToChannel(subscriber, `microsub:user:${userId}`, (data) => { + handleRedisEvent(userId, data); + }); + } catch { + // Subscription failed, remove from map + userSubscribers.delete(userId); + } +} + +/** + * Handle event received from Redis + * @param {string} userId - User ID + * @param {object} data - Event data + */ +function handleRedisEvent(userId, data) { + const { type, channelId, ...eventData } = data; + + switch (type) { + case "new-item": { + broadcastToUser(userId, "new-item", { channelId, ...eventData }); + break; + } + case "channel-update": { + broadcastToUser(userId, "channel-update", { channelId, ...eventData }); + break; + } + case "unread-count": { + broadcastToUser(userId, "unread-count", { channelId, ...eventData }); + break; + } + default: { + // Unknown event type, broadcast as generic event + broadcastToUser(userId, type, data); + } + } +} + +/** + * Get broker statistics + * @returns {object} Statistics + */ +export function getStats() { + const userCounts = new Map(); + for (const client of clients.values()) { + const count = userCounts.get(client.userId) || 0; + userCounts.set(client.userId, count + 1); + } + + return { + totalClients: clients.size, + uniqueUsers: userCounts.size, + userSubscribers: userSubscribers.size, + }; +} + +/** + * Clean up all connections + */ +export function cleanup() { + for (const client of clients.values()) { + clearInterval(client.pingInterval); + } + clients.clear(); + + for (const subscriber of userSubscribers.values()) { + try { + subscriber.quit(); + } catch { + // Ignore cleanup errors + } + } + userSubscribers.clear(); +} diff --git a/lib/search/indexer.js b/lib/search/indexer.js new file mode 100644 index 0000000..f3e1f63 --- /dev/null +++ b/lib/search/indexer.js @@ -0,0 +1,90 @@ +/** + * Search indexer for MongoDB text search + * @module search/indexer + */ + +/** + * Create text indexes for microsub items + * @param {object} application - Indiekit application + * @returns {Promise} + */ +export async function createSearchIndexes(application) { + const itemsCollection = application.collections.get("microsub_items"); + + // Create compound text index for full-text search + await itemsCollection.createIndex( + { + name: "text", + "content.text": "text", + "content.html": "text", + summary: "text", + "author.name": "text", + }, + { + name: "text_search", + weights: { + name: 10, + "content.text": 5, + summary: 3, + "author.name": 2, + }, + default_language: "english", + background: true, + }, + ); + + // Create index for channel + published for efficient timeline queries + await itemsCollection.createIndex( + { channelId: 1, published: -1 }, + { name: "channel_timeline" }, + ); + + // Create index for deduplication + await itemsCollection.createIndex( + { channelId: 1, uid: 1 }, + { name: "channel_uid", unique: true }, + ); + + // Create index for feed-based queries + await itemsCollection.createIndex({ feedId: 1 }, { name: "feed_items" }); +} + +/** + * Rebuild search indexes (drops and recreates) + * @param {object} application - Indiekit application + * @returns {Promise} + */ +export async function rebuildSearchIndexes(application) { + const itemsCollection = application.collections.get("microsub_items"); + + // Drop existing text index + try { + await itemsCollection.dropIndex("text_search"); + } catch { + // Index may not exist + } + + // Recreate indexes + await createSearchIndexes(application); +} + +/** + * Get search index stats + * @param {object} application - Indiekit application + * @returns {Promise} Index statistics + */ +export async function getSearchIndexStats(application) { + const itemsCollection = application.collections.get("microsub_items"); + + const indexes = await itemsCollection.indexes(); + const stats = await itemsCollection.stats(); + + return { + indexes: indexes.map((index) => ({ + name: index.name, + key: index.key, + })), + totalDocuments: stats.count, + size: stats.size, + }; +} diff --git a/lib/search/query.js b/lib/search/query.js new file mode 100644 index 0000000..918beb3 --- /dev/null +++ b/lib/search/query.js @@ -0,0 +1,198 @@ +/** + * Search query module for full-text search + * @module search/query + */ + +import { ObjectId } from "mongodb"; + +/** + * Search items using MongoDB text search + * @param {object} application - Indiekit application + * @param {ObjectId|string} channelId - Channel ObjectId + * @param {string} query - Search query string + * @param {object} options - Search options + * @param {number} [options.limit] - Max results (default 20) + * @param {number} [options.skip] - Skip results for pagination + * @param {boolean} [options.sortByScore] - Sort by relevance (default true) + * @returns {Promise} Array of matching items + */ +export async function searchItemsFullText( + application, + channelId, + query, + options = {}, +) { + const collection = application.collections.get("microsub_items"); + const { limit = 20, skip = 0, sortByScore = true } = options; + + const channelObjectId = + typeof channelId === "string" ? new ObjectId(channelId) : channelId; + + // Build the search query + const searchQuery = { + channelId: channelObjectId, + $text: { $search: query }, + }; + + // Build aggregation pipeline for scoring + const pipeline = [ + { $match: searchQuery }, + { $addFields: { score: { $meta: "textScore" } } }, + ]; + + if (sortByScore) { + pipeline.push( + { $sort: { score: -1, published: -1 } }, + { $skip: skip }, + { $limit: limit }, + ); + } else { + pipeline.push( + { $sort: { published: -1 } }, + { $skip: skip }, + { $limit: limit }, + ); + } + + const items = await collection.aggregate(pipeline).toArray(); + + return items.map((item) => transformToSearchResult(item)); +} + +/** + * Search items using regex fallback (for partial matching) + * @param {object} application - Indiekit application + * @param {ObjectId|string} channelId - Channel ObjectId + * @param {string} query - Search query string + * @param {object} options - Search options + * @returns {Promise} Array of matching items + */ +export async function searchItemsRegex( + application, + channelId, + query, + options = {}, +) { + const collection = application.collections.get("microsub_items"); + const { limit = 20 } = options; + + const channelObjectId = + typeof channelId === "string" ? new ObjectId(channelId) : channelId; + + // Escape regex special characters + const escapedQuery = query.replaceAll(/[$()*+.?[\\\]^{|}]/g, String.raw`\$&`); + const regex = new RegExp(escapedQuery, "i"); + + const items = await collection + .find({ + channelId: channelObjectId, + $or: [ + { name: regex }, + { "content.text": regex }, + { "content.html": regex }, + { summary: regex }, + { "author.name": regex }, + ], + }) + // eslint-disable-next-line unicorn/no-array-sort -- MongoDB cursor method, not Array#sort + .sort({ published: -1 }) + .limit(limit) + .toArray(); + + return items.map((item) => transformToSearchResult(item)); +} + +/** + * Search with automatic fallback + * Uses full-text search first, falls back to regex if no results + * @param {object} application - Indiekit application + * @param {ObjectId|string} channelId - Channel ObjectId + * @param {string} query - Search query string + * @param {object} options - Search options + * @returns {Promise} Array of matching items + */ +export async function searchWithFallback( + application, + channelId, + query, + options = {}, +) { + // Try full-text search first + try { + const results = await searchItemsFullText( + application, + channelId, + query, + options, + ); + if (results.length > 0) { + return results; + } + } catch { + // Text index might not exist, fall through to regex + } + + // Fall back to regex search + return searchItemsRegex(application, channelId, query, options); +} + +/** + * Transform database item to search result format + * @param {object} item - Database item + * @returns {object} Search result + */ +function transformToSearchResult(item) { + const result = { + type: item.type || "entry", + uid: item.uid, + url: item.url, + published: item.published?.toISOString(), + _id: item._id.toString(), + }; + + if (item.name) result.name = item.name; + if (item.content) result.content = item.content; + if (item.summary) result.summary = item.summary; + if (item.author) result.author = item.author; + if (item.photo?.length > 0) result.photo = item.photo; + if (item.score) result._score = item.score; + + return result; +} + +/** + * Get search suggestions (autocomplete) + * @param {object} application - Indiekit application + * @param {ObjectId|string} channelId - Channel ObjectId + * @param {string} prefix - Search prefix + * @param {number} limit - Max suggestions + * @returns {Promise} Array of suggestions + */ +export async function getSearchSuggestions( + application, + channelId, + prefix, + limit = 5, +) { + const collection = application.collections.get("microsub_items"); + + const channelObjectId = + typeof channelId === "string" ? new ObjectId(channelId) : channelId; + + const escapedPrefix = prefix.replaceAll( + /[$()*+.?[\\\]^{|}]/g, + String.raw`\$&`, + ); + const regex = new RegExp(`^${escapedPrefix}`, "i"); + + // Get unique names/titles that match prefix + const results = await collection + .aggregate([ + { $match: { channelId: channelObjectId, name: regex } }, + { $group: { _id: "$name" } }, + { $limit: limit }, + ]) + .toArray(); + + return results.map((r) => r._id).filter(Boolean); +} diff --git a/lib/storage/channels.js b/lib/storage/channels.js index 477f657..d013855 100644 --- a/lib/storage/channels.js +++ b/lib/storage/channels.js @@ -3,7 +3,12 @@ * @module storage/channels */ -import { generateChannelUid } from "../utils/uid.js"; +import { ObjectId } from "mongodb"; + +import { generateChannelUid } from "../utils/jf2.js"; + +import { deleteFeedsForChannel } from "./feeds.js"; +import { deleteItemsForChannel } from "./items.js"; /** * Get channels collection from application @@ -53,7 +58,7 @@ export async function createChannel(application, { name, userId }) { // Get max order for user const maxOrderResult = await collection .find({ userId }) - // eslint-disable-next-line unicorn/no-array-sort -- MongoDB cursor method + // eslint-disable-next-line unicorn/no-array-sort -- MongoDB cursor method, not Array#sort .sort({ order: -1 }) .limit(1) .toArray(); @@ -65,6 +70,10 @@ export async function createChannel(application, { name, userId }) { name, userId, order, + settings: { + excludeTypes: [], + excludeRegex: undefined, + }, createdAt: new Date(), updatedAt: new Date(), }; @@ -85,8 +94,12 @@ export async function getChannels(application, userId) { const itemsCollection = getItemsCollection(application); const filter = userId ? { userId } : {}; - // eslint-disable-next-line unicorn/no-array-callback-reference, unicorn/no-array-sort -- MongoDB methods - const channels = await collection.find(filter).sort({ order: 1 }).toArray(); + const channels = await collection + // eslint-disable-next-line unicorn/no-array-callback-reference -- filter is MongoDB query object + .find(filter) + // eslint-disable-next-line unicorn/no-array-sort -- MongoDB cursor method, not Array#sort + .sort({ order: 1 }) + .toArray(); // Get unread counts for each channel const channelsWithCounts = await Promise.all( @@ -134,6 +147,18 @@ export async function getChannel(application, uid, userId) { return collection.findOne(query); } +/** + * Get channel by MongoDB ObjectId + * @param {object} application - Indiekit application + * @param {ObjectId|string} id - Channel ObjectId + * @returns {Promise} Channel or null + */ +export async function getChannelById(application, id) { + const collection = getCollection(application); + const objectId = typeof id === "string" ? new ObjectId(id) : id; + return collection.findOne({ _id: objectId }); +} + /** * Update a channel * @param {object} application - Indiekit application @@ -162,7 +187,7 @@ export async function updateChannel(application, uid, updates, userId) { } /** - * Delete a channel and all its items + * Delete a channel and all its feeds and items * @param {object} application - Indiekit application * @param {string} uid - Channel UID * @param {string} [userId] - User ID @@ -170,7 +195,6 @@ export async function updateChannel(application, uid, updates, userId) { */ export async function deleteChannel(application, uid, userId) { const collection = getCollection(application); - const itemsCollection = getItemsCollection(application); const query = { uid }; if (userId) query.userId = userId; @@ -185,12 +209,11 @@ export async function deleteChannel(application, uid, userId) { return false; } - // Delete all items in channel - const itemsDeleted = await itemsCollection.deleteMany({ - channelId: channel._id, - }); + // Cascade delete: items first, then feeds, then channel + const itemsDeleted = await deleteItemsForChannel(application, channel._id); + const feedsDeleted = await deleteFeedsForChannel(application, channel._id); console.info( - `[Microsub] Deleted channel ${uid}: ${itemsDeleted.deletedCount} items`, + `[Microsub] Deleted channel ${uid}: ${feedsDeleted} feeds, ${itemsDeleted} items`, ); const result = await collection.deleteOne({ _id: channel._id }); @@ -220,6 +243,25 @@ export async function reorderChannels(application, channelUids, userId) { } } +/** + * Update channel settings + * @param {object} application - Indiekit application + * @param {string} uid - Channel UID + * @param {object} settings - Settings to update + * @param {Array} [settings.excludeTypes] - Types to exclude + * @param {string} [settings.excludeRegex] - Regex pattern to exclude + * @param {string} [userId] - User ID + * @returns {Promise} Updated channel + */ +export async function updateChannelSettings( + application, + uid, + settings, + userId, +) { + return updateChannel(application, uid, { settings }, userId); +} + /** * Ensure notifications channel exists * @param {object} application - Indiekit application @@ -244,6 +286,10 @@ export async function ensureNotificationsChannel(application, userId) { name: "Notifications", userId, order: -1, // Always first + settings: { + excludeTypes: [], + excludeRegex: undefined, + }, createdAt: new Date(), updatedAt: new Date(), }; diff --git a/lib/storage/feeds.js b/lib/storage/feeds.js new file mode 100644 index 0000000..5a9fb56 --- /dev/null +++ b/lib/storage/feeds.js @@ -0,0 +1,299 @@ +/** + * Feed subscription storage operations + * @module storage/feeds + */ + +import { ObjectId } from "mongodb"; + +import { deleteItemsForFeed } from "./items.js"; + +/** + * Get feeds collection from application + * @param {object} application - Indiekit application + * @returns {object} MongoDB collection + */ +function getCollection(application) { + return application.collections.get("microsub_feeds"); +} + +/** + * Create a new feed subscription + * @param {object} application - Indiekit application + * @param {object} data - Feed data + * @param {ObjectId} data.channelId - Channel ObjectId + * @param {string} data.url - Feed URL + * @param {string} [data.title] - Feed title + * @param {string} [data.photo] - Feed icon URL + * @returns {Promise} Created feed + */ +export async function createFeed( + application, + { channelId, url, title, photo }, +) { + const collection = getCollection(application); + + // Check if feed already exists in channel + const existing = await collection.findOne({ channelId, url }); + if (existing) { + return existing; + } + + const feed = { + channelId, + url, + title: title || undefined, + photo: photo || undefined, + tier: 1, // Start at tier 1 (2 minutes) + unmodified: 0, + nextFetchAt: new Date(), // Fetch immediately + lastFetchedAt: undefined, + websub: undefined, // Will be populated if hub is discovered + createdAt: new Date(), + updatedAt: new Date(), + }; + + await collection.insertOne(feed); + return feed; +} + +/** + * Get all feeds for a channel + * @param {object} application - Indiekit application + * @param {ObjectId|string} channelId - Channel ObjectId + * @returns {Promise} Array of feeds + */ +export async function getFeedsForChannel(application, channelId) { + const collection = getCollection(application); + const objectId = + typeof channelId === "string" ? new ObjectId(channelId) : channelId; + + return collection.find({ channelId: objectId }).toArray(); +} + +/** + * Get a feed by URL and channel + * @param {object} application - Indiekit application + * @param {ObjectId|string} channelId - Channel ObjectId + * @param {string} url - Feed URL + * @returns {Promise} Feed or null + */ +export async function getFeedByUrl(application, channelId, url) { + const collection = getCollection(application); + const objectId = + typeof channelId === "string" ? new ObjectId(channelId) : channelId; + + return collection.findOne({ channelId: objectId, url }); +} + +/** + * Get a feed by ID + * @param {object} application - Indiekit application + * @param {ObjectId|string} id - Feed ObjectId + * @returns {Promise} Feed or null + */ +export async function getFeedById(application, id) { + const collection = getCollection(application); + const objectId = typeof id === "string" ? new ObjectId(id) : id; + + return collection.findOne({ _id: objectId }); +} + +/** + * Update a feed + * @param {object} application - Indiekit application + * @param {ObjectId|string} id - Feed ObjectId + * @param {object} updates - Fields to update + * @returns {Promise} Updated feed + */ +export async function updateFeed(application, id, updates) { + const collection = getCollection(application); + const objectId = typeof id === "string" ? new ObjectId(id) : id; + + const result = await collection.findOneAndUpdate( + { _id: objectId }, + { + $set: { + ...updates, + updatedAt: new Date(), + }, + }, + { returnDocument: "after" }, + ); + + return result; +} + +/** + * Delete a feed subscription and all its items + * @param {object} application - Indiekit application + * @param {ObjectId|string} channelId - Channel ObjectId + * @param {string} url - Feed URL + * @returns {Promise} True if deleted + */ +export async function deleteFeed(application, channelId, url) { + const collection = getCollection(application); + const objectId = + typeof channelId === "string" ? new ObjectId(channelId) : channelId; + + // Find the feed first to get its ID for cascade delete + const feed = await collection.findOne({ channelId: objectId, url }); + if (!feed) { + return false; + } + + // Delete all items from this feed + const itemsDeleted = await deleteItemsForFeed(application, feed._id); + console.info(`[Microsub] Deleted ${itemsDeleted} items from feed ${url}`); + + // Delete the feed itself + const result = await collection.deleteOne({ _id: feed._id }); + return result.deletedCount > 0; +} + +/** + * Delete all feeds for a channel + * @param {object} application - Indiekit application + * @param {ObjectId|string} channelId - Channel ObjectId + * @returns {Promise} Number of deleted feeds + */ +export async function deleteFeedsForChannel(application, channelId) { + const collection = getCollection(application); + const objectId = + typeof channelId === "string" ? new ObjectId(channelId) : channelId; + + const result = await collection.deleteMany({ channelId: objectId }); + return result.deletedCount; +} + +/** + * Get feeds ready for polling + * @param {object} application - Indiekit application + * @returns {Promise} Array of feeds to fetch + */ +export async function getFeedsToFetch(application) { + const collection = getCollection(application); + const now = new Date(); + + return collection + .find({ + $or: [{ nextFetchAt: undefined }, { nextFetchAt: { $lte: now } }], + }) + .toArray(); +} + +/** + * Update feed after fetch + * @param {object} application - Indiekit application + * @param {ObjectId|string} id - Feed ObjectId + * @param {boolean} changed - Whether content changed + * @param {object} [extra] - Additional fields to update + * @returns {Promise} Updated feed + */ +export async function updateFeedAfterFetch( + application, + id, + changed, + extra = {}, +) { + const collection = getCollection(application); + const objectId = typeof id === "string" ? new ObjectId(id) : id; + + // If extra contains tier info, use that (from processor) + // Otherwise calculate locally (legacy behavior) + let updateData; + + if (extra.tier === undefined) { + // Get current feed state for legacy calculation + const feed = await collection.findOne({ _id: objectId }); + if (!feed) return; + + let tier = feed.tier; + let unmodified = feed.unmodified; + + if (changed) { + tier = Math.max(0, tier - 1); + unmodified = 0; + } else { + unmodified++; + if (unmodified >= 2) { + tier = Math.min(10, tier + 1); + unmodified = 0; + } + } + + const minutes = Math.ceil(Math.pow(2, tier)); + const nextFetchAt = new Date(Date.now() + minutes * 60 * 1000); + + updateData = { + tier, + unmodified, + nextFetchAt, + lastFetchedAt: new Date(), + updatedAt: new Date(), + }; + } else { + updateData = { + ...extra, + lastFetchedAt: new Date(), + updatedAt: new Date(), + }; + } + + return collection.findOneAndUpdate( + { _id: objectId }, + { $set: updateData }, + { returnDocument: "after" }, + ); +} + +/** + * Update feed WebSub subscription + * @param {object} application - Indiekit application + * @param {ObjectId|string} id - Feed ObjectId + * @param {object} websub - WebSub data + * @param {string} websub.hub - Hub URL + * @param {string} [websub.topic] - Feed topic URL + * @param {string} [websub.secret] - Subscription secret + * @param {number} [websub.leaseSeconds] - Lease duration + * @returns {Promise} Updated feed + */ +export async function updateFeedWebsub(application, id, websub) { + const collection = getCollection(application); + const objectId = typeof id === "string" ? new ObjectId(id) : id; + + const websubData = { + hub: websub.hub, + topic: websub.topic, + }; + + // Only set these if provided (subscription confirmed) + if (websub.secret) { + websubData.secret = websub.secret; + } + if (websub.leaseSeconds) { + websubData.leaseSeconds = websub.leaseSeconds; + websubData.expiresAt = new Date(Date.now() + websub.leaseSeconds * 1000); + } + + return collection.findOneAndUpdate( + { _id: objectId }, + { + $set: { + websub: websubData, + updatedAt: new Date(), + }, + }, + { returnDocument: "after" }, + ); +} + +/** + * Get feed by WebSub subscription ID + * Used for WebSub callback handling + * @param {object} application - Indiekit application + * @param {string} subscriptionId - Subscription ID (feed ObjectId as string) + * @returns {Promise} Feed or null + */ +export async function getFeedBySubscriptionId(application, subscriptionId) { + return getFeedById(application, subscriptionId); +} diff --git a/lib/storage/filters.js b/lib/storage/filters.js new file mode 100644 index 0000000..6493f32 --- /dev/null +++ b/lib/storage/filters.js @@ -0,0 +1,265 @@ +/** + * Filter storage operations (mute, block, channel filters) + * @module storage/filters + */ + +import { ObjectId } from "mongodb"; + +/** + * Get muted collection + * @param {object} application - Indiekit application + * @returns {object} MongoDB collection + */ +function getMutedCollection(application) { + return application.collections.get("microsub_muted"); +} + +/** + * Get blocked collection + * @param {object} application - Indiekit application + * @returns {object} MongoDB collection + */ +function getBlockedCollection(application) { + return application.collections.get("microsub_blocked"); +} + +/** + * Check if a URL is muted for a user/channel + * @param {object} application - Indiekit application + * @param {string} userId - User ID + * @param {ObjectId|string} channelId - Channel ObjectId + * @param {string} url - URL to check + * @returns {Promise} Whether the URL is muted + */ +export async function isMuted(application, userId, channelId, url) { + const collection = getMutedCollection(application); + const channelObjectId = + typeof channelId === "string" ? new ObjectId(channelId) : channelId; + + // Check for channel-specific mute + const channelMute = await collection.findOne({ + userId, + channelId: channelObjectId, + url, + }); + if (channelMute) return true; + + // Check for global mute (no channelId) + const globalMute = await collection.findOne({ + userId, + channelId: { $exists: false }, + url, + }); + return !!globalMute; +} + +/** + * Check if a URL is blocked for a user + * @param {object} application - Indiekit application + * @param {string} userId - User ID + * @param {string} url - URL to check + * @returns {Promise} Whether the URL is blocked + */ +export async function isBlocked(application, userId, url) { + const collection = getBlockedCollection(application); + const blocked = await collection.findOne({ userId, url }); + return !!blocked; +} + +/** + * Check if an item passes all filters + * @param {object} application - Indiekit application + * @param {string} userId - User ID + * @param {object} channel - Channel document with settings + * @param {object} item - Feed item to check + * @returns {Promise} Whether the item passes all filters + */ +export async function passesAllFilters(application, userId, channel, item) { + // Check if author URL is blocked + if ( + item.author?.url && + (await isBlocked(application, userId, item.author.url)) + ) { + return false; + } + + // Check if source URL is muted + if ( + item._source?.url && + (await isMuted(application, userId, channel._id, item._source.url)) + ) { + return false; + } + + // Check channel settings filters + if (channel?.settings) { + // Check excludeTypes + if (!passesTypeFilter(item, channel.settings)) { + return false; + } + + // Check excludeRegex + if (!passesRegexFilter(item, channel.settings)) { + return false; + } + } + + return true; +} + +/** + * Check if an item passes the excludeTypes filter + * @param {object} item - Feed item + * @param {object} settings - Channel settings + * @returns {boolean} Whether the item passes + */ +export function passesTypeFilter(item, settings) { + if (!settings.excludeTypes || settings.excludeTypes.length === 0) { + return true; + } + + const itemType = detectInteractionType(item); + return !settings.excludeTypes.includes(itemType); +} + +/** + * Check if an item passes the excludeRegex filter + * @param {object} item - Feed item + * @param {object} settings - Channel settings + * @returns {boolean} Whether the item passes + */ +export function passesRegexFilter(item, settings) { + if (!settings.excludeRegex) { + return true; + } + + try { + const regex = new RegExp(settings.excludeRegex, "i"); + const searchText = [ + item.name, + item.summary, + item.content?.text, + item.content?.html, + ] + .filter(Boolean) + .join(" "); + + return !regex.test(searchText); + } catch { + // Invalid regex, skip filter + return true; + } +} + +/** + * Detect the interaction type of an item + * @param {object} item - Feed item + * @returns {string} Interaction type + */ +export function detectInteractionType(item) { + if (item["like-of"] && item["like-of"].length > 0) { + return "like"; + } + if (item["repost-of"] && item["repost-of"].length > 0) { + return "repost"; + } + if (item["bookmark-of"] && item["bookmark-of"].length > 0) { + return "bookmark"; + } + if (item["in-reply-to"] && item["in-reply-to"].length > 0) { + return "reply"; + } + if (item.rsvp) { + return "rsvp"; + } + if (item.checkin) { + return "checkin"; + } + + return "post"; +} + +/** + * Get all muted URLs for a user/channel + * @param {object} application - Indiekit application + * @param {string} userId - User ID + * @param {ObjectId|string} [channelId] - Channel ObjectId (optional, for channel-specific) + * @returns {Promise} Array of muted URLs + */ +export async function getMutedUrls(application, userId, channelId) { + const collection = getMutedCollection(application); + const filter = { userId }; + + if (channelId) { + const channelObjectId = + typeof channelId === "string" ? new ObjectId(channelId) : channelId; + filter.channelId = channelObjectId; + } + + // eslint-disable-next-line unicorn/no-array-callback-reference -- filter is MongoDB query object + const muted = await collection.find(filter).toArray(); + return muted.map((m) => m.url); +} + +/** + * Get all blocked URLs for a user + * @param {object} application - Indiekit application + * @param {string} userId - User ID + * @returns {Promise} Array of blocked URLs + */ +export async function getBlockedUrls(application, userId) { + const collection = getBlockedCollection(application); + const blocked = await collection.find({ userId }).toArray(); + return blocked.map((b) => b.url); +} + +/** + * Update channel filter settings + * @param {object} application - Indiekit application + * @param {ObjectId|string} channelId - Channel ObjectId + * @param {object} filters - Filter settings to update + * @param {Array} [filters.excludeTypes] - Post types to exclude + * @param {string} [filters.excludeRegex] - Regex pattern to exclude + * @returns {Promise} Updated channel + */ +export async function updateChannelFilters(application, channelId, filters) { + const collection = application.collections.get("microsub_channels"); + const channelObjectId = + typeof channelId === "string" ? new ObjectId(channelId) : channelId; + + const updateFields = {}; + + if (filters.excludeTypes !== undefined) { + updateFields["settings.excludeTypes"] = filters.excludeTypes; + } + + if (filters.excludeRegex !== undefined) { + updateFields["settings.excludeRegex"] = filters.excludeRegex; + } + + const result = await collection.findOneAndUpdate( + { _id: channelObjectId }, + { $set: updateFields }, + { returnDocument: "after" }, + ); + + return result; +} + +/** + * Create indexes for filter collections + * @param {object} application - Indiekit application + * @returns {Promise} + */ +export async function createFilterIndexes(application) { + const mutedCollection = getMutedCollection(application); + const blockedCollection = getBlockedCollection(application); + + // Muted collection indexes + await mutedCollection.createIndex({ userId: 1, channelId: 1, url: 1 }); + await mutedCollection.createIndex({ userId: 1 }); + + // Blocked collection indexes + await blockedCollection.createIndex({ userId: 1, url: 1 }, { unique: true }); + await blockedCollection.createIndex({ userId: 1 }); +} diff --git a/lib/storage/items.js b/lib/storage/items.js index b80296a..fe9299b 100644 --- a/lib/storage/items.js +++ b/lib/storage/items.js @@ -21,6 +21,54 @@ function getCollection(application) { return application.collections.get("microsub_items"); } +/** + * Add an item to a channel + * @param {object} application - Indiekit application + * @param {object} data - Item data + * @param {ObjectId} data.channelId - Channel ObjectId + * @param {ObjectId} data.feedId - Feed ObjectId + * @param {string} data.uid - Unique item identifier + * @param {object} data.item - jf2 item data + * @returns {Promise} Created item or null if duplicate + */ +export async function addItem(application, { channelId, feedId, uid, item }) { + const collection = getCollection(application); + + // Check for duplicate + const existing = await collection.findOne({ channelId, uid }); + if (existing) { + return; // Duplicate, don't add + } + + const document = { + channelId, + feedId, + uid, + type: item.type || "entry", + url: item.url, + name: item.name || undefined, + content: item.content || undefined, + summary: item.summary || undefined, + published: item.published ? new Date(item.published) : new Date(), + updated: item.updated ? new Date(item.updated) : undefined, + author: item.author || undefined, + category: item.category || [], + photo: item.photo || [], + video: item.video || [], + audio: item.audio || [], + likeOf: item["like-of"] || item.likeOf || [], + repostOf: item["repost-of"] || item.repostOf || [], + bookmarkOf: item["bookmark-of"] || item.bookmarkOf || [], + inReplyTo: item["in-reply-to"] || item.inReplyTo || [], + source: item._source || undefined, + readBy: [], // Array of user IDs who have read this item + createdAt: new Date(), + }; + + await collection.insertOne(document); + return document; +} + /** * Get timeline items for a channel * @param {object} application - Indiekit application @@ -39,7 +87,6 @@ export async function getTimelineItems(application, channelId, options = {}) { const limit = parseLimit(options.limit); const baseQuery = { channelId: objectId }; - const query = buildPaginationQuery({ before: options.before, after: options.after, @@ -50,9 +97,9 @@ export async function getTimelineItems(application, channelId, options = {}) { // Fetch one extra to check if there are more const items = await collection - // eslint-disable-next-line unicorn/no-array-callback-reference -- MongoDB query object + // eslint-disable-next-line unicorn/no-array-callback-reference -- query is MongoDB query object .find(query) - // eslint-disable-next-line unicorn/no-array-sort -- MongoDB cursor method + // eslint-disable-next-line unicorn/no-array-sort -- MongoDB cursor method, not Array#sort .sort(sort) .limit(limit + 1) .toArray(); @@ -74,6 +121,50 @@ export async function getTimelineItems(application, channelId, options = {}) { }; } +/** + * Extract URL string from a media value + * @param {object|string} media - Media value (can be string URL or object) + * @returns {string|undefined} URL string + */ +function extractMediaUrl(media) { + if (!media) { + return; + } + if (typeof media === "string") { + return media; + } + if (typeof media === "object") { + return media.value || media.url || media.src; + } +} + +/** + * Normalize media array to URL strings + * @param {Array} mediaArray - Array of media items + * @returns {Array} Array of URL strings + */ +function normalizeMediaArray(mediaArray) { + if (!mediaArray || !Array.isArray(mediaArray)) { + return []; + } + return mediaArray.map((media) => extractMediaUrl(media)).filter(Boolean); +} + +/** + * Normalize author object to ensure photo is a URL string + * @param {object} author - Author object + * @returns {object} Normalized author + */ +function normalizeAuthor(author) { + if (!author) { + return; + } + return { + ...author, + photo: extractMediaUrl(author.photo), + }; +} + /** * Transform database item to jf2 format * @param {object} item - Database item @@ -95,11 +186,17 @@ function transformToJf2(item, userId) { if (item.content) jf2.content = item.content; if (item.summary) jf2.summary = item.summary; if (item.updated) jf2.updated = item.updated.toISOString(); - if (item.author) jf2.author = item.author; + if (item.author) jf2.author = normalizeAuthor(item.author); if (item.category?.length > 0) jf2.category = item.category; - if (item.photo?.length > 0) jf2.photo = item.photo; - if (item.video?.length > 0) jf2.video = item.video; - if (item.audio?.length > 0) jf2.audio = item.audio; + + // Normalize media arrays to ensure they contain URL strings + const photos = normalizeMediaArray(item.photo); + const videos = normalizeMediaArray(item.video); + const audios = normalizeMediaArray(item.audio); + + if (photos.length > 0) jf2.photo = photos; + if (videos.length > 0) jf2.video = videos; + if (audios.length > 0) jf2.audio = audios; // Interaction types if (item.likeOf?.length > 0) jf2["like-of"] = item.likeOf; @@ -113,11 +210,57 @@ function transformToJf2(item, userId) { return jf2; } +/** + * Get an item by ID (MongoDB _id or uid) + * @param {object} application - Indiekit application + * @param {ObjectId|string} id - Item ObjectId or uid string + * @param {string} [userId] - User ID for read state + * @returns {Promise} jf2 item or undefined + */ +export async function getItemById(application, id, userId) { + const collection = getCollection(application); + + let item; + + // Try MongoDB ObjectId first + try { + const objectId = typeof id === "string" ? new ObjectId(id) : id; + item = await collection.findOne({ _id: objectId }); + } catch { + // Invalid ObjectId format, will try uid lookup + } + + // If not found by _id, try uid + if (!item) { + item = await collection.findOne({ uid: id }); + } + + if (!item) { + return; + } + + return transformToJf2(item, userId); +} + +/** + * Get items by UIDs + * @param {object} application - Indiekit application + * @param {Array} uids - Array of item UIDs + * @param {string} [userId] - User ID for read state + * @returns {Promise} Array of jf2 items + */ +export async function getItemsByUids(application, uids, userId) { + const collection = getCollection(application); + + const items = await collection.find({ uid: { $in: uids } }).toArray(); + return items.map((item) => transformToJf2(item, userId)); +} + /** * Mark items as read * @param {object} application - Indiekit application * @param {ObjectId|string} channelId - Channel ObjectId - * @param {Array} entryIds - Array of entry IDs to mark as read + * @param {Array} entryIds - Array of entry IDs to mark as read (can be ObjectId, uid, or URL) * @param {string} userId - User ID * @returns {Promise} Number of items updated */ @@ -126,12 +269,22 @@ export async function markItemsRead(application, channelId, entryIds, userId) { const channelObjectId = typeof channelId === "string" ? new ObjectId(channelId) : channelId; + console.info( + `[Microsub] markItemsRead called for channel ${channelId}, entries:`, + entryIds, + `userId: ${userId}`, + ); + // Handle "last-read-entry" special value if (entryIds.includes("last-read-entry")) { + // Mark all items in channel as read const result = await collection.updateMany( { channelId: channelObjectId }, { $addToSet: { readBy: userId } }, ); + console.info( + `[Microsub] Marked all items as read: ${result.modifiedCount} updated`, + ); return result.modifiedCount; } @@ -146,7 +299,7 @@ export async function markItemsRead(application, channelId, entryIds, userId) { }) .filter(Boolean); - // Match by _id, uid, or url + // Build query to match by _id, uid, or url (Microsub spec uses URLs as entry identifiers) const result = await collection.updateMany( { channelId: channelObjectId, @@ -159,6 +312,9 @@ export async function markItemsRead(application, channelId, entryIds, userId) { { $addToSet: { readBy: userId } }, ); + console.info( + `[Microsub] markItemsRead result: ${result.modifiedCount} items updated`, + ); return result.modifiedCount; } @@ -166,7 +322,7 @@ export async function markItemsRead(application, channelId, entryIds, userId) { * Mark items as unread * @param {object} application - Indiekit application * @param {ObjectId|string} channelId - Channel ObjectId - * @param {Array} entryIds - Array of entry IDs to mark as unread + * @param {Array} entryIds - Array of entry IDs to mark as unread (can be ObjectId, uid, or URL) * @param {string} userId - User ID * @returns {Promise} Number of items updated */ @@ -211,7 +367,7 @@ export async function markItemsUnread( * Remove items from channel * @param {object} application - Indiekit application * @param {ObjectId|string} channelId - Channel ObjectId - * @param {Array} entryIds - Array of entry IDs to remove + * @param {Array} entryIds - Array of entry IDs to remove (can be ObjectId, uid, or URL) * @returns {Promise} Number of items removed */ export async function removeItems(application, channelId, entryIds) { @@ -243,6 +399,110 @@ export async function removeItems(application, channelId, entryIds) { return result.deletedCount; } +/** + * Delete all items for a channel + * @param {object} application - Indiekit application + * @param {ObjectId|string} channelId - Channel ObjectId + * @returns {Promise} Number of deleted items + */ +export async function deleteItemsForChannel(application, channelId) { + const collection = getCollection(application); + const objectId = + typeof channelId === "string" ? new ObjectId(channelId) : channelId; + + const result = await collection.deleteMany({ channelId: objectId }); + return result.deletedCount; +} + +/** + * Delete items for a specific feed + * @param {object} application - Indiekit application + * @param {ObjectId|string} feedId - Feed ObjectId + * @returns {Promise} Number of deleted items + */ +export async function deleteItemsForFeed(application, feedId) { + const collection = getCollection(application); + const objectId = typeof feedId === "string" ? new ObjectId(feedId) : feedId; + + const result = await collection.deleteMany({ feedId: objectId }); + return result.deletedCount; +} + +/** + * Get unread count for a channel + * @param {object} application - Indiekit application + * @param {ObjectId|string} channelId - Channel ObjectId + * @param {string} userId - User ID + * @returns {Promise} Unread count + */ +export async function getUnreadCount(application, channelId, userId) { + const collection = getCollection(application); + const objectId = + typeof channelId === "string" ? new ObjectId(channelId) : channelId; + + return collection.countDocuments({ + channelId: objectId, + readBy: { $ne: userId }, + }); +} + +/** + * Search items by text + * @param {object} application - Indiekit application + * @param {ObjectId|string} channelId - Channel ObjectId + * @param {string} query - Search query + * @param {number} [limit] - Max results + * @returns {Promise} Array of matching items + */ +export async function searchItems(application, channelId, query, limit = 20) { + const collection = getCollection(application); + const objectId = + typeof channelId === "string" ? new ObjectId(channelId) : channelId; + + // Use regex search (consider adding text index for better performance) + const regex = new RegExp(query, "i"); + const items = await collection + .find({ + channelId: objectId, + $or: [ + { name: regex }, + { "content.text": regex }, + { "content.html": regex }, + { summary: regex }, + ], + }) + // eslint-disable-next-line unicorn/no-array-sort -- MongoDB cursor method, not Array#sort + .sort({ published: -1 }) + .limit(limit) + .toArray(); + + return items.map((item) => transformToJf2(item)); +} + +/** + * Delete items by author URL (for blocking) + * @param {object} application - Indiekit application + * @param {string} userId - User ID (for filtering user's channels) + * @param {string} authorUrl - Author URL to delete items from + * @returns {Promise} Number of deleted items + */ +export async function deleteItemsByAuthorUrl(application, userId, authorUrl) { + const collection = getCollection(application); + const channelsCollection = application.collections.get("microsub_channels"); + + // Get all channel IDs for this user + const userChannels = await channelsCollection.find({ userId }).toArray(); + const channelIds = userChannels.map((c) => c._id); + + // Delete all items from blocked author in user's channels + const result = await collection.deleteMany({ + channelId: { $in: channelIds }, + "author.url": authorUrl, + }); + + return result.deletedCount; +} + /** * Create indexes for efficient queries * @param {object} application - Indiekit application @@ -254,7 +514,31 @@ export async function createIndexes(application) { // Primary query indexes await collection.createIndex({ channelId: 1, published: -1 }); await collection.createIndex({ channelId: 1, uid: 1 }, { unique: true }); + await collection.createIndex({ feedId: 1 }); // URL matching index for mark_read operations await collection.createIndex({ channelId: 1, url: 1 }); + + // Full-text search index with weights + // Higher weight = more importance in relevance scoring + await collection.createIndex( + { + name: "text", + "content.text": "text", + "content.html": "text", + summary: "text", + "author.name": "text", + }, + { + name: "text_search", + weights: { + name: 10, // Titles most important + summary: 5, // Summaries second + "content.text": 3, // Content third + "content.html": 2, // HTML content lower + "author.name": 1, // Author names lowest + }, + default_language: "english", + }, + ); } diff --git a/lib/storage/read-state.js b/lib/storage/read-state.js new file mode 100644 index 0000000..64376a0 --- /dev/null +++ b/lib/storage/read-state.js @@ -0,0 +1,109 @@ +/** + * Read state tracking utilities + * @module storage/read-state + */ + +import { markItemsRead, markItemsUnread, getUnreadCount } from "./items.js"; + +/** + * Mark entries as read for a user + * @param {object} application - Indiekit application + * @param {string} channelUid - Channel UID + * @param {Array} entries - Entry IDs to mark as read + * @param {string} userId - User ID + * @returns {Promise} Number of entries marked + */ +export async function markRead(application, channelUid, entries, userId) { + const channelsCollection = application.collections.get("microsub_channels"); + const channel = await channelsCollection.findOne({ uid: channelUid }); + + if (!channel) { + return 0; + } + + return markItemsRead(application, channel._id, entries, userId); +} + +/** + * Mark entries as unread for a user + * @param {object} application - Indiekit application + * @param {string} channelUid - Channel UID + * @param {Array} entries - Entry IDs to mark as unread + * @param {string} userId - User ID + * @returns {Promise} Number of entries marked + */ +export async function markUnread(application, channelUid, entries, userId) { + const channelsCollection = application.collections.get("microsub_channels"); + const channel = await channelsCollection.findOne({ uid: channelUid }); + + if (!channel) { + return 0; + } + + return markItemsUnread(application, channel._id, entries, userId); +} + +/** + * Get unread count for a channel + * @param {object} application - Indiekit application + * @param {string} channelUid - Channel UID + * @param {string} userId - User ID + * @returns {Promise} Unread count + */ +export async function getChannelUnreadCount(application, channelUid, userId) { + const channelsCollection = application.collections.get("microsub_channels"); + const channel = await channelsCollection.findOne({ uid: channelUid }); + + if (!channel) { + return 0; + } + + return getUnreadCount(application, channel._id, userId); +} + +/** + * Get unread counts for all channels + * @param {object} application - Indiekit application + * @param {string} userId - User ID + * @returns {Promise} Map of channel UID to unread count + */ +export async function getAllUnreadCounts(application, userId) { + const channelsCollection = application.collections.get("microsub_channels"); + const itemsCollection = application.collections.get("microsub_items"); + + // Aggregate unread counts per channel + const pipeline = [ + { + $match: { + readBy: { $ne: userId }, + }, + }, + { + $group: { + _id: "$channelId", + count: { $sum: 1 }, + }, + }, + ]; + + const results = await itemsCollection.aggregate(pipeline).toArray(); + + // Get channel UIDs + const channelIds = results.map((r) => r._id); + const channels = await channelsCollection + .find({ _id: { $in: channelIds } }) + .toArray(); + + const channelMap = new Map(channels.map((c) => [c._id.toString(), c.uid])); + + // Build result map + const unreadCounts = new Map(); + for (const result of results) { + const uid = channelMap.get(result._id.toString()); + if (uid) { + unreadCounts.set(uid, result.count); + } + } + + return unreadCounts; +} diff --git a/lib/utils/auth.js b/lib/utils/auth.js index f052df4..f4bbab6 100644 --- a/lib/utils/auth.js +++ b/lib/utils/auth.js @@ -11,7 +11,7 @@ * 2. request.session.me (from token introspection) * 3. application.publication.me (single-user fallback) * @param {object} request - Express request - * @returns {string} User ID + * @returns {string|undefined} User ID */ export function getUserId(request) { // Check session for explicit userId @@ -31,5 +31,6 @@ export function getUserId(request) { } // Final fallback: use "default" as user ID for single-user instances + // This ensures read state is tracked even without explicit user identity return "default"; } diff --git a/lib/utils/jf2.js b/lib/utils/jf2.js new file mode 100644 index 0000000..b85c7f1 --- /dev/null +++ b/lib/utils/jf2.js @@ -0,0 +1,170 @@ +/** + * jf2 utility functions for Microsub + * @module utils/jf2 + */ + +import { createHash } from "node:crypto"; + +/** + * Generate a unique ID for an item based on feed URL and item identifier + * @param {string} feedUrl - Feed URL + * @param {string} itemId - Item ID or URL + * @returns {string} Unique item ID + */ +export function generateItemUid(feedUrl, itemId) { + const input = `${feedUrl}:${itemId}`; + return createHash("sha256").update(input).digest("hex").slice(0, 24); +} + +/** + * Generate a random channel UID + * @returns {string} 24-character random string + */ +export function generateChannelUid() { + const chars = "abcdefghijklmnopqrstuvwxyz0123456789"; + let result = ""; + for (let index = 0; index < 24; index++) { + result += chars.charAt(Math.floor(Math.random() * chars.length)); + } + return result; +} + +/** + * Create a jf2 Item from normalized feed data + * @param {object} data - Normalized item data + * @param {object} source - Feed source metadata + * @returns {object} jf2 Item object + */ +export function createJf2Item(data, source) { + return { + type: "entry", + uid: data.uid, + url: data.url, + name: data.name || undefined, + content: data.content || undefined, + summary: data.summary || undefined, + published: data.published, + updated: data.updated || undefined, + author: data.author || undefined, + category: data.category || [], + photo: data.photo || [], + video: data.video || [], + audio: data.audio || [], + // Interaction types + "like-of": data.likeOf || [], + "repost-of": data.repostOf || [], + "bookmark-of": data.bookmarkOf || [], + "in-reply-to": data.inReplyTo || [], + // Internal properties (prefixed with _) + _id: data._id, + _is_read: data._is_read || false, + _source: source, + }; +} + +/** + * Create a jf2 Card (author/person) + * @param {object} data - Author data + * @returns {object} jf2 Card object + */ +export function createJf2Card(data) { + if (!data) return; + + return { + type: "card", + name: data.name || undefined, + url: data.url || undefined, + photo: data.photo || undefined, + }; +} + +/** + * Create a jf2 Content object + * @param {string} text - Plain text content + * @param {string} html - HTML content + * @returns {object|undefined} jf2 Content object + */ +export function createJf2Content(text, html) { + if (!text && !html) return; + + return { + text: text || stripHtml(html), + html: html || undefined, + }; +} + +/** + * Strip HTML tags from string + * @param {string} html - HTML string + * @returns {string} Plain text + */ +export function stripHtml(html) { + if (!html) return ""; + return html.replaceAll(/<[^>]*>/g, "").trim(); +} + +/** + * Create a jf2 Feed response + * @param {object} options - Feed options + * @param {Array} options.items - Array of jf2 items + * @param {object} options.paging - Pagination cursors + * @returns {object} jf2 Feed object + */ +export function createJf2Feed({ items, paging }) { + const feed = { + items: items || [], + }; + + if (paging) { + feed.paging = {}; + if (paging.before) feed.paging.before = paging.before; + if (paging.after) feed.paging.after = paging.after; + } + + return feed; +} + +/** + * Create a Channel response object + * @param {object} channel - Channel data + * @param {number} unreadCount - Number of unread items + * @returns {object} Channel object for API response + */ +export function createChannelResponse(channel, unreadCount = 0) { + return { + uid: channel.uid, + name: channel.name, + unread: unreadCount > 0 ? unreadCount : false, + }; +} + +/** + * Create a Feed response object + * @param {object} feed - Feed data + * @returns {object} Feed object for API response + */ +export function createFeedResponse(feed) { + return { + type: "feed", + url: feed.url, + name: feed.title || undefined, + photo: feed.photo || undefined, + }; +} + +/** + * Detect interaction type from item properties + * @param {object} item - jf2 item + * @returns {string|undefined} Interaction type + */ +export function detectInteractionType(item) { + if (item["like-of"]?.length > 0 || item.likeOf?.length > 0) return "like"; + if (item["repost-of"]?.length > 0 || item.repostOf?.length > 0) + return "repost"; + if (item["bookmark-of"]?.length > 0 || item.bookmarkOf?.length > 0) + return "bookmark"; + if (item["in-reply-to"]?.length > 0 || item.inReplyTo?.length > 0) + return "reply"; + if (item.checkin) return "checkin"; + return; +} diff --git a/lib/utils/pagination.js b/lib/utils/pagination.js index 96cc3dc..1be59ed 100644 --- a/lib/utils/pagination.js +++ b/lib/utils/pagination.js @@ -5,16 +5,6 @@ import { ObjectId } from "mongodb"; -/** - * Default pagination limit - */ -export const DEFAULT_LIMIT = 20; - -/** - * Maximum pagination limit - */ -export const MAX_LIMIT = 100; - /** * Encode a cursor from timestamp and ID * @param {Date} timestamp - Item timestamp @@ -32,7 +22,7 @@ export function encodeCursor(timestamp, id) { /** * Decode a cursor string * @param {string} cursor - Base64-encoded cursor - * @returns {object|undefined} Decoded cursor with timestamp and id + * @returns {object|null} Decoded cursor with timestamp and id */ export function decodeCursor(cursor) { if (!cursor) return; @@ -95,6 +85,8 @@ export function buildPaginationQuery({ before, after, baseQuery = {} }) { * @returns {object} MongoDB sort object */ export function buildPaginationSort(before) { + // When using 'before', we fetch newer items, so sort ascending then reverse + // Otherwise, sort descending (newest first) if (before) { return { published: 1, _id: 1 }; } @@ -116,16 +108,23 @@ export function generatePagingCursors(items, limit, hasMore, before) { const paging = {}; + // If we fetched with 'before', results are in ascending order + // Reverse them and set cursors accordingly if (before) { items.reverse(); + // There are older items (the direction we came from) paging.after = encodeCursor(items.at(-1).published, items.at(-1)._id); if (hasMore) { + // There are newer items ahead paging.before = encodeCursor(items[0].published, items[0]._id); } } else { + // Normal descending order if (hasMore) { + // There are older items paging.after = encodeCursor(items.at(-1).published, items.at(-1)._id); } + // If we have items, there might be newer ones if (items.length > 0) { paging.before = encodeCursor(items[0].published, items[0]._id); } @@ -134,6 +133,16 @@ export function generatePagingCursors(items, limit, hasMore, before) { return paging; } +/** + * Default pagination limit + */ +export const DEFAULT_LIMIT = 20; + +/** + * Maximum pagination limit + */ +export const MAX_LIMIT = 100; + /** * Parse and validate limit parameter * @param {string|number} limit - Requested limit diff --git a/lib/utils/uid.js b/lib/utils/uid.js deleted file mode 100644 index 1b4eecd..0000000 --- a/lib/utils/uid.js +++ /dev/null @@ -1,17 +0,0 @@ -/** - * UID generation utilities for Microsub - * @module utils/uid - */ - -/** - * Generate a random channel UID - * @returns {string} 24-character random string - */ -export function generateChannelUid() { - const chars = "abcdefghijklmnopqrstuvwxyz0123456789"; - let result = ""; - for (let index = 0; index < 24; index++) { - result += chars.charAt(Math.floor(Math.random() * chars.length)); - } - return result; -} diff --git a/lib/utils/validation.js b/lib/utils/validation.js index ca1049b..a2a02df 100644 --- a/lib/utils/validation.js +++ b/lib/utils/validation.js @@ -6,9 +6,42 @@ import { IndiekitError } from "@indiekit/error"; /** - * Valid Microsub actions (PR 1: channels and timeline only) + * Valid Microsub actions */ -export const VALID_ACTIONS = ["channels", "timeline"]; +export const VALID_ACTIONS = [ + "channels", + "timeline", + "follow", + "unfollow", + "search", + "preview", + "mute", + "unmute", + "block", + "unblock", + "events", +]; + +/** + * Valid channel methods + */ +export const VALID_CHANNEL_METHODS = ["delete", "order"]; + +/** + * Valid timeline methods + */ +export const VALID_TIMELINE_METHODS = ["mark_read", "mark_unread", "remove"]; + +/** + * Valid exclude types for channel filtering + */ +export const VALID_EXCLUDE_TYPES = [ + "like", + "repost", + "bookmark", + "reply", + "checkin", +]; /** * Validate action parameter @@ -49,6 +82,29 @@ export function validateChannel(channel, required = true) { } } +/** + * Validate URL parameter + * @param {string} url - URL to validate + * @param {string} [paramName] - Parameter name for error message + * @param parameterName + * @throws {IndiekitError} If URL is invalid + */ +export function validateUrl(url, parameterName = "url") { + if (!url) { + throw new IndiekitError(`Missing required parameter: ${parameterName}`, { + status: 400, + }); + } + + try { + new URL(url); + } catch { + throw new IndiekitError(`Invalid URL: ${url}`, { + status: 400, + }); + } +} + /** * Validate entry/entries parameter * @param {string|Array} entry - Entry ID(s) to validate @@ -93,11 +149,43 @@ export function validateChannelName(name) { } } +/** + * Validate exclude types array + * @param {Array} types - Array of exclude types + * @returns {Array} Validated exclude types + */ +export function validateExcludeTypes(types) { + if (!types || !Array.isArray(types)) { + return []; + } + + return types.filter((type) => VALID_EXCLUDE_TYPES.includes(type)); +} + +/** + * Validate regex pattern + * @param {string} pattern - Regex pattern to validate + * @returns {string|null} Valid pattern or null + */ +export function validateExcludeRegex(pattern) { + if (!pattern || typeof pattern !== "string") { + return; + } + + try { + new RegExp(pattern); + return pattern; + } catch { + return; + } +} + /** * Parse array parameter from request * Handles both array[] and array[0], array[1] formats * @param {object} body - Request body - * @param {string} parameterName - Parameter name + * @param {string} paramName - Parameter name + * @param parameterName * @returns {Array} Parsed array */ export function parseArrayParameter(body, parameterName) { diff --git a/lib/webmention/processor.js b/lib/webmention/processor.js new file mode 100644 index 0000000..ae74487 --- /dev/null +++ b/lib/webmention/processor.js @@ -0,0 +1,214 @@ +/** + * Webmention processor + * @module webmention/processor + */ + +import { getRedisClient, publishEvent } from "../cache/redis.js"; +import { ensureNotificationsChannel } from "../storage/channels.js"; + +import { verifyWebmention } from "./verifier.js"; + +/** + * Get notifications collection + * @param {object} application - Indiekit application + * @returns {object} MongoDB collection + */ +function getCollection(application) { + return application.collections.get("microsub_notifications"); +} + +/** + * Process a webmention + * @param {object} application - Indiekit application + * @param {string} source - Source URL + * @param {string} target - Target URL + * @param {string} [userId] - User ID (for user-specific notifications) + * @returns {Promise} Processing result + */ +export async function processWebmention(application, source, target, userId) { + // Verify the webmention + const verification = await verifyWebmention(source, target); + + if (!verification.verified) { + console.log( + `[Microsub] Webmention verification failed: ${verification.error}`, + ); + return { + success: false, + error: verification.error, + }; + } + + // Ensure notifications channel exists + const channel = await ensureNotificationsChannel(application, userId); + + // Check for existing notification (update if exists) + const collection = getCollection(application); + const existing = await collection.findOne({ + source, + target, + ...(userId && { userId }), + }); + + const notification = { + source, + target, + userId, + channelId: channel._id, + type: verification.type, + author: verification.author, + content: verification.content, + url: verification.url, + published: verification.published + ? new Date(verification.published) + : new Date(), + verified: true, + readBy: [], + updatedAt: new Date(), + }; + + if (existing) { + // Update existing notification + await collection.updateOne({ _id: existing._id }, { $set: notification }); + notification._id = existing._id; + } else { + // Insert new notification + notification.createdAt = new Date(); + await collection.insertOne(notification); + } + + // Publish real-time event + const redis = getRedisClient(application); + if (redis && userId) { + await publishEvent(redis, `microsub:user:${userId}`, { + type: "new-notification", + channelId: channel._id.toString(), + notification: transformNotification(notification), + }); + } + + console.log( + `[Microsub] Webmention processed: ${verification.type} from ${source}`, + ); + + return { + success: true, + type: verification.type, + id: notification._id?.toString(), + }; +} + +/** + * Delete a webmention (when source no longer links to target) + * @param {object} application - Indiekit application + * @param {string} source - Source URL + * @param {string} target - Target URL + * @returns {Promise} Whether deletion was successful + */ +export async function deleteWebmention(application, source, target) { + const collection = getCollection(application); + const result = await collection.deleteOne({ source, target }); + return result.deletedCount > 0; +} + +/** + * Get notifications for a user + * @param {object} application - Indiekit application + * @param {string} userId - User ID + * @param {object} options - Query options + * @returns {Promise} Array of notifications + */ +export async function getNotifications(application, userId, options = {}) { + const collection = getCollection(application); + const { limit = 20, unreadOnly = false } = options; + + const query = { userId }; + if (unreadOnly) { + query.readBy = { $ne: userId }; + } + + /* eslint-disable unicorn/no-array-callback-reference, unicorn/no-array-sort -- MongoDB cursor methods */ + const notifications = await collection + .find(query) + .sort({ published: -1 }) + .limit(limit) + .toArray(); + /* eslint-enable unicorn/no-array-callback-reference, unicorn/no-array-sort */ + + return notifications.map((n) => transformNotification(n, userId)); +} + +/** + * Mark notifications as read + * @param {object} application - Indiekit application + * @param {string} userId - User ID + * @param {Array} ids - Notification IDs to mark as read + * @returns {Promise} Number of notifications updated + */ +export async function markNotificationsRead(application, userId, ids) { + const collection = getCollection(application); + const { ObjectId } = await import("mongodb"); + + const objectIds = ids.map((id) => { + try { + return new ObjectId(id); + } catch { + return id; + } + }); + + const result = await collection.updateMany( + { _id: { $in: objectIds } }, + { $addToSet: { readBy: userId } }, + ); + + return result.modifiedCount; +} + +/** + * Get unread notification count + * @param {object} application - Indiekit application + * @param {string} userId - User ID + * @returns {Promise} Unread count + */ +export async function getUnreadNotificationCount(application, userId) { + const collection = getCollection(application); + return collection.countDocuments({ + userId, + readBy: { $ne: userId }, + }); +} + +/** + * Transform notification to API format + * @param {object} notification - Database notification + * @param {string} [userId] - User ID for read state + * @returns {object} Transformed notification + */ +function transformNotification(notification, userId) { + return { + type: "entry", + uid: notification._id?.toString(), + url: notification.url || notification.source, + published: notification.published?.toISOString(), + author: notification.author, + content: notification.content, + _source: notification.source, + _target: notification.target, + _type: notification.type, // like, reply, repost, bookmark, mention + _is_read: userId ? notification.readBy?.includes(userId) : false, + }; +} + +/** + * Create indexes for notifications + * @param {object} application - Indiekit application + * @returns {Promise} + */ +export async function createNotificationIndexes(application) { + const collection = getCollection(application); + + await collection.createIndex({ userId: 1, published: -1 }); + await collection.createIndex({ source: 1, target: 1 }); + await collection.createIndex({ userId: 1, readBy: 1 }); +} diff --git a/lib/webmention/receiver.js b/lib/webmention/receiver.js new file mode 100644 index 0000000..0a6dfe8 --- /dev/null +++ b/lib/webmention/receiver.js @@ -0,0 +1,56 @@ +/** + * Webmention receiver + * @module webmention/receiver + */ + +import { getUserId } from "../utils/auth.js"; + +import { processWebmention } from "./processor.js"; + +/** + * Receive a webmention + * POST /microsub/webmention + * @param {object} request - Express request + * @param {object} response - Express response + */ +export async function receive(request, response) { + const { source, target } = request.body; + + if (!source || !target) { + return response.status(400).json({ + error: "invalid_request", + error_description: "Missing source or target parameter", + }); + } + + // Validate URLs + try { + new URL(source); + new URL(target); + } catch { + return response.status(400).json({ + error: "invalid_request", + error_description: "Invalid source or target URL", + }); + } + + const { application } = request.app.locals; + const userId = getUserId(request); + + // Return 202 Accepted immediately (processing asynchronously) + response.status(202).json({ + status: "accepted", + message: "Webmention queued for processing", + }); + + // Process webmention in background + setImmediate(async () => { + try { + await processWebmention(application, source, target, userId); + } catch (error) { + console.error(`[Microsub] Error processing webmention: ${error.message}`); + } + }); +} + +export const webmentionReceiver = { receive }; diff --git a/lib/webmention/verifier.js b/lib/webmention/verifier.js new file mode 100644 index 0000000..5296d1c --- /dev/null +++ b/lib/webmention/verifier.js @@ -0,0 +1,308 @@ +/** + * Webmention verification + * @module webmention/verifier + */ + +import { mf2 } from "microformats-parser"; + +/** + * Verify a webmention + * @param {string} source - Source URL + * @param {string} target - Target URL + * @returns {Promise} Verification result + */ +export async function verifyWebmention(source, target) { + try { + // Fetch the source URL + const response = await fetch(source, { + headers: { + Accept: "text/html, application/xhtml+xml", + "User-Agent": "Indiekit Microsub/1.0 (+https://getindiekit.com)", + }, + redirect: "follow", + }); + + if (!response.ok) { + return { + verified: false, + error: `Source returned ${response.status}`, + }; + } + + const content = await response.text(); + const finalUrl = response.url; + + // Check if source links to target + if (!containsLink(content, target)) { + return { + verified: false, + error: "Source does not link to target", + }; + } + + // Parse microformats + const parsed = mf2(content, { baseUrl: finalUrl }); + const entry = findEntry(parsed, target); + + if (!entry) { + // Still valid, just no h-entry context + return { + verified: true, + type: "mention", + author: undefined, + content: undefined, + }; + } + + // Determine webmention type + const mentionType = detectMentionType(entry, target); + + // Extract author + const author = extractAuthor(entry, parsed); + + // Extract content + const webmentionContent = extractContent(entry); + + return { + verified: true, + type: mentionType, + author, + content: webmentionContent, + url: getFirst(entry.properties.url) || source, + published: getFirst(entry.properties.published), + }; + } catch (error) { + return { + verified: false, + error: `Verification failed: ${error.message}`, + }; + } +} + +/** + * Check if content contains a link to target + * @param {string} content - HTML content + * @param {string} target - Target URL to find + * @returns {boolean} Whether the link exists + */ +function containsLink(content, target) { + // Normalize target URL for matching + const normalizedTarget = target.replace(/\/$/, ""); + + // Check for href attribute containing target + const hrefPattern = new RegExp( + `href=["']${escapeRegex(normalizedTarget)}/?["']`, + "i", + ); + if (hrefPattern.test(content)) { + return true; + } + + // Also check without quotes (some edge cases) + return content.includes(target) || content.includes(normalizedTarget); +} + +/** + * Find the h-entry that references the target + * @param {object} parsed - Parsed microformats + * @param {string} target - Target URL + * @returns {object|undefined} The h-entry or undefined + */ +function findEntry(parsed, target) { + const normalizedTarget = target.replace(/\/$/, ""); + + for (const item of parsed.items) { + // Check if this entry references the target + if ( + item.type?.includes("h-entry") && + entryReferencesTarget(item, normalizedTarget) + ) { + return item; + } + + // Check children + if (item.children) { + for (const child of item.children) { + if ( + child.type?.includes("h-entry") && + entryReferencesTarget(child, normalizedTarget) + ) { + return child; + } + } + } + } + + // Return first h-entry as fallback + for (const item of parsed.items) { + if (item.type?.includes("h-entry")) { + return item; + } + } + + return; +} + +/** + * Check if an entry references the target URL + * @param {object} entry - h-entry object + * @param {string} target - Normalized target URL + * @returns {boolean} Whether the entry references the target + */ +function entryReferencesTarget(entry, target) { + const properties = entry.properties || {}; + + // Check interaction properties + const interactionProperties = [ + "in-reply-to", + "like-of", + "repost-of", + "bookmark-of", + ]; + + for (const property of interactionProperties) { + const values = properties[property] || []; + for (const value of values) { + const url = + typeof value === "string" ? value : value?.properties?.url?.[0]; + if (url && normalizeUrl(url) === target) { + return true; + } + } + } + + return false; +} + +/** + * Detect the type of webmention + * @param {object} entry - h-entry object + * @param {string} target - Target URL + * @returns {string} Mention type + */ +function detectMentionType(entry, target) { + const properties = entry.properties || {}; + const normalizedTarget = target.replace(/\/$/, ""); + + // Check for specific interaction types + if (matchesTarget(properties["like-of"], normalizedTarget)) { + return "like"; + } + if (matchesTarget(properties["repost-of"], normalizedTarget)) { + return "repost"; + } + if (matchesTarget(properties["bookmark-of"], normalizedTarget)) { + return "bookmark"; + } + if (matchesTarget(properties["in-reply-to"], normalizedTarget)) { + return "reply"; + } + + return "mention"; +} + +/** + * Check if any value in array matches target + * @param {Array} values - Array of values + * @param {string} target - Target URL to match + * @returns {boolean} Whether any value matches + */ +function matchesTarget(values, target) { + if (!values || values.length === 0) return false; + + for (const value of values) { + const url = typeof value === "string" ? value : value?.properties?.url?.[0]; + if (url && normalizeUrl(url) === target) { + return true; + } + } + + return false; +} + +/** + * Extract author from entry or page + * @param {object} entry - h-entry object + * @param {object} parsed - Full parsed microformats + * @returns {object|undefined} Author object + */ +function extractAuthor(entry, parsed) { + const author = getFirst(entry.properties?.author); + + if (typeof author === "string") { + return { name: author }; + } + + if (author?.type?.includes("h-card")) { + return { + type: "card", + name: getFirst(author.properties?.name), + url: getFirst(author.properties?.url), + photo: getFirst(author.properties?.photo), + }; + } + + // Try to find author from page's h-card + const hcard = parsed.items.find((item) => item.type?.includes("h-card")); + if (hcard) { + return { + type: "card", + name: getFirst(hcard.properties?.name), + url: getFirst(hcard.properties?.url), + photo: getFirst(hcard.properties?.photo), + }; + } + + return; +} + +/** + * Extract content from entry + * @param {object} entry - h-entry object + * @returns {object|undefined} Content object + */ +function extractContent(entry) { + const content = getFirst(entry.properties?.content); + + if (!content) { + const summary = getFirst(entry.properties?.summary); + const name = getFirst(entry.properties?.name); + return summary || name ? { text: summary || name } : undefined; + } + + if (typeof content === "string") { + return { text: content }; + } + + return { + text: content.value, + html: content.html, + }; +} + +/** + * Get first item from array + * @param {Array|*} value - Value or array + * @returns {*} First value + */ +function getFirst(value) { + return Array.isArray(value) ? value[0] : value; +} + +/** + * Normalize URL for comparison + * @param {string} url - URL to normalize + * @returns {string} Normalized URL + */ +function normalizeUrl(url) { + return url.replace(/\/$/, ""); +} + +/** + * Escape special regex characters + * @param {string} string - String to escape + * @returns {string} Escaped string + */ +function escapeRegex(string) { + return string.replaceAll(/[$()*+.?[\\\]^{|}]/g, String.raw`\$&`); +} diff --git a/lib/websub/discovery.js b/lib/websub/discovery.js new file mode 100644 index 0000000..325187f --- /dev/null +++ b/lib/websub/discovery.js @@ -0,0 +1,129 @@ +/** + * WebSub hub discovery + * @module websub/discovery + */ + +/** + * Discover WebSub hub from HTTP response headers and content + * @param {object} response - Fetch response object + * @param {string} content - Response body content + * @returns {object|undefined} WebSub info { hub, self } + */ +export function discoverWebsub(response, content) { + // Try to find hub and self URLs from Link headers first + const linkHeader = response.headers.get("link"); + const fromHeaders = linkHeader ? parseLinkHeader(linkHeader) : {}; + + // Fall back to content parsing + const fromContent = parseContentForLinks(content); + + const hub = fromHeaders.hub || fromContent.hub; + const self = fromHeaders.self || fromContent.self; + + if (hub) { + return { hub, self }; + } + + return; +} + +/** + * Parse Link header for hub and self URLs + * @param {string} linkHeader - Link header value + * @returns {object} { hub, self } + */ +function parseLinkHeader(linkHeader) { + const result = {}; + const links = linkHeader.split(","); + + for (const link of links) { + const parts = link.trim().split(";"); + if (parts.length < 2) continue; + + const urlMatch = parts[0].match(/<([^>]+)>/); + if (!urlMatch) continue; + + const url = urlMatch[1]; + const relationship = parts + .slice(1) + .find((p) => p.trim().startsWith("rel=")) + ?.match(/rel=["']?([^"'\s;]+)["']?/)?.[1]; + + if (relationship === "hub") { + result.hub = url; + } else if (relationship === "self") { + result.self = url; + } + } + + return result; +} + +/** + * Parse content for hub and self URLs (Atom, RSS, HTML) + * @param {string} content - Response body + * @returns {object} { hub, self } + */ +function parseContentForLinks(content) { + const result = {}; + + // Try HTML elements + const htmlHubMatch = content.match( + /]+rel=["']?hub["']?[^>]+href=["']([^"']+)["']/i, + ); + if (htmlHubMatch) { + result.hub = htmlHubMatch[1]; + } + + const htmlSelfMatch = content.match( + /]+rel=["']?self["']?[^>]+href=["']([^"']+)["']/i, + ); + if (htmlSelfMatch) { + result.self = htmlSelfMatch[1]; + } + + // Also try the reverse order (href before rel) + if (!result.hub) { + const htmlHubMatch2 = content.match( + /]+href=["']([^"']+)["'][^>]+rel=["']?hub["']?/i, + ); + if (htmlHubMatch2) { + result.hub = htmlHubMatch2[1]; + } + } + + if (!result.self) { + const htmlSelfMatch2 = content.match( + /]+href=["']([^"']+)["'][^>]+rel=["']?self["']?/i, + ); + if (htmlSelfMatch2) { + result.self = htmlSelfMatch2[1]; + } + } + + // Try Atom elements + if (!result.hub) { + const atomHubMatch = content.match( + /]+rel=["']?hub["']?[^>]+href=["']([^"']+)["']/i, + ); + if (atomHubMatch) { + result.hub = atomHubMatch[1]; + } + } + + return result; +} + +/** + * Check if a hub URL is valid + * @param {string} hubUrl - Hub URL to validate + * @returns {boolean} Whether the URL is valid + */ +export function isValidHubUrl(hubUrl) { + try { + const url = new URL(hubUrl); + return url.protocol === "https:" || url.protocol === "http:"; + } catch { + return false; + } +} diff --git a/lib/websub/handler.js b/lib/websub/handler.js new file mode 100644 index 0000000..de512eb --- /dev/null +++ b/lib/websub/handler.js @@ -0,0 +1,163 @@ +/** + * WebSub callback handler + * @module websub/handler + */ + +import { parseFeed } from "../feeds/parser.js"; +import { processFeed } from "../polling/processor.js"; +import { getFeedBySubscriptionId, updateFeedWebsub } from "../storage/feeds.js"; + +import { verifySignature } from "./subscriber.js"; + +/** + * Verify WebSub subscription + * GET /microsub/websub/:id + * @param {object} request - Express request + * @param {object} response - Express response + */ +export async function verify(request, response) { + const { id } = request.params; + const { + "hub.topic": topic, + "hub.challenge": challenge, + "hub.lease_seconds": leaseSeconds, + } = request.query; + + if (!challenge) { + return response.status(400).send("Missing hub.challenge"); + } + + const { application } = request.app.locals; + const feed = await getFeedBySubscriptionId(application, id); + + if (!feed) { + return response.status(404).send("Subscription not found"); + } + + // Verify topic matches (allow both feed URL and topic URL) + const expectedTopic = feed.websub?.topic || feed.url; + if (topic !== feed.url && topic !== expectedTopic) { + return response.status(400).send("Topic mismatch"); + } + + // Update lease seconds if provided + if (leaseSeconds) { + const seconds = Number.parseInt(leaseSeconds, 10); + if (seconds > 0) { + await updateFeedWebsub(application, id, { + hub: feed.websub?.hub, + topic: expectedTopic, + leaseSeconds: seconds, + secret: feed.websub?.secret, + }); + } + } + + // Mark subscription as active (not pending) + if (feed.websub?.pending) { + await updateFeedWebsub(application, id, { + hub: feed.websub?.hub, + topic: expectedTopic, + secret: feed.websub?.secret, + leaseSeconds: feed.websub?.leaseSeconds, + pending: false, + }); + } + + console.log(`[Microsub] WebSub subscription verified for ${feed.url}`); + + // Return challenge to verify subscription + response.type("text/plain").send(challenge); +} + +/** + * Receive WebSub notification + * POST /microsub/websub/:id + * @param {object} request - Express request + * @param {object} response - Express response + */ +export async function receive(request, response) { + const { id } = request.params; + const { application } = request.app.locals; + + const feed = await getFeedBySubscriptionId(application, id); + if (!feed) { + return response.status(404).send("Subscription not found"); + } + + // Verify X-Hub-Signature if we have a secret + if (feed.websub?.secret) { + const signature = + request.headers["x-hub-signature-256"] || + request.headers["x-hub-signature"]; + + if (!signature) { + return response.status(401).send("Missing signature"); + } + + // Get raw body for signature verification + const rawBody = + typeof request.body === "string" + ? request.body + : JSON.stringify(request.body); + + if (!verifySignature(signature, rawBody, feed.websub.secret)) { + console.warn(`[Microsub] Invalid WebSub signature for ${feed.url}`); + return response.status(401).send("Invalid signature"); + } + } + + // Acknowledge receipt immediately + response.status(200).send("OK"); + + // Process pushed content in background + setImmediate(async () => { + try { + await processWebsubContent( + application, + feed, + request.headers["content-type"], + request.body, + ); + } catch (error) { + console.error( + `[Microsub] Error processing WebSub content for ${feed.url}: ${error.message}`, + ); + } + }); +} + +/** + * Process WebSub pushed content + * @param {object} application - Indiekit application + * @param {object} feed - Feed document + * @param {string} contentType - Content-Type header + * @param {string|object} body - Request body + * @returns {Promise} + */ +async function processWebsubContent(application, feed, contentType, body) { + // Convert body to string if needed + const content = typeof body === "string" ? body : JSON.stringify(body); + + try { + // Parse the pushed content + const parsed = await parseFeed(content, feed.url, { contentType }); + + console.log( + `[Microsub] Processing ${parsed.items.length} items from WebSub push for ${feed.url}`, + ); + + // Process like a normal feed fetch but with pre-parsed content + // This reuses the existing feed processing logic + await processFeed(application, { + ...feed, + _websubContent: parsed, + }); + } catch (error) { + console.error( + `[Microsub] Failed to parse WebSub content for ${feed.url}: ${error.message}`, + ); + } +} + +export const websubHandler = { verify, receive }; diff --git a/lib/websub/subscriber.js b/lib/websub/subscriber.js new file mode 100644 index 0000000..edea4ae --- /dev/null +++ b/lib/websub/subscriber.js @@ -0,0 +1,181 @@ +/** + * WebSub subscriber + * @module websub/subscriber + */ + +import crypto from "node:crypto"; + +import { updateFeedWebsub } from "../storage/feeds.js"; + +const DEFAULT_LEASE_SECONDS = 86_400 * 7; // 7 days + +/** + * Subscribe to a WebSub hub + * @param {object} application - Indiekit application + * @param {object} feed - Feed document with websub.hub + * @param {string} callbackUrl - Callback URL for this subscription + * @returns {Promise} Whether subscription was initiated + */ +export async function subscribe(application, feed, callbackUrl) { + if (!feed.websub?.hub) { + return false; + } + + const topic = feed.websub.topic || feed.url; + const secret = generateSecret(); + + try { + const response = await fetch(feed.websub.hub, { + method: "POST", + headers: { + "Content-Type": "application/x-www-form-urlencoded", + }, + body: new URLSearchParams({ + "hub.mode": "subscribe", + "hub.topic": topic, + "hub.callback": callbackUrl, + "hub.secret": secret, + "hub.lease_seconds": String(DEFAULT_LEASE_SECONDS), + }), + }); + + // 202 Accepted means subscription is pending verification + // 204 No Content means subscription was immediately accepted + if (response.status === 202 || response.status === 204) { + // Store the secret for signature verification + await updateFeedWebsub(application, feed._id, { + hub: feed.websub.hub, + topic, + secret, + pending: true, + }); + return true; + } + + console.error( + `[Microsub] WebSub subscription failed: ${response.status} ${response.statusText}`, + ); + return false; + } catch (error) { + console.error(`[Microsub] WebSub subscription error: ${error.message}`); + return false; + } +} + +/** + * Unsubscribe from a WebSub hub + * @param {object} application - Indiekit application + * @param {object} feed - Feed document with websub.hub + * @param {string} callbackUrl - Callback URL for this subscription + * @returns {Promise} Whether unsubscription was initiated + */ +export async function unsubscribe(application, feed, callbackUrl) { + if (!feed.websub?.hub) { + return false; + } + + const topic = feed.websub.topic || feed.url; + + try { + const response = await fetch(feed.websub.hub, { + method: "POST", + headers: { + "Content-Type": "application/x-www-form-urlencoded", + }, + body: new URLSearchParams({ + "hub.mode": "unsubscribe", + "hub.topic": topic, + "hub.callback": callbackUrl, + }), + }); + + if (response.status === 202 || response.status === 204) { + // Clear WebSub data from feed + await updateFeedWebsub(application, feed._id, { + hub: feed.websub.hub, + topic, + secret: undefined, + leaseSeconds: undefined, + pending: false, + }); + return true; + } + + return false; + } catch (error) { + console.error(`[Microsub] WebSub unsubscribe error: ${error.message}`); + return false; + } +} + +/** + * Generate a random secret for signature verification + * @returns {string} Random hex string + */ +function generateSecret() { + return crypto.randomBytes(32).toString("hex"); +} + +/** + * Verify WebSub signature + * @param {string} signature - X-Hub-Signature header value + * @param {Buffer|string} body - Request body + * @param {string} secret - Subscription secret + * @returns {boolean} Whether signature is valid + */ +export function verifySignature(signature, body, secret) { + if (!signature || !secret) { + return false; + } + + // Signature format: sha1= or sha256= + const [algorithm, hash] = signature.split("="); + if (!algorithm || !hash) { + return false; + } + + // Normalize algorithm name + const algo = algorithm.toLowerCase().replace("sha", "sha"); + + try { + const expectedHash = crypto + .createHmac(algo, secret) + .update(body) + .digest("hex"); + + // Use timing-safe comparison + return crypto.timingSafeEqual( + Buffer.from(hash, "hex"), + Buffer.from(expectedHash, "hex"), + ); + } catch { + return false; + } +} + +/** + * Check if a WebSub subscription is about to expire + * @param {object} feed - Feed document + * @param {number} [thresholdSeconds] - Seconds before expiry to consider "expiring" + * @returns {boolean} Whether subscription is expiring soon + */ +export function isSubscriptionExpiring(feed, thresholdSeconds = 86_400) { + if (!feed.websub?.expiresAt) { + return false; + } + + const expiresAt = new Date(feed.websub.expiresAt); + const threshold = new Date(Date.now() + thresholdSeconds * 1000); + + return expiresAt <= threshold; +} + +/** + * Get callback URL for a feed + * @param {string} baseUrl - Base URL of the Microsub endpoint + * @param {string} feedId - Feed ID + * @returns {string} Callback URL + */ +export function getCallbackUrl(baseUrl, feedId) { + return `${baseUrl}/microsub/websub/${feedId}`; +} diff --git a/locales/en.json b/locales/en.json index 9d1c0ed..488f369 100644 --- a/locales/en.json +++ b/locales/en.json @@ -1,14 +1,88 @@ { "microsub": { - "title": "Microsub", + "reader": { + "title": "Reader", + "empty": "No items to display", + "markAllRead": "Mark all as read", + "newer": "Newer", + "older": "Older" + }, "channels": { - "title": "Channels" + "title": "Channels", + "name": "Channel name", + "new": "New channel", + "create": "Create channel", + "delete": "Delete channel", + "settings": "Channel settings", + "empty": "No channels yet. Create one to get started.", + "notifications": "Notifications" }, "timeline": { - "title": "Timeline" + "title": "Timeline", + "empty": "No items in this channel", + "markRead": "Mark as read", + "markUnread": "Mark as unread", + "remove": "Remove" + }, + "feeds": { + "title": "Feeds", + "follow": "Follow", + "subscribe": "Subscribe to a feed", + "unfollow": "Unfollow", + "empty": "No feeds followed in this channel", + "url": "Feed URL", + "urlPlaceholder": "https://example.com/feed.xml" + }, + "item": { + "reply": "Reply", + "like": "Like", + "repost": "Repost", + "bookmark": "Bookmark", + "viewOriginal": "View original" + }, + "compose": { + "title": "Compose", + "content": "What's on your mind?", + "submit": "Post", + "cancel": "Cancel", + "replyTo": "Replying to", + "likeOf": "Liking", + "repostOf": "Reposting", + "bookmarkOf": "Bookmarking" + }, + "settings": { + "title": "{{channel}} settings", + "excludeTypes": "Exclude interaction types", + "excludeTypesHelp": "Select types of posts to hide from this channel", + "excludeRegex": "Exclude pattern", + "excludeRegexHelp": "Regular expression to filter out matching content", + "save": "Save settings", + "dangerZone": "Danger zone", + "deleteWarning": "Deleting this channel will permanently remove all feeds and items. This action cannot be undone.", + "deleteConfirm": "Are you sure you want to delete this channel and all its content?", + "delete": "Delete channel", + "types": { + "like": "Likes", + "repost": "Reposts", + "bookmark": "Bookmarks", + "reply": "Replies", + "checkin": "Check-ins" + } + }, + "search": { + "title": "Search", + "placeholder": "Enter URL or search term", + "submit": "Search", + "noResults": "No results found" + }, + "preview": { + "title": "Preview", + "subscribe": "Subscribe to this feed" }, "error": { "channelNotFound": "Channel not found", + "feedNotFound": "Feed not found", + "invalidUrl": "Invalid URL", "invalidAction": "Invalid action" } } diff --git a/package.json b/package.json index ad17fcf..881ec56 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@rmdes/indiekit-endpoint-microsub", - "version": "1.0.12", + "version": "1.0.13", "description": "Microsub endpoint for Indiekit. Enables subscribing to feeds and reading content using the Microsub protocol.", "keywords": [ "indiekit", @@ -15,12 +15,6 @@ "name": "Ricardo Mendes", "url": "https://rmendes.net" }, - "contributors": [ - { - "name": "Paul Robert Lloyd", - "url": "https://paulrobertlloyd.com" - } - ], "license": "MIT", "engines": { "node": ">=20" @@ -28,8 +22,10 @@ "type": "module", "main": "index.js", "files": [ + "assets", "lib", "locales", + "views", "index.js" ], "bugs": { @@ -37,12 +33,20 @@ }, "repository": { "type": "git", - "url": "git+https://github.com/rmdes/indiekit-endpoint-microsub.git" + "url": "https://github.com/rmdes/indiekit-endpoint-microsub.git" }, "dependencies": { "@indiekit/error": "^1.0.0-beta.25", + "@indiekit/frontend": "^1.0.0-beta.25", + "@indiekit/util": "^1.0.0-beta.25", + "debug": "^4.3.2", "express": "^5.0.0", - "mongodb": "^6.0.0" + "feedparser": "^2.2.10", + "htmlparser2": "^9.0.0", + "ioredis": "^5.3.0", + "luxon": "^3.4.0", + "microformats-parser": "^2.0.0", + "sanitize-html": "^2.11.0" }, "publishConfig": { "access": "public" diff --git a/views/404.njk b/views/404.njk new file mode 100644 index 0000000..8e401c4 --- /dev/null +++ b/views/404.njk @@ -0,0 +1,17 @@ +{% extends "document.njk" %} + +{% block main %} + +{% endblock %} diff --git a/views/channel-new.njk b/views/channel-new.njk new file mode 100644 index 0000000..d6c8193 --- /dev/null +++ b/views/channel-new.njk @@ -0,0 +1,31 @@ +{% extends "layouts/reader.njk" %} + +{% block reader %} +
+ + {{ icon("previous") }} {{ __("microsub.channels.title") }} + + +

{{ __("microsub.channels.new") }}

+ +
+ {{ input({ + id: "name", + name: "name", + label: __("microsub.channels.name"), + required: true, + autocomplete: "off", + attributes: { autofocus: true } + }) }} + +
+ {{ button({ + text: __("microsub.channels.create") + }) }} + + {{ __("Cancel") }} + +
+
+
+{% endblock %} diff --git a/views/channel.njk b/views/channel.njk new file mode 100644 index 0000000..79e12a8 --- /dev/null +++ b/views/channel.njk @@ -0,0 +1,102 @@ +{% extends "layouts/reader.njk" %} + +{% block reader %} +
+
+ + {{ icon("previous") }} {{ __("microsub.channels.title") }} + +
+
+ + + +
+ + {{ icon("syndicate") }} {{ __("microsub.feeds.title") }} + + + {{ icon("updatePost") }} {{ __("microsub.channels.settings") }} + +
+
+ + {% if items.length > 0 %} +
+ {% for item in items %} + {% include "partials/item-card.njk" %} + {% endfor %} +
+ + {% if paging %} + + {% endif %} + {% else %} +
+ {{ icon("syndicate") }} +

{{ __("microsub.timeline.empty") }}

+ + {{ __("microsub.feeds.subscribe") }} + +
+ {% endif %} +
+ + +{% endblock %} diff --git a/views/compose.njk b/views/compose.njk new file mode 100644 index 0000000..f1386bb --- /dev/null +++ b/views/compose.njk @@ -0,0 +1,98 @@ +{% extends "layouts/reader.njk" %} + +{% block reader %} +
+ + {{ icon("previous") }} {{ __("Back") }} + + +

{{ __("microsub.compose.title") }}

+ + {% if replyTo and replyTo is string %} +
+ {{ icon("reply") }} {{ __("microsub.compose.replyTo") }}: + + {{ replyTo | replace("https://", "") | replace("http://", "") }} + +
+ {% endif %} + + {% if likeOf and likeOf is string %} +
+ {{ icon("like") }} {{ __("microsub.compose.likeOf") }}: + + {{ likeOf | replace("https://", "") | replace("http://", "") }} + +
+ {% endif %} + + {% if repostOf and repostOf is string %} +
+ {{ icon("repost") }} {{ __("microsub.compose.repostOf") }}: + + {{ repostOf | replace("https://", "") | replace("http://", "") }} + +
+ {% endif %} + + {% if bookmarkOf and bookmarkOf is string %} +
+ {{ icon("bookmark") }} {{ __("microsub.compose.bookmarkOf") }}: + + {{ bookmarkOf | replace("https://", "") | replace("http://", "") }} + +
+ {% endif %} + +
+ {% if replyTo %} + + {% endif %} + {% if likeOf %} + + {% endif %} + {% if repostOf %} + + {% endif %} + {% if bookmarkOf %} + + {% endif %} + + {% set isAction = likeOf or repostOf or bookmarkOf %} + + {% if not isAction %} + {{ textarea({ + label: __("microsub.compose.content"), + id: "content", + name: "content", + rows: 5, + attributes: { autofocus: true } + }) }} +
+ 0 characters +
+ {% endif %} + +
+ {{ button({ + text: __("microsub.compose.submit") + }) }} + + {{ __("microsub.compose.cancel") }} + +
+
+
+ + {% if not isAction %} + + {% endif %} +{% endblock %} diff --git a/views/feeds.njk b/views/feeds.njk new file mode 100644 index 0000000..0861a51 --- /dev/null +++ b/views/feeds.njk @@ -0,0 +1,69 @@ +{% extends "layouts/reader.njk" %} + +{% block reader %} +
+
+ + {{ icon("previous") }} {{ channel.name }} + +
+ +

{{ __("microsub.feeds.title") }}

+ + {% if feeds.length > 0 %} +
+ {% for feed in feeds %} +
+
+ {% if feed.photo %} + + {% endif %} +
+ {{ feed.title or feed.url }} + + {{ feed.url | replace("https://", "") | replace("http://", "") }} + +
+
+
+ + {{ button({ + text: __("microsub.feeds.unfollow"), + classes: "button--secondary button--small" + }) }} +
+
+ {% endfor %} +
+ {% else %} +
+ {{ icon("syndicate") }} +

{{ __("microsub.feeds.empty") }}

+
+ {% endif %} + +
+

{{ __("microsub.feeds.follow") }}

+
+ {{ input({ + id: "url", + name: "url", + label: __("microsub.feeds.url"), + type: "url", + required: true, + placeholder: __("microsub.feeds.urlPlaceholder"), + autocomplete: "off" + }) }} +
+ {{ button({ text: __("microsub.feeds.follow") }) }} +
+
+
+
+{% endblock %} diff --git a/views/item.njk b/views/item.njk new file mode 100644 index 0000000..0d738b6 --- /dev/null +++ b/views/item.njk @@ -0,0 +1,151 @@ +{% extends "layouts/reader.njk" %} + +{% block reader %} +
+ + {{ icon("previous") }} {{ __("Back") }} + + + {% if item.author %} +
+ {% if item.author.photo %} + + {% endif %} +
+ + {% if item.author.url %} + {{ item.author.name or item.author.url }} + {% else %} + {{ item.author.name or "Unknown" }} + {% endif %} + + {% if item.published %} + + {% endif %} +
+
+ {% endif %} + + {# Context for interactions #} + {% if item["in-reply-to"] or item["like-of"] or item["repost-of"] or item["bookmark-of"] %} +
+ {% if item["in-reply-to"] and item["in-reply-to"].length > 0 %} +

+ {{ icon("reply") }} {{ __("Reply to") }}: + + {{ item["in-reply-to"][0] | replace("https://", "") | replace("http://", "") }} + +

+ {% endif %} + {% if item["like-of"] and item["like-of"].length > 0 %} +

+ {{ icon("like") }} {{ __("Liked") }}: + + {{ item["like-of"][0] | replace("https://", "") | replace("http://", "") }} + +

+ {% endif %} + {% if item["repost-of"] and item["repost-of"].length > 0 %} +

+ {{ icon("repost") }} {{ __("Reposted") }}: + + {{ item["repost-of"][0] | replace("https://", "") | replace("http://", "") }} + +

+ {% endif %} + {% if item["bookmark-of"] and item["bookmark-of"].length > 0 %} +

+ {{ icon("bookmark") }} {{ __("Bookmarked") }}: + + {{ item["bookmark-of"][0] | replace("https://", "") | replace("http://", "") }} + +

+ {% endif %} +
+ {% endif %} + + {% if item.name %} +

{{ item.name }}

+ {% endif %} + + {% if item.content %} +
+ {% if item.content.html %} + {{ item.content.html | safe }} + {% else %} + {{ item.content.text }} + {% endif %} +
+ {% endif %} + + {# Categories #} + {% if item.category and item.category.length > 0 %} +
+ {% for cat in item.category %} + #{{ cat | replace("#", "") }} + {% endfor %} +
+ {% endif %} + + {# Photos #} + {% if item.photo and item.photo.length > 0 %} +
+ {% for photo in item.photo %} + + + + {% endfor %} +
+ {% endif %} + + {# Video #} + {% if item.video and item.video.length > 0 %} +
+ {% for video in item.video %} + + {% endfor %} +
+ {% endif %} + + {# Audio #} + {% if item.audio and item.audio.length > 0 %} +
+ {% for audio in item.audio %} + + {% endfor %} +
+ {% endif %} + + +
+{% endblock %} diff --git a/views/layouts/reader.njk b/views/layouts/reader.njk new file mode 100644 index 0000000..7928557 --- /dev/null +++ b/views/layouts/reader.njk @@ -0,0 +1,10 @@ +{# + Microsub Reader Layout + Extends document.njk and adds reader-specific stylesheet +#} +{% extends "document.njk" %} + +{% block content %} + +{% block reader %}{% endblock %} +{% endblock %} diff --git a/views/partials/actions.njk b/views/partials/actions.njk new file mode 100644 index 0000000..d8b09f5 --- /dev/null +++ b/views/partials/actions.njk @@ -0,0 +1,15 @@ +{# Item action buttons #} +
+ + {{ icon("reply") }} + + + {{ icon("like") }} + + + {{ icon("repost") }} + + + {{ icon("public") }} + +
diff --git a/views/partials/author.njk b/views/partials/author.njk new file mode 100644 index 0000000..7741373 --- /dev/null +++ b/views/partials/author.njk @@ -0,0 +1,17 @@ +{# Author display #} +{% if author %} +
+ {% if author.photo %} + + {% endif %} +
+ + {% if author.url %} + {{ author.name or author.url }} + {% else %} + {{ author.name or "Unknown" }} + {% endif %} + +
+
+{% endif %} diff --git a/views/partials/item-card.njk b/views/partials/item-card.njk new file mode 100644 index 0000000..54e97b7 --- /dev/null +++ b/views/partials/item-card.njk @@ -0,0 +1,179 @@ +{# + Item card for timeline display + Inspired by Aperture/Monocle reader +#} +
+ + {# Context bar for interactions (Aperture pattern) #} + {# Helper to extract URL from value that may be string or object #} + {% macro getUrl(val) %}{{ val.url or val.value or val if val is string else val }}{% endmacro %} + + {% if item["like-of"] and item["like-of"].length > 0 %} + {% set contextUrl = item['like-of'][0].url or item['like-of'][0].value or item['like-of'][0] %} +
+ {{ icon("like") }} + Liked + + {{ contextUrl | replace("https://", "") | replace("http://", "") | truncate(50) }} + +
+ {% elif item["repost-of"] and item["repost-of"].length > 0 %} + {% set contextUrl = item['repost-of'][0].url or item['repost-of'][0].value or item['repost-of'][0] %} +
+ {{ icon("repost") }} + Reposted + + {{ contextUrl | replace("https://", "") | replace("http://", "") | truncate(50) }} + +
+ {% elif item["in-reply-to"] and item["in-reply-to"].length > 0 %} + {% set contextUrl = item['in-reply-to'][0].url or item['in-reply-to'][0].value or item['in-reply-to'][0] %} +
+ {{ icon("reply") }} + Reply to + + {{ contextUrl | replace("https://", "") | replace("http://", "") | truncate(50) }} + +
+ {% elif item["bookmark-of"] and item["bookmark-of"].length > 0 %} + {% set contextUrl = item['bookmark-of'][0].url or item['bookmark-of'][0].value or item['bookmark-of'][0] %} +
+ {{ icon("bookmark") }} + Bookmarked + + {{ contextUrl | replace("https://", "") | replace("http://", "") | truncate(50) }} + +
+ {% endif %} + + + {# Author #} + {% if item.author %} +
+ {% if item.author.photo %} + + {% endif %} +
+ {{ item.author.name or "Unknown" }} + {% if item._source %} + {{ item._source.name or item._source.url }} + {% elif item.author.url %} + {{ item.author.url | replace("https://", "") | replace("http://", "") }} + {% endif %} +
+
+ {% endif %} + + {# Title (for articles) #} + {% if item.name %} +

{{ item.name }}

+ {% endif %} + + {# Content with overflow handling #} + {% if item.summary or item.content %} +
+ {% if item.content.html %} + {{ item.content.html | safe | striptags | truncate(400) }} + {% elif item.content.text %} + {{ item.content.text | truncate(400) }} + {% elif item.summary %} + {{ item.summary | truncate(400) }} + {% endif %} +
+ {% endif %} + + {# Categories/Tags #} + {% if item.category and item.category.length > 0 %} +
+ {% for cat in item.category | slice(0, 5) %} + #{{ cat | replace("#", "") }} + {% endfor %} +
+ {% endif %} + + {# Photo grid (Aperture multi-photo pattern) #} + {% if item.photo and item.photo.length > 0 %} + {% set photoCount = item.photo.length if item.photo.length <= 4 else 4 %} +
+ {% for photo in item.photo | slice(0, 4) %} + + {% endfor %} +
+ {% endif %} + + {# Video preview #} + {% if item.video and item.video.length > 0 %} +
+ +
+ {% endif %} + + {# Audio preview #} + {% if item.audio and item.audio.length > 0 %} +
+ +
+ {% endif %} + + {# Footer with date and actions #} +
+ {% if item.published %} + + {% endif %} + {% if not item._is_read %} + ● + {% endif %} +
+ + + {# Inline actions (Aperture pattern) #} +
+ {% if item.url %} + + {{ icon("external") }} + + + {% endif %} + + {{ icon("reply") }} + + + + {{ icon("like") }} + + + + {{ icon("repost") }} + + + {% if not item._is_read %} + + {% endif %} +
+
diff --git a/views/partials/timeline.njk b/views/partials/timeline.njk new file mode 100644 index 0000000..f027364 --- /dev/null +++ b/views/partials/timeline.njk @@ -0,0 +1,10 @@ +{# Timeline of items #} +
+ {% if items.length > 0 %} + {% for item in items %} + {% include "partials/item-card.njk" %} + {% endfor %} + {% else %} + {{ prose({ text: __("microsub.timeline.empty") }) }} + {% endif %} +
diff --git a/views/reader.njk b/views/reader.njk new file mode 100644 index 0000000..4a71db8 --- /dev/null +++ b/views/reader.njk @@ -0,0 +1,41 @@ +{% extends "layouts/reader.njk" %} + +{% block reader %} +
+ {% if channels.length > 0 %} +
+ {% for channel in channels %} + + + {% if channel.uid === "notifications" %} + {{ icon("mention") }} + {% endif %} + {{ channel.name }} + + {% if channel.unread %} + + {% if channel.unread !== true %}{{ channel.unread }}{% endif %} + + {% endif %} + + {% endfor %} +
+
+ + {{ icon("syndicate") }} {{ __("microsub.feeds.follow") }} + + + {{ icon("createPost") }} {{ __("microsub.channels.new") }} + +
+ {% else %} +
+ {{ icon("syndicate") }} +

{{ __("microsub.channels.empty") }}

+ + {{ __("microsub.channels.new") }} + +
+ {% endif %} +
+{% endblock %} diff --git a/views/search.njk b/views/search.njk new file mode 100644 index 0000000..d03f896 --- /dev/null +++ b/views/search.njk @@ -0,0 +1,61 @@ +{% extends "layouts/reader.njk" %} + +{% block reader %} +
+ + {{ icon("previous") }} {{ __("microsub.channels.title") }} + + +

{{ __("microsub.search.title") }}

+ +
+ {{ input({ + id: "query", + name: "query", + label: __("microsub.search.placeholder"), + type: "url", + required: true, + placeholder: "https://example.com", + autocomplete: "off", + value: query, + attributes: { autofocus: true } + }) }} +
+ {{ button({ text: __("microsub.search.submit") }) }} +
+
+ + {% if results and results.length > 0 %} +
+

{{ __("microsub.search.title") }}

+
+ {% for result in results %} +
+
+ {{ result.title or "Feed" }} + {{ result.url | replace("https://", "") | replace("http://", "") }} +
+ +
+ {% endfor %} +
+
+ {% elif searched %} +
+

{{ __("microsub.search.noResults") }}

+
+ {% endif %} +
+{% endblock %} diff --git a/views/settings.njk b/views/settings.njk new file mode 100644 index 0000000..0c79a17 --- /dev/null +++ b/views/settings.njk @@ -0,0 +1,75 @@ +{% extends "layouts/reader.njk" %} + +{% block reader %} +
+ + {{ icon("previous") }} {{ channel.name }} + + +

{{ __("microsub.settings.title", { channel: channel.name }) }}

+ +
+ {{ checkboxes({ + name: "excludeTypes", + values: channel.settings.excludeTypes, + fieldset: { + legend: __("microsub.settings.excludeTypes") + }, + hint: __("microsub.settings.excludeTypesHelp"), + items: [ + { + label: __("microsub.settings.types.like"), + value: "like" + }, + { + label: __("microsub.settings.types.repost"), + value: "repost" + }, + { + label: __("microsub.settings.types.bookmark"), + value: "bookmark" + }, + { + label: __("microsub.settings.types.reply"), + value: "reply" + }, + { + label: __("microsub.settings.types.checkin"), + value: "checkin" + } + ] + }) }} + + {{ input({ + id: "excludeRegex", + name: "excludeRegex", + label: __("microsub.settings.excludeRegex"), + hint: __("microsub.settings.excludeRegexHelp"), + value: channel.settings.excludeRegex + }) }} + +
+ {{ button({ + text: __("microsub.settings.save") + }) }} + + {{ __("Cancel") }} + +
+
+ + {% if channel.uid !== "notifications" %} +
+
+

{{ __("microsub.settings.dangerZone") }}

+

{{ __("microsub.settings.deleteWarning") }}

+
+ {{ button({ + text: __("microsub.settings.delete"), + classes: "button--danger" + }) }} +
+
+ {% endif %} +
+{% endblock %}