Harden listening endpoint sync, stats, and runtime guards

2026-03-09 15:04:49 +01:00
parent fed5657957
commit f8aa318342
3 changed files with 258 additions and 1 deletions
--- a/.env.example
+++ b/.env.example
@@ -27,6 +27,17 @@ WEBMENTIONS_PROXY_MOUNT_PATH=/webmentions-api
 # Cache TTL in seconds for proxied webmention.io API responses
 WEBMENTIONS_PROXY_CACHE_TTL=60

+# Optional listening endpoint update cadence (milliseconds)
+# Lower values increase freshness but add upstream API load.
+LISTENING_CACHE_TTL=120000
+LISTENING_SYNC_INTERVAL=180000
+
+# Optional per-source listening overrides (milliseconds)
+FUNKWHALE_CACHE_TTL=
+FUNKWHALE_SYNC_INTERVAL=
+LASTFM_CACHE_TTL=
+LASTFM_SYNC_INTERVAL=
+
 # Syndication endpoint mount path
 # Default in indiekit.config.mjs is /syndicate
 SYNDICATE_MOUNT_PATH=/syndicate
@@ -38,6 +49,6 @@ BLUESKY_PASSWORD=

 # Mastodon syndicator settings
 # MASTODON_USER should be your username without @
-MASTODON_URL=https://mastodon.social
+MASTODON_URL=
 MASTODON_USER=
 MASTODON_ACCESS_TOKEN=
--- a/indiekit.config.mjs
+++ b/indiekit.config.mjs
@@ -38,6 +38,48 @@ const funkwhaleUsername = process.env.FUNKWHALE_USERNAME;
 const funkwhaleToken = process.env.FUNKWHALE_TOKEN;
 const lastfmApiKey = process.env.LASTFM_API_KEY;
 const lastfmUsername = process.env.LASTFM_USERNAME;
+const listeningCacheTtlRaw = Number.parseInt(
+  process.env.LISTENING_CACHE_TTL || "120000",
+  10,
+);
+const listeningCacheTtl = Number.isFinite(listeningCacheTtlRaw)
+  ? Math.max(30000, listeningCacheTtlRaw)
+  : 120000;
+const listeningSyncIntervalRaw = Number.parseInt(
+  process.env.LISTENING_SYNC_INTERVAL || "180000",
+  10,
+);
+const listeningSyncInterval = Number.isFinite(listeningSyncIntervalRaw)
+  ? Math.max(60000, listeningSyncIntervalRaw)
+  : 180000;
+const funkwhaleCacheTtlRaw = Number.parseInt(
+  process.env.FUNKWHALE_CACHE_TTL || String(listeningCacheTtl),
+  10,
+);
+const funkwhaleCacheTtl = Number.isFinite(funkwhaleCacheTtlRaw)
+  ? Math.max(30000, funkwhaleCacheTtlRaw)
+  : listeningCacheTtl;
+const funkwhaleSyncIntervalRaw = Number.parseInt(
+  process.env.FUNKWHALE_SYNC_INTERVAL || String(listeningSyncInterval),
+  10,
+);
+const funkwhaleSyncInterval = Number.isFinite(funkwhaleSyncIntervalRaw)
+  ? Math.max(60000, funkwhaleSyncIntervalRaw)
+  : listeningSyncInterval;
+const lastfmCacheTtlRaw = Number.parseInt(
+  process.env.LASTFM_CACHE_TTL || String(listeningCacheTtl),
+  10,
+);
+const lastfmCacheTtl = Number.isFinite(lastfmCacheTtlRaw)
+  ? Math.max(30000, lastfmCacheTtlRaw)
+  : listeningCacheTtl;
+const lastfmSyncIntervalRaw = Number.parseInt(
+  process.env.LASTFM_SYNC_INTERVAL || String(listeningSyncInterval),
+  10,
+);
+const lastfmSyncInterval = Number.isFinite(lastfmSyncIntervalRaw)
+  ? Math.max(60000, lastfmSyncIntervalRaw)
+  : listeningSyncInterval;
 const blueskyHandle = (process.env.BLUESKY_HANDLE || "")
  .trim()
  .replace(/^@+/, "");
@@ -281,11 +323,15 @@ export default {
    instanceUrl: funkwhaleInstance,
    username: funkwhaleUsername,
    token: funkwhaleToken,
+    cacheTtl: funkwhaleCacheTtl,
+    syncInterval: funkwhaleSyncInterval,
  },
  "@rmdes/indiekit-endpoint-lastfm": {
    mountPath: "/lastfmapi",
    apiKey: lastfmApiKey,
    username: lastfmUsername,
+    cacheTtl: lastfmCacheTtl,
+    syncInterval: lastfmSyncInterval,
  },
  "@rmdes/indiekit-endpoint-podroll": {
    mountPath: podrollMountPath,
--- a/scripts/patch-listening-endpoint-runtime-guards.mjs
+++ b/scripts/patch-listening-endpoint-runtime-guards.mjs
@@ -200,6 +200,206 @@ const patchSpecs = [
      "node_modules/@indiekit/indiekit/node_modules/@rmdes/indiekit-endpoint-funkwhale/lib/controllers/stats.js",
    ],
  },
+  {
+    name: "funkwhale-stats-db-getter",
+    marker: "use application database getter for public stats routes",
+    oldSnippet: `      // Try database first, fall back to cache for public routes
+      const db = request.app.locals.database;
+      let stats;`,
+    newSnippet: `      // Try database first, fall back to cache for public routes
+      // use application database getter for public stats routes
+      const db =
+        request.app.locals.application.getFunkwhaleDb?.() ||
+        request.app.locals.database;
+      let stats;`,
+    candidates: [
+      "node_modules/@rmdes/indiekit-endpoint-funkwhale/lib/controllers/stats.js",
+      "node_modules/@indiekit/indiekit/node_modules/@rmdes/indiekit-endpoint-funkwhale/lib/controllers/stats.js",
+    ],
+  },
+  {
+    name: "funkwhale-trends-db-getter",
+    marker: "use application database getter for public trends routes",
+    oldSnippet: `      const db = request.app.locals.database;
+      const days = Math.min(parseInt(request.query.days) || 30, 90);`,
+    newSnippet: `      // use application database getter for public trends routes
+      const db =
+        request.app.locals.application.getFunkwhaleDb?.() ||
+        request.app.locals.database;
+      const days = Math.min(parseInt(request.query.days) || 30, 90);`,
+    candidates: [
+      "node_modules/@rmdes/indiekit-endpoint-funkwhale/lib/controllers/stats.js",
+      "node_modules/@indiekit/indiekit/node_modules/@rmdes/indiekit-endpoint-funkwhale/lib/controllers/stats.js",
+    ],
+  },
+  {
+    name: "funkwhale-sync-date-storage",
+    marker: "store listenedAt/syncedAt as Date objects",
+    oldSnippet: `    listenedAt: new Date(listening.creation_date).toISOString(),
+    syncedAt: new Date().toISOString(),`,
+    newSnippet: `    // store listenedAt/syncedAt as Date objects
+    listenedAt: new Date(listening.creation_date),
+    syncedAt: new Date(),`,
+    candidates: [
+      "node_modules/@rmdes/indiekit-endpoint-funkwhale/lib/sync.js",
+      "node_modules/@indiekit/indiekit/node_modules/@rmdes/indiekit-endpoint-funkwhale/lib/sync.js",
+    ],
+  },
+  {
+    name: "funkwhale-stats-date-coercion",
+    marker: "support string and Date listenedAt values in period filters",
+    oldSnippet: `function getDateMatch(period) {
+  const now = new Date();
+  switch (period) {
+    case "week":
+      return { listenedAt: { $gte: new Date(now - 7 * 24 * 60 * 60 * 1000) } };
+    case "month":
+      return { listenedAt: { $gte: new Date(now - 30 * 24 * 60 * 60 * 1000) } };
+    default:
+      return {};
+  }
+}`,
+    newSnippet: `function getDateMatch(period) {
+  const now = new Date();
+  let threshold = null;
+
+  switch (period) {
+    case "week":
+      threshold = new Date(now - 7 * 24 * 60 * 60 * 1000);
+      break;
+    case "month":
+      threshold = new Date(now - 30 * 24 * 60 * 60 * 1000);
+      break;
+    default:
+      return {};
+  }
+
+  // support string and Date listenedAt values in period filters
+  return {
+    $expr: {
+      $gte: [{ $toDate: "$listenedAt" }, threshold],
+    },
+  };
+}`,
+    candidates: [
+      "node_modules/@rmdes/indiekit-endpoint-funkwhale/lib/stats.js",
+      "node_modules/@indiekit/indiekit/node_modules/@rmdes/indiekit-endpoint-funkwhale/lib/stats.js",
+    ],
+  },
+  {
+    name: "funkwhale-trends-date-coercion",
+    marker: "support string and Date listenedAt values in trends aggregation",
+    oldSnippet: `  return collection
+    .aggregate([
+      { $match: { listenedAt: { $gte: startDate } } },
+      {
+        $group: {
+          _id: {
+            $dateToString: { format: "%Y-%m-%d", date: "$listenedAt" },
+          },`,
+    newSnippet: `  return collection
+    .aggregate([
+      {
+        // support string and Date listenedAt values in trends aggregation
+        $addFields: {
+          listenedAtDate: { $toDate: "$listenedAt" },
+        },
+      },
+      { $match: { listenedAtDate: { $gte: startDate } } },
+      {
+        $group: {
+          _id: {
+            $dateToString: { format: "%Y-%m-%d", date: "$listenedAtDate" },
+          },`,
+    candidates: [
+      "node_modules/@rmdes/indiekit-endpoint-funkwhale/lib/stats.js",
+      "node_modules/@indiekit/indiekit/node_modules/@rmdes/indiekit-endpoint-funkwhale/lib/stats.js",
+    ],
+  },
+  {
+    name: "lastfm-sync-date-storage",
+    marker: "store scrobbledAt/syncedAt as Date objects",
+    oldSnippet: `    scrobbledAt: scrobbledAtDate.toISOString(),
+    syncedAt: new Date().toISOString(),`,
+    newSnippet: `    // store scrobbledAt/syncedAt as Date objects
+    scrobbledAt: scrobbledAtDate,
+    syncedAt: new Date(),`,
+    candidates: [
+      "node_modules/@rmdes/indiekit-endpoint-lastfm/lib/sync.js",
+      "node_modules/@indiekit/indiekit/node_modules/@rmdes/indiekit-endpoint-lastfm/lib/sync.js",
+    ],
+  },
+  {
+    name: "lastfm-stats-date-coercion",
+    marker: "support string and Date scrobbledAt values in period filters",
+    oldSnippet: `function getDateMatch(period) {
+  const now = new Date();
+  switch (period) {
+    case "week":
+      return { scrobbledAt: { $gte: new Date(now - 7 * 24 * 60 * 60 * 1000) } };
+    case "month":
+      return { scrobbledAt: { $gte: new Date(now - 30 * 24 * 60 * 60 * 1000) } };
+    default:
+      return {};
+  }
+}`,
+    newSnippet: `function getDateMatch(period) {
+  const now = new Date();
+  let threshold = null;
+
+  switch (period) {
+    case "week":
+      threshold = new Date(now - 7 * 24 * 60 * 60 * 1000);
+      break;
+    case "month":
+      threshold = new Date(now - 30 * 24 * 60 * 60 * 1000);
+      break;
+    default:
+      return {};
+  }
+
+  // support string and Date scrobbledAt values in period filters
+  return {
+    $expr: {
+      $gte: [{ $toDate: "$scrobbledAt" }, threshold],
+    },
+  };
+}`,
+    candidates: [
+      "node_modules/@rmdes/indiekit-endpoint-lastfm/lib/stats.js",
+      "node_modules/@indiekit/indiekit/node_modules/@rmdes/indiekit-endpoint-lastfm/lib/stats.js",
+    ],
+  },
+  {
+    name: "lastfm-trends-date-coercion",
+    marker: "support string and Date scrobbledAt values in trends aggregation",
+    oldSnippet: `  return collection
+    .aggregate([
+      { $match: { scrobbledAt: { $gte: startDate } } },
+      {
+        $group: {
+          _id: {
+            $dateToString: { format: "%Y-%m-%d", date: "$scrobbledAt" },
+          },`,
+    newSnippet: `  return collection
+    .aggregate([
+      {
+        // support string and Date scrobbledAt values in trends aggregation
+        $addFields: {
+          scrobbledAtDate: { $toDate: "$scrobbledAt" },
+        },
+      },
+      { $match: { scrobbledAtDate: { $gte: startDate } } },
+      {
+        $group: {
+          _id: {
+            $dateToString: { format: "%Y-%m-%d", date: "$scrobbledAtDate" },
+          },`,
+    candidates: [
+      "node_modules/@rmdes/indiekit-endpoint-lastfm/lib/stats.js",
+      "node_modules/@indiekit/indiekit/node_modules/@rmdes/indiekit-endpoint-lastfm/lib/stats.js",
+    ],
+  },
 ];

 async function exists(filePath) {