fix: only sync new YouTube likes, not existing ones

First sync snapshots all current liked video IDs into a
youtubeLikesSeen collection without creating posts. Subsequent
syncs only create posts for likes not in the seen set.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
svemagie
2026-03-18 21:38:51 +01:00
parent 72cc2238f4
commit 252a228653

View File

@@ -1,13 +1,14 @@
/** /**
* YouTube Likes → Indiekit "like" posts sync * YouTube Likes → Indiekit "like" posts sync
* *
* Fetches the authenticated user's liked videos and creates * On first run after connecting, snapshots all current liked video IDs
* corresponding "like" posts via the Micropub posts collection. * as "known" without creating posts. Subsequent syncs only create posts
* for newly liked videos (ones not in the known set and not already
* in the posts collection).
*/ */
import { YouTubeClient } from "./youtube-client.js"; import { YouTubeClient } from "./youtube-client.js";
import { getValidAccessToken } from "./oauth.js"; import { getValidAccessToken } from "./oauth.js";
import crypto from "node:crypto";
/** /**
* Generate a deterministic slug from a YouTube video ID. * Generate a deterministic slug from a YouTube video ID.
@@ -18,11 +19,54 @@ function slugFromVideoId(videoId) {
return `yt-like-${videoId}`; return `yt-like-${videoId}`;
} }
/**
* Fetch all current liked video IDs (up to maxPages) and store them
* as the baseline. No posts are created.
* @returns {Promise<number>} number of IDs snapshotted
*/
async function snapshotExistingLikes(db, client, accessToken, maxPages) {
const collection = db.collection("youtubeLikesSeen");
await collection.createIndex({ videoId: 1 }, { unique: true });
let pageToken;
let count = 0;
for (let page = 0; page < maxPages; page++) {
const result = await client.getLikedVideos(accessToken, 50, pageToken);
const ops = result.videos.map((v) => ({
updateOne: {
filter: { videoId: v.id },
update: { $setOnInsert: { videoId: v.id, seenAt: new Date() } },
upsert: true,
},
}));
if (ops.length > 0) {
await collection.bulkWrite(ops, { ordered: false });
count += ops.length;
}
pageToken = result.nextPageToken;
if (!pageToken) break;
}
// Mark that the baseline snapshot is done
await db.collection("youtubeMeta").updateOne(
{ key: "likes_baseline" },
{ $set: { key: "likes_baseline", completedAt: new Date(), count } },
{ upsert: true },
);
return count;
}
/** /**
* Sync liked videos into the Indiekit posts collection. * Sync liked videos into the Indiekit posts collection.
* *
* For each liked video that doesn't already have a corresponding post * First-run behaviour: snapshots all existing likes as "seen" without
* in the database we insert a new "like" post document. * creating posts. Only likes that appear after the baseline are turned
* into posts.
* *
* @param {object} opts * @param {object} opts
* @param {import("mongodb").Db} opts.db * @param {import("mongodb").Db} opts.db
@@ -30,7 +74,7 @@ function slugFromVideoId(videoId) {
* @param {object} opts.publication - Indiekit publication config * @param {object} opts.publication - Indiekit publication config
* @param {import("mongodb").Collection} [opts.postsCollection] * @param {import("mongodb").Collection} [opts.postsCollection]
* @param {number} [opts.maxPages=3] - max pages to fetch (50 likes/page) * @param {number} [opts.maxPages=3] - max pages to fetch (50 likes/page)
* @returns {Promise<{synced: number, skipped: number, total: number, error?: string}>} * @returns {Promise<{synced: number, skipped: number, total: number, baselined?: number, error?: string}>}
*/ */
export async function syncLikes({ db, youtubeConfig, publication, postsCollection, maxPages = 3 }) { export async function syncLikes({ db, youtubeConfig, publication, postsCollection, maxPages = 3 }) {
const { apiKey, oauth } = youtubeConfig; const { apiKey, oauth } = youtubeConfig;
@@ -38,7 +82,6 @@ export async function syncLikes({ db, youtubeConfig, publication, postsCollectio
return { synced: 0, skipped: 0, total: 0, error: "OAuth not configured" }; return { synced: 0, skipped: 0, total: 0, error: "OAuth not configured" };
} }
// Get a valid access token (autorefreshes if needed)
let accessToken; let accessToken;
try { try {
accessToken = await getValidAccessToken(db, { accessToken = await getValidAccessToken(db, {
@@ -55,6 +98,17 @@ export async function syncLikes({ db, youtubeConfig, publication, postsCollectio
const client = new YouTubeClient({ apiKey: apiKey || "unused", cacheTtl: 0 }); const client = new YouTubeClient({ apiKey: apiKey || "unused", cacheTtl: 0 });
// --- First-run: snapshot existing likes, create zero posts ---
const baseline = await db.collection("youtubeMeta").findOne({ key: "likes_baseline" });
if (!baseline) {
console.log("[YouTube] First sync — snapshotting existing likes (no posts will be created)");
const count = await snapshotExistingLikes(db, client, accessToken, maxPages);
console.log(`[YouTube] Baselined ${count} existing liked videos`);
return { synced: 0, skipped: 0, total: count, baselined: count };
}
// --- Normal sync: only create posts for new likes ---
const seenCollection = db.collection("youtubeLikesSeen");
let synced = 0; let synced = 0;
let skipped = 0; let skipped = 0;
let total = 0; let total = 0;
@@ -68,10 +122,26 @@ export async function syncLikes({ db, youtubeConfig, publication, postsCollectio
total = result.totalResults; total = result.totalResults;
for (const video of result.videos) { for (const video of result.videos) {
const slug = slugFromVideoId(video.id); const videoId = video.id;
const videoUrl = `https://www.youtube.com/watch?v=${video.id}`;
// Check if we already synced this like // Already seen? Skip.
const seen = await seenCollection.findOne({ videoId });
if (seen) {
skipped++;
continue;
}
// Mark as seen immediately (even if post insert fails, don't retry)
await seenCollection.updateOne(
{ videoId },
{ $setOnInsert: { videoId, seenAt: new Date() } },
{ upsert: true },
);
const slug = slugFromVideoId(videoId);
const videoUrl = `https://www.youtube.com/watch?v=${videoId}`;
// Also skip if a post already exists (belt-and-suspenders)
if (postsCollection) { if (postsCollection) {
const existing = await postsCollection.findOne({ const existing = await postsCollection.findOne({
$or: [ $or: [
@@ -85,7 +155,6 @@ export async function syncLikes({ db, youtubeConfig, publication, postsCollectio
} }
} }
// Build the post path/url from the publication's like post type config
const postPath = likePostType?.post?.path const postPath = likePostType?.post?.path
? likePostType.post.path.replace("{slug}", slug) ? likePostType.post.path.replace("{slug}", slug)
: `content/likes/${slug}.md`; : `content/likes/${slug}.md`;
@@ -105,11 +174,11 @@ export async function syncLikes({ db, youtubeConfig, publication, postsCollectio
text: `Liked "${video.title}" by ${video.channelTitle} on YouTube`, text: `Liked "${video.title}" by ${video.channelTitle} on YouTube`,
html: `Liked "<a href="${videoUrl}">${escapeHtml(video.title)}</a>" by ${escapeHtml(video.channelTitle)} on YouTube`, html: `Liked "<a href="${videoUrl}">${escapeHtml(video.title)}</a>" by ${escapeHtml(video.channelTitle)} on YouTube`,
}, },
published: video.publishedAt || new Date().toISOString(), published: new Date().toISOString(),
url: postUrl, url: postUrl,
visibility: "public", visibility: "public",
"post-status": "published", "post-status": "published",
"youtube-video-id": video.id, "youtube-video-id": videoId,
"youtube-channel": video.channelTitle, "youtube-channel": video.channelTitle,
"youtube-thumbnail": video.thumbnail || "", "youtube-thumbnail": video.thumbnail || "",
}, },
@@ -125,7 +194,6 @@ export async function syncLikes({ db, youtubeConfig, publication, postsCollectio
if (!pageToken) break; if (!pageToken) break;
} }
// Update sync metadata
await db.collection("youtubeMeta").updateOne( await db.collection("youtubeMeta").updateOne(
{ key: "likes_sync" }, { key: "likes_sync" },
{ {
@@ -158,7 +226,7 @@ export async function getLastSyncStatus(db) {
* @param {object} options - endpoint options * @param {object} options - endpoint options
*/ */
export function startLikesSync(Indiekit, options) { export function startLikesSync(Indiekit, options) {
const interval = options.likes?.syncInterval || 3_600_000; // default 1 hour const interval = options.likes?.syncInterval || 3_600_000;
async function run() { async function run() {
const db = Indiekit.database; const db = Indiekit.database;
@@ -178,15 +246,16 @@ export function startLikesSync(Indiekit, options) {
if (result.synced > 0) { if (result.synced > 0) {
console.log(`[YouTube] Likes sync: ${result.synced} new, ${result.skipped} skipped`); console.log(`[YouTube] Likes sync: ${result.synced} new, ${result.skipped} skipped`);
} }
if (result.baselined) {
console.log(`[YouTube] Baseline complete: ${result.baselined} existing likes recorded`);
}
} catch (err) { } catch (err) {
console.error("[YouTube] Likes sync error:", err.message); console.error("[YouTube] Likes sync error:", err.message);
} }
} }
// First run after a short delay to let the DB connect
setTimeout(() => { setTimeout(() => {
run().catch(() => {}); run().catch(() => {});
// Then repeat on interval
setInterval(() => run().catch(() => {}), interval); setInterval(() => run().catch(() => {}), interval);
}, 15_000); }, 15_000);
} }