mirror of
https://github.com/svemagie/indiekit-endpoint-youtube.git
synced 2026-04-02 15:54:59 +02:00
fix: only sync new YouTube likes, not existing ones
First sync snapshots all current liked video IDs into a youtubeLikesSeen collection without creating posts. Subsequent syncs only create posts for likes not in the seen set. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -1,13 +1,14 @@
|
|||||||
/**
|
/**
|
||||||
* YouTube Likes → Indiekit "like" posts sync
|
* YouTube Likes → Indiekit "like" posts sync
|
||||||
*
|
*
|
||||||
* Fetches the authenticated user's liked videos and creates
|
* On first run after connecting, snapshots all current liked video IDs
|
||||||
* corresponding "like" posts via the Micropub posts collection.
|
* as "known" without creating posts. Subsequent syncs only create posts
|
||||||
|
* for newly liked videos (ones not in the known set and not already
|
||||||
|
* in the posts collection).
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { YouTubeClient } from "./youtube-client.js";
|
import { YouTubeClient } from "./youtube-client.js";
|
||||||
import { getValidAccessToken } from "./oauth.js";
|
import { getValidAccessToken } from "./oauth.js";
|
||||||
import crypto from "node:crypto";
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Generate a deterministic slug from a YouTube video ID.
|
* Generate a deterministic slug from a YouTube video ID.
|
||||||
@@ -18,11 +19,54 @@ function slugFromVideoId(videoId) {
|
|||||||
return `yt-like-${videoId}`;
|
return `yt-like-${videoId}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fetch all current liked video IDs (up to maxPages) and store them
|
||||||
|
* as the baseline. No posts are created.
|
||||||
|
* @returns {Promise<number>} number of IDs snapshotted
|
||||||
|
*/
|
||||||
|
async function snapshotExistingLikes(db, client, accessToken, maxPages) {
|
||||||
|
const collection = db.collection("youtubeLikesSeen");
|
||||||
|
await collection.createIndex({ videoId: 1 }, { unique: true });
|
||||||
|
|
||||||
|
let pageToken;
|
||||||
|
let count = 0;
|
||||||
|
|
||||||
|
for (let page = 0; page < maxPages; page++) {
|
||||||
|
const result = await client.getLikedVideos(accessToken, 50, pageToken);
|
||||||
|
|
||||||
|
const ops = result.videos.map((v) => ({
|
||||||
|
updateOne: {
|
||||||
|
filter: { videoId: v.id },
|
||||||
|
update: { $setOnInsert: { videoId: v.id, seenAt: new Date() } },
|
||||||
|
upsert: true,
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
|
||||||
|
if (ops.length > 0) {
|
||||||
|
await collection.bulkWrite(ops, { ordered: false });
|
||||||
|
count += ops.length;
|
||||||
|
}
|
||||||
|
|
||||||
|
pageToken = result.nextPageToken;
|
||||||
|
if (!pageToken) break;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Mark that the baseline snapshot is done
|
||||||
|
await db.collection("youtubeMeta").updateOne(
|
||||||
|
{ key: "likes_baseline" },
|
||||||
|
{ $set: { key: "likes_baseline", completedAt: new Date(), count } },
|
||||||
|
{ upsert: true },
|
||||||
|
);
|
||||||
|
|
||||||
|
return count;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Sync liked videos into the Indiekit posts collection.
|
* Sync liked videos into the Indiekit posts collection.
|
||||||
*
|
*
|
||||||
* For each liked video that doesn't already have a corresponding post
|
* First-run behaviour: snapshots all existing likes as "seen" without
|
||||||
* in the database we insert a new "like" post document.
|
* creating posts. Only likes that appear after the baseline are turned
|
||||||
|
* into posts.
|
||||||
*
|
*
|
||||||
* @param {object} opts
|
* @param {object} opts
|
||||||
* @param {import("mongodb").Db} opts.db
|
* @param {import("mongodb").Db} opts.db
|
||||||
@@ -30,7 +74,7 @@ function slugFromVideoId(videoId) {
|
|||||||
* @param {object} opts.publication - Indiekit publication config
|
* @param {object} opts.publication - Indiekit publication config
|
||||||
* @param {import("mongodb").Collection} [opts.postsCollection]
|
* @param {import("mongodb").Collection} [opts.postsCollection]
|
||||||
* @param {number} [opts.maxPages=3] - max pages to fetch (50 likes/page)
|
* @param {number} [opts.maxPages=3] - max pages to fetch (50 likes/page)
|
||||||
* @returns {Promise<{synced: number, skipped: number, total: number, error?: string}>}
|
* @returns {Promise<{synced: number, skipped: number, total: number, baselined?: number, error?: string}>}
|
||||||
*/
|
*/
|
||||||
export async function syncLikes({ db, youtubeConfig, publication, postsCollection, maxPages = 3 }) {
|
export async function syncLikes({ db, youtubeConfig, publication, postsCollection, maxPages = 3 }) {
|
||||||
const { apiKey, oauth } = youtubeConfig;
|
const { apiKey, oauth } = youtubeConfig;
|
||||||
@@ -38,7 +82,6 @@ export async function syncLikes({ db, youtubeConfig, publication, postsCollectio
|
|||||||
return { synced: 0, skipped: 0, total: 0, error: "OAuth not configured" };
|
return { synced: 0, skipped: 0, total: 0, error: "OAuth not configured" };
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get a valid access token (auto‑refreshes if needed)
|
|
||||||
let accessToken;
|
let accessToken;
|
||||||
try {
|
try {
|
||||||
accessToken = await getValidAccessToken(db, {
|
accessToken = await getValidAccessToken(db, {
|
||||||
@@ -55,6 +98,17 @@ export async function syncLikes({ db, youtubeConfig, publication, postsCollectio
|
|||||||
|
|
||||||
const client = new YouTubeClient({ apiKey: apiKey || "unused", cacheTtl: 0 });
|
const client = new YouTubeClient({ apiKey: apiKey || "unused", cacheTtl: 0 });
|
||||||
|
|
||||||
|
// --- First-run: snapshot existing likes, create zero posts ---
|
||||||
|
const baseline = await db.collection("youtubeMeta").findOne({ key: "likes_baseline" });
|
||||||
|
if (!baseline) {
|
||||||
|
console.log("[YouTube] First sync — snapshotting existing likes (no posts will be created)");
|
||||||
|
const count = await snapshotExistingLikes(db, client, accessToken, maxPages);
|
||||||
|
console.log(`[YouTube] Baselined ${count} existing liked videos`);
|
||||||
|
return { synced: 0, skipped: 0, total: count, baselined: count };
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Normal sync: only create posts for new likes ---
|
||||||
|
const seenCollection = db.collection("youtubeLikesSeen");
|
||||||
let synced = 0;
|
let synced = 0;
|
||||||
let skipped = 0;
|
let skipped = 0;
|
||||||
let total = 0;
|
let total = 0;
|
||||||
@@ -68,10 +122,26 @@ export async function syncLikes({ db, youtubeConfig, publication, postsCollectio
|
|||||||
total = result.totalResults;
|
total = result.totalResults;
|
||||||
|
|
||||||
for (const video of result.videos) {
|
for (const video of result.videos) {
|
||||||
const slug = slugFromVideoId(video.id);
|
const videoId = video.id;
|
||||||
const videoUrl = `https://www.youtube.com/watch?v=${video.id}`;
|
|
||||||
|
|
||||||
// Check if we already synced this like
|
// Already seen? Skip.
|
||||||
|
const seen = await seenCollection.findOne({ videoId });
|
||||||
|
if (seen) {
|
||||||
|
skipped++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Mark as seen immediately (even if post insert fails, don't retry)
|
||||||
|
await seenCollection.updateOne(
|
||||||
|
{ videoId },
|
||||||
|
{ $setOnInsert: { videoId, seenAt: new Date() } },
|
||||||
|
{ upsert: true },
|
||||||
|
);
|
||||||
|
|
||||||
|
const slug = slugFromVideoId(videoId);
|
||||||
|
const videoUrl = `https://www.youtube.com/watch?v=${videoId}`;
|
||||||
|
|
||||||
|
// Also skip if a post already exists (belt-and-suspenders)
|
||||||
if (postsCollection) {
|
if (postsCollection) {
|
||||||
const existing = await postsCollection.findOne({
|
const existing = await postsCollection.findOne({
|
||||||
$or: [
|
$or: [
|
||||||
@@ -85,7 +155,6 @@ export async function syncLikes({ db, youtubeConfig, publication, postsCollectio
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Build the post path/url from the publication's like post type config
|
|
||||||
const postPath = likePostType?.post?.path
|
const postPath = likePostType?.post?.path
|
||||||
? likePostType.post.path.replace("{slug}", slug)
|
? likePostType.post.path.replace("{slug}", slug)
|
||||||
: `content/likes/${slug}.md`;
|
: `content/likes/${slug}.md`;
|
||||||
@@ -105,11 +174,11 @@ export async function syncLikes({ db, youtubeConfig, publication, postsCollectio
|
|||||||
text: `Liked "${video.title}" by ${video.channelTitle} on YouTube`,
|
text: `Liked "${video.title}" by ${video.channelTitle} on YouTube`,
|
||||||
html: `Liked "<a href="${videoUrl}">${escapeHtml(video.title)}</a>" by ${escapeHtml(video.channelTitle)} on YouTube`,
|
html: `Liked "<a href="${videoUrl}">${escapeHtml(video.title)}</a>" by ${escapeHtml(video.channelTitle)} on YouTube`,
|
||||||
},
|
},
|
||||||
published: video.publishedAt || new Date().toISOString(),
|
published: new Date().toISOString(),
|
||||||
url: postUrl,
|
url: postUrl,
|
||||||
visibility: "public",
|
visibility: "public",
|
||||||
"post-status": "published",
|
"post-status": "published",
|
||||||
"youtube-video-id": video.id,
|
"youtube-video-id": videoId,
|
||||||
"youtube-channel": video.channelTitle,
|
"youtube-channel": video.channelTitle,
|
||||||
"youtube-thumbnail": video.thumbnail || "",
|
"youtube-thumbnail": video.thumbnail || "",
|
||||||
},
|
},
|
||||||
@@ -125,7 +194,6 @@ export async function syncLikes({ db, youtubeConfig, publication, postsCollectio
|
|||||||
if (!pageToken) break;
|
if (!pageToken) break;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Update sync metadata
|
|
||||||
await db.collection("youtubeMeta").updateOne(
|
await db.collection("youtubeMeta").updateOne(
|
||||||
{ key: "likes_sync" },
|
{ key: "likes_sync" },
|
||||||
{
|
{
|
||||||
@@ -158,7 +226,7 @@ export async function getLastSyncStatus(db) {
|
|||||||
* @param {object} options - endpoint options
|
* @param {object} options - endpoint options
|
||||||
*/
|
*/
|
||||||
export function startLikesSync(Indiekit, options) {
|
export function startLikesSync(Indiekit, options) {
|
||||||
const interval = options.likes?.syncInterval || 3_600_000; // default 1 hour
|
const interval = options.likes?.syncInterval || 3_600_000;
|
||||||
|
|
||||||
async function run() {
|
async function run() {
|
||||||
const db = Indiekit.database;
|
const db = Indiekit.database;
|
||||||
@@ -178,15 +246,16 @@ export function startLikesSync(Indiekit, options) {
|
|||||||
if (result.synced > 0) {
|
if (result.synced > 0) {
|
||||||
console.log(`[YouTube] Likes sync: ${result.synced} new, ${result.skipped} skipped`);
|
console.log(`[YouTube] Likes sync: ${result.synced} new, ${result.skipped} skipped`);
|
||||||
}
|
}
|
||||||
|
if (result.baselined) {
|
||||||
|
console.log(`[YouTube] Baseline complete: ${result.baselined} existing likes recorded`);
|
||||||
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error("[YouTube] Likes sync error:", err.message);
|
console.error("[YouTube] Likes sync error:", err.message);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// First run after a short delay to let the DB connect
|
|
||||||
setTimeout(() => {
|
setTimeout(() => {
|
||||||
run().catch(() => {});
|
run().catch(() => {});
|
||||||
// Then repeat on interval
|
|
||||||
setInterval(() => run().catch(() => {}), interval);
|
setInterval(() => run().catch(() => {}), interval);
|
||||||
}, 15_000);
|
}, 15_000);
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user