Recall

A hand-written excerpt from the poller

Trimmed from processAndSaveToSupabase.js (January 2024). Lines with keys, the per-source parsing and logging are left out; the comments marked trimmed are mine, everything else is as written.

async function main() {
    console.log('START:: ' + new Date(Date.now()).toISOString());
    const t0 = performance.now(); 
    const { data, error } = await supabase
        .from('share_data')
        .select()
        .eq('is_processed', false);
    // …trimmed

    for(let i = 0; i < data.length; i++) {
        let shareItem = data[i];

        let shareUrl = shareItem.share_url;
        let ytVideoID = extractYouTubeVideoId(shareUrl);

        if(ytVideoID != null) {
            // …trimmed: skip videos already in youtube_data, otherwise queue the ID
        } else {
            let htmlContent = await fetchHTML(shareUrl);
            // …trimmed: parse the page and save the article as Markdown
        }
    }

    if(ytVideoIDList.length > 0) {
        let videoIDLists = extractYTVideoLists(ytVideoIDList);
        console.log(videoIDLists.length);
        for(i2 = 0; i2 < videoIDLists.length; i2++) {
            await getVideoInfo(ytVideoIDList.join(','), shareIDList, ytVideoIDList, shareURLList);
        }
    }
    // …trimmed
}

function extractYTVideoLists(ytVideoIDList) {
    let maxVideoIDs = 49;
    let resultArray = [];

    for(let i1 = 0; i1 < ytVideoIDList.length; i1+=maxVideoIDs) {
        resultArray.push(ytVideoIDList.slice(i1, i1 + maxVideoIDs));
    }

    return resultArray;
}

Looking back. The batching loop passes the whole list of IDs to every call instead of each batch of 49. It never broke, because saves came in ones and twos, but 120 links shared at once would have failed. Today I’d pass videoIDLists[i2] and add a test that saves 120 links in one go.