From 923f9a716b61ef54575ad47398580689fdb9ca15 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Mon, 6 Aug 2018 07:59:42 -0500 Subject: [PATCH] Add fix for channel job --- src/invidious.cr | 6 +++++- src/invidious/jobs.cr | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index 6b1be6ed..0913a137 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -83,9 +83,13 @@ crawl_threads.times do end end +total_channels = PG_DB.query_one("SELECT count(*) FROM channels", as: Int64) channel_threads.times do |i| + limit = total_channels / channel_threads + offset = limit.not_nil! * i + spawn do - refresh_channels(PG_DB) + refresh_channels(PG_DB, limit, offset) end end diff --git a/src/invidious/jobs.cr b/src/invidious/jobs.cr index 7b3f0bfa..dc44b029 100644 --- a/src/invidious/jobs.cr +++ b/src/invidious/jobs.cr @@ -44,9 +44,9 @@ def crawl_videos(db) end end -def refresh_channels(db) +def refresh_channels(db, limit = 0, offset = 0) loop do - db.query("SELECT id FROM channels ORDER BY updated") do |rs| + db.query("SELECT id FROM channels ORDER BY updated limit $1 offset $2", limit, offset) do |rs| rs.each do client = make_client(YT_URL)