40 lines
937 B
Text
40 lines
937 B
Text
|
#!/usr/bin/env ruby
|
||
|
|
# frozen_string_literal: true
|
||
|
|
|
||
|
|
$stdout.sync = true
|
||
|
|
|
||
|
|
require "rufus-scheduler"
|
||
|
|
require_relative "../boot"
|
||
|
|
require "provider"
|
||
|
|
require "provider/adapters"
|
||
|
|
|
||
|
|
dry_run = ARGV.include?("--dry-run")
|
||
|
|
|
||
|
|
scheduler = Rufus::Scheduler.new(max_work_threads: DB.pool.max_size) unless dry_run
|
||
|
|
|
||
|
|
# Backfill all providers on startup (staggered to avoid thundering herd)
|
||
|
|
Provider.all.shuffle.each do |provider|
|
||
|
|
if dry_run
|
||
|
|
puts "startup: backfill[#{provider.key.downcase}]"
|
||
|
|
else
|
||
|
|
scheduler.in("0s") do
|
||
|
|
provider.backfill
|
||
|
|
end
|
||
|
|
end
|
||
|
|
end
|
||
|
|
|
||
|
|
# Schedule each provider based on its publish_schedule cron expression.
|
||
|
|
Provider.all.each do |provider|
|
||
|
|
next unless provider.publish_schedule
|
||
|
|
|
||
|
|
if dry_run
|
||
|
|
puts "cron: #{provider.publish_schedule} backfill[#{provider.key.downcase}]"
|
||
|
|
else
|
||
|
|
scheduler.cron(provider.publish_schedule, overlap: false) do
|
||
|
|
provider.backfill
|
||
|
|
end
|
||
|
|
end
|
||
|
|
end
|
||
|
|
|
||
|
|
scheduler.join unless dry_run
|