Unfloyded/Legamunt
0
0
Fork 0
Code Activity

Change indexing frequency from 5 minutes to 1 minute, add locks to schedulers (#26304)

Browse source
This commit is contained in:
Eugen Rochko 2023-08-03 11:04:05 +02:00 • committed by GitHub
parent f55f0ab0c3
commit a0fad5c8bb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 11 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

6
app/workers/scheduler/indexing_scheduler.rb
View file

@ -4,7 +4,7 @@ class Scheduler::IndexingScheduler
include Sidekiq::Worker
include Redisable
sidekiq_options retry: 0
sidekiq_options retry: 0, lock: :until_executed, lock_ttl: 1.day.to_i
IMPORT_BATCH_SIZE = 1000
SCAN_BATCH_SIZE = 10 * IMPORT_BATCH_SIZE
@ -16,9 +16,7 @@ class Scheduler::IndexingScheduler
with_redis do |redis|
redis.sscan_each("chewy:queue:#{type.name}", count: SCAN_BATCH_SIZE).each_slice(IMPORT_BATCH_SIZE) do |ids|
type.import!(ids)
redis.pipelined do |pipeline|
pipeline.srem("chewy:queue:#{type.name}", ids)
end
redis.srem("chewy:queue:#{type.name}", ids)
end
end
end