Unfloyded/Legamunt
0
0
Fork 0
Code Activity

Change importers to avoid a few inefficiencies (#26721)

Browse source
This commit is contained in:
Claire 2023-08-31 19:04:27 +02:00 • committed by GitHub
parent ecd76fa413
commit 9bb2fb6b14
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 34 additions and 33 deletions
Download patch file Download diff file Expand all files Collapse all files

35
app/lib/importer/statuses_index_importer.rb
View file

@ -13,32 +13,25 @@ class Importer::StatusesIndexImporter < Importer::BaseImporter
scope.find_in_batches(batch_size: @batch_size) do |tmp|
in_work_unit(tmp.map(&:status_id)) do |status_ids|
bulk = ActiveRecord::Base.connection_pool.with_connection do
Chewy::Index::Import::BulkBuilder.new(index, to_index: index.adapter.default_scope.where(id: status_ids)).bulk_body
end
indexed = 0
deleted = 0
# We can't use the delete_if proc to do the filtering because delete_if
# is called before rendering the data and we need to filter based
# on the results of the filter, so this filtering happens here instead
bulk.map! do |entry|
new_entry = if entry[:index] && entry.dig(:index, :data, 'searchable_by').blank?
{ delete: entry[:index].except(:data) }
else
entry
end
if new_entry[:index]
indexed += 1
else
deleted += 1
bulk = ActiveRecord::Base.connection_pool.with_connection do
to_index = index.adapter.default_scope.where(id: status_ids)
crutches = Chewy::Index::Crutch::Crutches.new index, to_index
to_index.map do |object|
# This is unlikely to happen, but the post may have been
# un-interacted with since it was queued for indexing
if object.searchable_by.empty?
deleted += 1
{ delete: { _id: object.id } }
else
{ index: { _id: object.id, data: index.compose(object, crutches, fields: []) } }
end
end
new_entry
end
indexed = bulk.size - deleted
Chewy::Index::Import::BulkRequest.new(index).perform(bulk)
[indexed, deleted]