Add support to delete a document during index_many

[sobrinho]

Hi there!

Assuming that I know a document must be deleted during the index_many, we could batch the deletes as well:

# app/indices/movies_index.rb
class MoviesIndex < Typesensual::Index
  # The schema of the collection
  schema do
    enable_nested_fields

    field 'title', type: 'string'
    field 'release_date\..*', type: 'int32', facet: true
    field 'average_rating', type: 'float', facet: true
    field 'user_count', type: 'int32'
    field 'genres', type: 'string[]', facet: true
  end

  def index(ids)
    Movies.where(id: ids).includes(:genres).find_each do |movie|
      if movie.soft_deleted?
        yield :delete, {
          id: movie.id,
        }
      else
        yield(
          id: movie.id,
          title: movie.title,
          release_date: {
            year: movie.release_date.year,
            month: movie.release_date.month,
            day: movie.release_date.day
          },
          average_rating: movie.average_rating,
          user_count: movie.user_count,
          genres: movie.genres.map(&:name)
        )
      end
    end
  end
end

I'm not sure about the syntax but something like this would be nice to give a collection of ids and the index decide to delete and even use the delete_many from the typesense client.

PoC:

# Current implementation
def insert_many!(docs, batch_size: 100)
  docs.lazy.each_slice(batch_size).with_object([]) do |slice, failures|
    results = typesense_collection.documents.import(slice, return_id: true, action: 'upsert')
    failures.push(*results.reject { |result| result['success'] })
  end
end

# PoC (not tested)
def insert_many!(docs, batch_size: 100)
  docs.lazy.each_slice(batch_size).with_object([]) do |slice, failures|
    to_delete, to_upsert = slice.partition_by { |item| item[0] == :delete }
    to_delete_results = typesense_collection.documents.delete(filter_by: "id:#{to_delete.pluck(:id).to_json}")
    failures.push(*to_delete_results.reject { |result| result['success'] })
    to_upsert_results = typesense_collection.documents.import(slice, return_id: true, action: 'upsert')
    failures.push(*to_upsert_results.reject { |result| result['success'] })
  end
end

[sobrinho]

Current workaround:

def index(ids)
  to_delete = []

  Movies.where(id: ids).includes(:genres).find_each do |movie|
    if movie.soft_deleted?
      to_delete << movie.id
    else
      yield(...)
    end
  end

  if to_delete.any?
    collection.remove_many!(filter_by: "id:#{to_delete.to_json}")
  end
end