From e164b75f7e79e053fb5712911b6e1cc13c33b22d Mon Sep 17 00:00:00 2001 From: Victor Nunes Date: Wed, 5 Jun 2024 14:11:50 -0300 Subject: [PATCH] Implement using the official Elasticsearch library. Create a new implementation using the official Elasticsearch library in a separate package, without impacting those who use the Olivere implementation. Refine the abstraction of Elasticsearch Search functionality even more, enabling the use of this library only. --- README.md | 106 +--- examples/official/filter.go | 81 +++ examples/official/main.go | 36 ++ examples/{ => olivere}/main.go | 0 go.mod | 5 +- go.sum | 6 + official/v7/client.go | 45 ++ official/v7/client_mock.go | 24 + official/v7/entity_errors.go | 48 ++ official/v7/entity_filter.go | 95 ++++ official/v7/entity_response.go | 161 ++++++ official/v7/entity_sort.go | 28 + official/v7/helper_log.go | 49 ++ official/v7/internal/retrier/retrier.go | 37 ++ official/v7/querybuilders/entity_query.go | 6 + official/v7/querybuilders/script.go | 103 ++++ .../v7/querybuilders/search_queries_bool.go | 199 +++++++ .../v7/querybuilders/search_queries_exists.go | 45 ++ .../querybuilders/search_queries_match_all.go | 61 +++ .../search_queries_multi_match.go | 266 ++++++++++ .../v7/querybuilders/search_queries_nested.go | 77 +++ .../v7/querybuilders/search_queries_range.go | 151 ++++++ .../v7/querybuilders/search_queries_term.go | 63 +++ .../v7/querybuilders/search_queries_terms.go | 84 +++ .../querybuilders/search_queries_terms_set.go | 93 ++++ .../v7/querybuilders/search_terms_lookup.go | 72 +++ official/v7/search.go | 86 +++ official/v7/search_querybuilder.go | 501 ++++++++++++++++++ official/v7/search_test.go | 289 ++++++++++ 29 files changed, 2713 insertions(+), 104 deletions(-) create mode 100644 examples/official/filter.go create mode 100644 examples/official/main.go rename examples/{ => olivere}/main.go (100%) create mode 100644 official/v7/client.go create mode 100644 official/v7/client_mock.go create mode 100644 official/v7/entity_errors.go create mode 100644 official/v7/entity_filter.go create mode 100644 official/v7/entity_response.go create mode 100644 official/v7/entity_sort.go create mode 100644 official/v7/helper_log.go create mode 100644 official/v7/internal/retrier/retrier.go create mode 100644 official/v7/querybuilders/entity_query.go create mode 100644 official/v7/querybuilders/script.go create mode 100644 official/v7/querybuilders/search_queries_bool.go create mode 100644 official/v7/querybuilders/search_queries_exists.go create mode 100644 official/v7/querybuilders/search_queries_match_all.go create mode 100644 official/v7/querybuilders/search_queries_multi_match.go create mode 100644 official/v7/querybuilders/search_queries_nested.go create mode 100644 official/v7/querybuilders/search_queries_range.go create mode 100644 official/v7/querybuilders/search_queries_term.go create mode 100644 official/v7/querybuilders/search_queries_terms.go create mode 100644 official/v7/querybuilders/search_queries_terms_set.go create mode 100644 official/v7/querybuilders/search_terms_lookup.go create mode 100644 official/v7/search.go create mode 100644 official/v7/search_querybuilder.go create mode 100644 official/v7/search_test.go diff --git a/README.md b/README.md index 930664e..5a76cfd 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ ## 1. Description -ElasticUtil is a generic library that assists in the use of Elasticsearch, using olivere/elastic library. It is possible to create olivere/elastic's queries using only one struct. It is also possible to translate errors and responses. +ElasticUtil is a generic library that assists in the use of Elasticsearch, using olivere/elastic library and the offical library. It is possible to create elastic's queries using only one struct. It is also possible to translate errors and responses. ## 2. Technology Stack @@ -46,111 +46,11 @@ ElasticUtil is a generic library that assists in the use of Elasticsearch, using go mod tidy ``` -- ### Usage - - - Import the package - - ```go - import ( - "github.com/arquivei/elasticutil" - ) - ``` - - - Define a filter struct - - ```go - type ExampleFilterMust struct { - Names []string `es:"Name"` - SocialNames []string `es:"SocialName"` - Ages []uint64 `es:"Age"` - HasCovid *bool - CreatedAt *elasticutil.TimeRange - AgeRange *elasticutil.IntRange `es:"Age"` - CovidInfo elasticutil.Nested `es:"Covid"` - NameOrSocialName elasticutil.FullTextSearchShould `es:"Name,SocialName"` - } - - type ExampleFilterExists struct { - HasCovidInfo elasticutil.Nested `es:"Covid"` - HasAge *bool `es:"Age"` - } - - type ExampleCovidInfo struct { - HasCovidInfo *bool `es:"Covid"` - Symptoms []string `es:"Covid.Symptom"` - FirstSymptomDate *elasticutil.TimeRange `es:"Covid.Date"` - } - ``` - - - And now, it's time! - - ```go - requestFilter := elasticutil.Filter{ - Must: ExampleFilterMust{ - Names: []string{"John", "Mary"}, - Ages: []uint64{16, 17, 18, 25, 26}, - HasCovid: refBool(true), - CovidInfo: elasticutil.NewNested( - ExampleCovidInfo{ - Symptoms: []string{"cough"}, - FirstSymptomDate: &elasticutil.TimeRange{ - From: time.Date(2019, time.November, 28, 15, 27, 39, 49, time.UTC), - To: time.Date(2020, time.November, 28, 15, 27, 39, 49, time.UTC), - }, - }, - ), - CreatedAt: &elasticutil.TimeRange{ - From: time.Date(2020, time.November, 28, 15, 27, 39, 49, time.UTC), - To: time.Date(2021, time.November, 28, 15, 27, 39, 49, time.UTC), - }, - AgeRange: &elasticutil.IntRange{ - From: 15, - To: 30, - }, - NameOrSocialName: elasticutil.NewFullTextSearchShould([]string{"John", "Mary", "Rebecca"}), - }, - MustNot: ExampleFilterMust{ - Names: []string{"Lary"}, - AgeRange: &elasticutil.IntRange{ - From: 29, - To: 30, - }, - }, - Exists: ExampleFilterExists{ - HasCovidInfo: elasticutil.NewNested( - ExampleCovidInfo{ - HasCovidInfo: refBool(true), - }, - ), - HasAge: refBool(true), - }, - } - - // BuildElasticBoolQuery builds a olivere/elastic's query based on Filter. - elasticQuery, err := elasticutil.BuildElasticBoolQuery(context.Background(), requestFilter) - if err != nil { - panic(err) - } - - // MarshalQuery transforms a olivere/elastic's query in a string for log and test - // purpose. - verboseElasticQuery := elasticutil.MarshalQuery(elasticQuery) - - fmt.Println(verboseElasticQuery) - ``` - - ### Examples - - [Sample usage](https://github.com/arquivei/elasticutil/blob/master/examples/main.go) + - [Olivere](https://github.com/arquivei/elasticutil/blob/master/examples/olivere/main.go) -## 4. Changelog - - - **ElasticUtil 0.1.0 (May 27, 2022)** - - - [New] Decoupling this package from Arquivei's API projects. - - [New] Setting github's workflow with golangci-lint - - [New] Example for usage. - - [New] Documents: Code of Conduct, Contributing, License and Readme. + - [Official library](https://github.com/arquivei/elasticutil/blob/master/examples/official/main.go) ## 5. Collaborators diff --git a/examples/official/filter.go b/examples/official/filter.go new file mode 100644 index 0000000..94149c9 --- /dev/null +++ b/examples/official/filter.go @@ -0,0 +1,81 @@ +package main + +import ( + "time" + + elasticutil "github.com/arquivei/elasticutil/official/v7" + "github.com/arquivei/elasticutil/official/v7/querybuilders" +) + +type ExampleFilterMust struct { + Names []string `es:"Name"` + SocialNames []string `es:"SocialName"` + Ages []uint64 `es:"Age"` + HasCovid *bool + CreatedAt *elasticutil.TimeRange + AgeRange *elasticutil.IntRange `es:"Age"` + CovidInfo elasticutil.Nested `es:"Covid"` + NameOrSocialName elasticutil.FullTextSearchShould `es:"Name,SocialName"` + MyCustomSearch elasticutil.CustomSearch +} + +type ExampleFilterExists struct { + HasCovidInfo elasticutil.Nested `es:"Covid"` + HasAge *bool `es:"Age"` +} + +type ExampleCovidInfo struct { + HasCovidInfo *bool `es:"Covid"` + Symptoms []string `es:"Covid.Symptom"` + FirstSymptomDate *elasticutil.TimeRange `es:"Covid.Date"` +} + +func createFilter() elasticutil.Filter { + return elasticutil.Filter{ + Must: ExampleFilterMust{ + Names: []string{"John", "Mary"}, + Ages: []uint64{16, 17, 18, 25, 26}, + HasCovid: refBool(true), + CovidInfo: elasticutil.NewNested( + ExampleCovidInfo{ + Symptoms: []string{"cough"}, + FirstSymptomDate: &elasticutil.TimeRange{ + From: time.Date(2019, time.November, 28, 15, 27, 39, 49, time.UTC), + To: time.Date(2020, time.November, 28, 15, 27, 39, 49, time.UTC), + }, + }, + ), + CreatedAt: &elasticutil.TimeRange{ + From: time.Date(2020, time.November, 28, 15, 27, 39, 49, time.UTC), + To: time.Date(2021, time.November, 28, 15, 27, 39, 49, time.UTC), + }, + AgeRange: &elasticutil.IntRange{ + From: 15, + To: 30, + }, + NameOrSocialName: elasticutil.NewFullTextSearchShould([]string{"John", "Mary", "Rebecca"}), + MyCustomSearch: elasticutil.NewCustomSearch(func() (querybuilders.Query, error) { + return querybuilders.NewBoolQuery().Must(querybuilders.NewTermQuery("Name", "John")), nil + }), + }, + MustNot: ExampleFilterMust{ + Names: []string{"Lary"}, + AgeRange: &elasticutil.IntRange{ + From: 29, + To: 30, + }, + }, + Exists: ExampleFilterExists{ + HasCovidInfo: elasticutil.NewNested( + ExampleCovidInfo{ + HasCovidInfo: refBool(true), + }, + ), + HasAge: refBool(true), + }, + } +} + +func refBool(b bool) *bool { + return &b +} diff --git a/examples/official/main.go b/examples/official/main.go new file mode 100644 index 0000000..cb25c40 --- /dev/null +++ b/examples/official/main.go @@ -0,0 +1,36 @@ +package main + +import ( + "context" + "fmt" + + elasticutil "github.com/arquivei/elasticutil/official/v7" +) + +func main() { + client := elasticutil.MustNewClient("") + + response, err := client.Search( + context.Background(), + elasticutil.SearchConfig{ + Filter: createFilter(), + Indexes: []string{""}, + Size: 5, + IgnoreUnavailable: true, + AllowNoIndices: true, + TrackTotalHits: true, + Sort: elasticutil.Sorters{ + Sorters: []elasticutil.Sorter{ + { + Field: "ID", + Ascending: true, + }, + }, + }, + SearchAfter: "", + }, + ) + + fmt.Println(response, err) + +} diff --git a/examples/main.go b/examples/olivere/main.go similarity index 100% rename from examples/main.go rename to examples/olivere/main.go diff --git a/go.mod b/go.mod index 9624d69..e880124 100644 --- a/go.mod +++ b/go.mod @@ -6,19 +6,22 @@ toolchain go1.21.3 require ( github.com/arquivei/foundationkit v0.6.1 + github.com/elastic/go-elasticsearch/v7 v7.17.10 github.com/olivere/elastic/v7 v7.0.32 + github.com/rs/zerolog v1.31.0 github.com/stretchr/testify v1.9.0 ) require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/go-kit/kit v0.13.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/rs/zerolog v1.31.0 // indirect + github.com/stretchr/objx v0.5.2 // indirect golang.org/x/sys v0.16.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 665f2b0..fc9b5ab 100644 --- a/go.sum +++ b/go.sum @@ -3,8 +3,12 @@ github.com/arquivei/foundationkit v0.6.1/go.mod h1:OC6R9oJgGD8C+HTNAZcZaV6lZCk+3 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/elastic/go-elasticsearch/v7 v7.17.10 h1:TCQ8i4PmIJuBunvBS6bwT2ybzVFxxUhhltAs3Gyu1yo= +github.com/elastic/go-elasticsearch/v7 v7.17.10/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= +github.com/go-kit/kit v0.13.0 h1:OoneCcHKHQ03LfBpoQCUfCluwd2Vt3ohz+kvbJneZAU= +github.com/go-kit/kit v0.13.0/go.mod h1:phqEHMMUbyrCFCTgH48JueqrM3md2HcAZ8N3XE4FKDg= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= @@ -26,6 +30,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A= github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/official/v7/client.go b/official/v7/client.go new file mode 100644 index 0000000..ab32bb6 --- /dev/null +++ b/official/v7/client.go @@ -0,0 +1,45 @@ +package v7 + +import ( + "context" + "net/http" + + "github.com/arquivei/elasticutil/official/v7/internal/retrier" + es "github.com/elastic/go-elasticsearch/v7" +) + +// Client represents the Elasticsearch's client of the official lib. +type Client interface { + Search(context.Context, SearchConfig) (SearchResponse, error) +} + +type esClient struct { + client *es.Client +} + +// NewClient returns a new Client using the @urls. +func NewClient(urls ...string) (Client, error) { + client, err := es.NewClient(es.Config{ + Addresses: urls, + RetryBackoff: retrier.NewSimpleBackoff(10, 100), + Transport: &http.Transport{ + DisableCompression: false, + }, + }) + if err != nil { + return nil, err + } + return &esClient{ + client: client, + }, nil +} + +// MustNewClient returns a new Client using the @urls. It panics +// instead of returning an error. +func MustNewClient(urls ...string) Client { + client, err := NewClient(urls...) + if err != nil { + panic(err) + } + return client +} diff --git a/official/v7/client_mock.go b/official/v7/client_mock.go new file mode 100644 index 0000000..cad5126 --- /dev/null +++ b/official/v7/client_mock.go @@ -0,0 +1,24 @@ +package v7 + +import ( + "context" + + "github.com/stretchr/testify/mock" +) + +type mockClient struct { + mock.Mock +} + +func (m *mockClient) Search(_ context.Context, sc SearchConfig) (SearchResponse, error) { + args := m.Called(sc) + return args.Get(0).(SearchResponse), args.Error(1) +} + +// MustNewClientMock returns a mocked Client that uses Search method and returns @expectedResponse +// and @expectedError for the giving @input. +func MustNewClientMockSearch(input SearchConfig, expectedResponse SearchResponse, expectedError error) Client { + m := mockClient{} + m.On("Search", input).Return(expectedResponse, expectedError) + return &m +} diff --git a/official/v7/entity_errors.go b/official/v7/entity_errors.go new file mode 100644 index 0000000..354914d --- /dev/null +++ b/official/v7/entity_errors.go @@ -0,0 +1,48 @@ +package v7 + +import "github.com/arquivei/foundationkit/errors" + +// ---------- Errors + +// ErrNotAllShardsReplied is returned when no all elasticsearch's shards +// successfully reply. +var ErrNotAllShardsReplied = errors.New("not all shards replied") + +// ErrNilResponse is returned when elasticsearch returns no error, but +// returns a nil response. +var ErrNilResponse = errors.New("nil response") + +// ---------- Codes + +// ErrCodeBadRequest is returned when elasticsearch returns a +// status 400. +var ErrCodeBadRequest = errors.Code("bad request") + +// ErrCodeBadGateway is returned when elasticsearch client returns an error. +var ErrCodeBadGateway = errors.Code("bad gateway") + +// ErrCodeUnexpectedResponse is returned when the elasticsearch returned +// unexpected data +var ErrCodeUnexpectedResponse = errors.Code("unexpected response") + +// ---------- + +func filterMustBeAStructError(kind string) error { + return errors.New("[" + kind + "] filter must be a struct") +} + +func structNotSupportedError(name string) error { + return errors.New("[" + name + "] struct is not supported") +} + +func typeNotSupportedError(name, t string) error { + return errors.New("[" + name + "] is of unknown type: " + t) +} + +func fullTextSearchTypeNotSupported(name string) error { + return errors.New("[" + name + "] full text search value is not supported") +} + +func multiMatchSearchTypeNotSupported(name string) error { + return errors.New("[" + name + "] multi match search value is not supported") +} diff --git a/official/v7/entity_filter.go b/official/v7/entity_filter.go new file mode 100644 index 0000000..7a17fc3 --- /dev/null +++ b/official/v7/entity_filter.go @@ -0,0 +1,95 @@ +package v7 + +import ( + "time" + + "github.com/arquivei/elasticutil/official/v7/querybuilders" +) + +// Filter is a struct that eill be transformed in a olivere/elastic's query. +// +// "Must" and "MustNot" is for the terms, range and multi match query. +// "Exists" is for the exists query. +// For nested queries, uses the Nested type. +type Filter struct { + Must interface{} + MustNot interface{} + Exists interface{} +} + +// Ranges is an interface that represents one of the following range type: +// time.time, uint64 and float64. +type Ranges interface { + time.Time | uint64 | float64 +} + +// TimeRange represents a time range with a beginning and an end. +type TimeRange struct { + From time.Time + To time.Time +} + +// IntRange represents an int range with a beginning and an end. +type IntRange struct { + From uint64 + To uint64 +} + +// FloatRange represents a float range with a beginning and an end. +type FloatRange struct { + From float64 + To float64 +} + +// Nested represents a nested query. +type Nested struct { + payload interface{} +} + +// NewNested creates a Nested struct with the given payload. +func NewNested(payload interface{}) Nested { + return Nested{payload} +} + +// FullTextSearchMust Represents a Must's Full Text Search. +type FullTextSearchMust struct { + payload interface{} +} + +// NewFullTextSearchMust creates a FullTextSearchMust struct with the given payload. +func NewFullTextSearchMust(payload interface{}) FullTextSearchMust { + return FullTextSearchMust{payload} +} + +// FullTextSearchMust Represents a Should's Full Text Search. +type FullTextSearchShould struct { + payload interface{} +} + +// NewFullTextSearchShould creates a FullTextSearchShould struct with the given payload. +func NewFullTextSearchShould(payload interface{}) FullTextSearchShould { + return FullTextSearchShould{payload} +} + +// CustomSearch is the struct that contains the CustomQuery function. +type CustomSearch struct { + GetQuery CustomQuery +} + +// CustomQuery is the type function that will return the custom query. +type CustomQuery func() (querybuilders.Query, error) + +// NewCustomSearch creates a CustomSearch struct with the given CustomQuery function. +func NewCustomSearch(query CustomQuery) CustomSearch { + return CustomSearch{query} +} + +// MultiMatchSearchShould Represents a Should's Multi Match Search. +type MultiMatchSearchShould struct { + payload interface{} +} + +// NewMultiMatchSearchShould creates a MultiMatchSearchShould struct with the given payload. +func NewMultiMatchSearchShould(payload interface{}) MultiMatchSearchShould { + return MultiMatchSearchShould{payload} +} diff --git a/official/v7/entity_response.go b/official/v7/entity_response.go new file mode 100644 index 0000000..7732a27 --- /dev/null +++ b/official/v7/entity_response.go @@ -0,0 +1,161 @@ +package v7 + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + + "github.com/arquivei/foundationkit/errors" + "github.com/elastic/go-elasticsearch/v7/esapi" +) + +// SearchResponse represents the response for Search method. +type SearchResponse struct { + IDs []string + Paginator string + Total int + Took int +} + +type envelopeResponse struct { + Took int + Hits struct { + Total struct { + Value int + } + Hits []*envelopeHits `json:"Hits"` + } + Shards *shardsInfo `json:"_shards,omitempty"` +} + +type envelopeHits struct { + ID string `json:"_id"` + Sort []interface{} `json:"sort"` +} + +type shardsInfo struct { + Total int `json:"total"` + Successful int `json:"successful"` + Failed int `json:"failed"` +} + +func parseResponse(ctx context.Context, response *esapi.Response) (SearchResponse, error) { + const op = errors.Op("parseResponse") + + var searchResponse SearchResponse + + var r envelopeResponse + err := json.NewDecoder(response.Body).Decode(&r) + if err != nil { + return SearchResponse{}, errors.E(op, err) + } + + searchResponse.Total = r.Hits.Total.Value + searchResponse.Took = r.Took + + enrichLogWithTook(ctx, r.Took) + enrichLogWithShards(ctx, getTotalShards(r)) + + err = checkShards(r) + if err != nil { + return searchResponse, errors.E(op, err, ErrCodeBadGateway) + } + + if len(r.Hits.Hits) < 1 { + return searchResponse, nil + } + + for _, hit := range r.Hits.Hits { + searchResponse.IDs = append(searchResponse.IDs, hit.ID) + } + + paginator, err := getPaginatorFromHits(r.Hits.Hits) + if err != nil { + return SearchResponse{}, errors.E(op, err) + } + + searchResponse.Paginator = paginator + return searchResponse, nil +} + +func getPaginatorFromHits(hits []*envelopeHits) (string, error) { + const op errors.Op = "getPaginatorFromHits" + + if len(hits) == 0 { + return "", nil + } + + lastHit := hits[len(hits)-1] + if lastHit == nil || lastHit.Sort == nil { + return "", nil + } + paginator, err := json.Marshal(lastHit.Sort) + if err != nil { + return "", errors.E(op, err) + } + return string(paginator), nil +} + +func getTotalShards(r envelopeResponse) int { + if r.Shards != nil { + return r.Shards.Total + } + return 0 +} + +func checkShards(r envelopeResponse) error { + if r.Shards != nil && r.Shards.Failed > 0 { + return errors.E( + ErrNotAllShardsReplied, + errors.KV("replied", r.Shards.Successful), + errors.KV("failed", r.Shards.Failed), + errors.KV("total", r.Shards.Total), + ) + } + return nil +} + +func checkErrorFromResponse(response *esapi.Response) error { + if response == nil { + return ErrNilResponse + } + + if !response.IsError() { + return nil + } + + var responseBody map[string]interface{} + decodeError := json.NewDecoder(response.Body).Decode(&responseBody) + if decodeError != nil { + return decodeError + } + + errorBody, ok := responseBody["error"].(map[string]interface{}) + if !ok { + return errors.New("failed to decode error from response") + } + + rootCause, ok := errorBody["root_cause"].([]interface{}) + if !ok { + return errors.New("failed to decode root cause from error response") + } + + var reasonCause interface{} + if len(rootCause) > 0 { + rootCauses, ok := rootCause[0].(map[string]interface{}) + if !ok { + return errors.New("failed to decode root cause map from error response") + } + + reasonCause = rootCauses["reason"] + } + + err := fmt.Errorf("[%s] %s: %s: %s", response.Status(), errorBody["type"], errorBody["reason"], reasonCause) + + if response.StatusCode == http.StatusBadRequest { + err = errors.E(err, ErrCodeBadRequest) + } + + return err +} diff --git a/official/v7/entity_sort.go b/official/v7/entity_sort.go new file mode 100644 index 0000000..c2fa81b --- /dev/null +++ b/official/v7/entity_sort.go @@ -0,0 +1,28 @@ +package v7 + +// Sorters represents a list of Sorter. +type Sorters struct { + Sorters []Sorter +} + +// Sorter represents a Elasticsearch´s sort structure. +type Sorter struct { + Field string + Ascending bool +} + +func (s Sorter) String() string { + direction := "asc" + if !s.Ascending { + direction = "desc" + } + return s.Field + ":" + direction +} + +func (ss Sorters) Strings() []string { + response := make([]string, 0, len(ss.Sorters)) + for _, sorter := range ss.Sorters { + response = append(response, sorter.String()) + } + return response +} diff --git a/official/v7/helper_log.go b/official/v7/helper_log.go new file mode 100644 index 0000000..df4371c --- /dev/null +++ b/official/v7/helper_log.go @@ -0,0 +1,49 @@ +package v7 + +import ( + "context" + "time" + + "github.com/arquivei/foundationkit/contextmap" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" +) + +func enrichLogWithIndexes(ctx context.Context, indexes []string) { + log.Ctx(ctx).UpdateContext(func(zc zerolog.Context) zerolog.Context { + return zc.Strs("elastic_indexes", indexes) + }) + + contextmap.Ctx(ctx).Set("elastic_indexes", indexes) +} + +func enrichLogWithQuery(ctx context.Context, query string) { + log.Ctx(ctx).UpdateContext(func(zc zerolog.Context) zerolog.Context { + return zc.Str("elastic_query", truncate(query, 3000)) + }) +} + +func enrichLogWithTook(ctx context.Context, took int) { + log.Ctx(ctx).UpdateContext(func(zc zerolog.Context) zerolog.Context { + return zc.Dur("elastic_took_internal", time.Duration(took)*time.Millisecond) + }) +} + +func enrichLogWithShards(ctx context.Context, shards int) { + log.Ctx(ctx).UpdateContext(func(zc zerolog.Context) zerolog.Context { + return zc.Int("elastic_shards", shards) + }) +} + +func truncate(str string, size int) string { + if size <= 0 { + return "" + } + + runes := []rune(str) + if len(runes) <= size { + return str + } + + return string(runes[:size]) +} diff --git a/official/v7/internal/retrier/retrier.go b/official/v7/internal/retrier/retrier.go new file mode 100644 index 0000000..1fb2097 --- /dev/null +++ b/official/v7/internal/retrier/retrier.go @@ -0,0 +1,37 @@ +package retrier + +import ( + "sync" + "time" +) + +// NewSimpleBackoff creates a SimpleBackoff algorithm with the specified +// list of fixed intervals in milliseconds. +func NewSimpleBackoff(ticks ...int) func(attempt int) time.Duration { + s := &simpleBackoff{ + ticks: ticks, + } + return s.RetryBackoff +} + +// simpleBackoff takes a list of fixed values for backoff intervals. +// Each call to RetryBackoff returns the next value from that fixed list. +// After each value is returned, subsequent calls to Next will only return +// the last element. +type simpleBackoff struct { + sync.Mutex + ticks []int +} + +// RetryBackoff implements a backoff function for SimpleBackoff. +func (b *simpleBackoff) RetryBackoff(retry int) time.Duration { + b.Lock() + defer b.Unlock() + + if retry >= len(b.ticks) { + return 0 + } + + ms := b.ticks[retry] + return time.Duration(ms) * time.Millisecond +} diff --git a/official/v7/querybuilders/entity_query.go b/official/v7/querybuilders/entity_query.go new file mode 100644 index 0000000..0f90ee4 --- /dev/null +++ b/official/v7/querybuilders/entity_query.go @@ -0,0 +1,6 @@ +package querybuilders + +// Query is an interface for every type of Elasticsearch's query. +type Query interface { + Source() (interface{}, error) +} diff --git a/official/v7/querybuilders/script.go b/official/v7/querybuilders/script.go new file mode 100644 index 0000000..d4c6e51 --- /dev/null +++ b/official/v7/querybuilders/script.go @@ -0,0 +1,103 @@ +package querybuilders + +import ( + "encoding/json" + "fmt" + "strings" +) + +// Script holds all the parameters necessary to compile or find in cache +// and then execute a script. +// +// See https://www.elastic.co/guide/en/elasticsearch/reference/7.0/modules-scripting.html +// for details of scripting. +type Script struct { + script string + typ string + lang string + params map[string]interface{} +} + +// NewScript creates and initializes a new Script. By default, it is of +// type "inline". Use NewScriptStored for a stored script (where type is "id"). +func NewScript(script string) *Script { + return &Script{ + script: script, + typ: "inline", + params: make(map[string]interface{}), + } +} + +// Script is either the cache key of the script to be compiled/executed +// or the actual script source code for inline scripts. For indexed +// scripts this is the id used in the request. For file scripts this is +// the file name. +func (s *Script) Script(script string) *Script { + s.script = script + return s +} + +// Type sets the type of script: "inline" or "id". +func (s *Script) Type(typ string) *Script { + s.typ = typ + return s +} + +// Lang sets the language of the script. The default scripting language +// is Painless ("painless"). +// See https://www.elastic.co/guide/en/elasticsearch/reference/7.0/modules-scripting.html +// for details. +func (s *Script) Lang(lang string) *Script { + s.lang = lang + return s +} + +// Param adds a key/value pair to the parameters that this script will be executed with. +func (s *Script) Param(name string, value interface{}) *Script { + if s.params == nil { + s.params = make(map[string]interface{}) + } + s.params[name] = value + return s +} + +// Params sets the map of parameters this script will be executed with. +func (s *Script) Params(params map[string]interface{}) *Script { + s.params = params + return s +} + +// Source returns the JSON serializable data for this Script. +func (s *Script) Source() (interface{}, error) { + if s.typ == "" && s.lang == "" && len(s.params) == 0 { + return s.script, nil + } + source := make(map[string]interface{}) + // Beginning with 6.0, the type can only be "source" or "id" + if s.typ == "" || s.typ == "inline" { + src := s.rawScriptSource(s.script) + source["source"] = src + } else { + source["id"] = s.script + } + if s.lang != "" { + source["lang"] = s.lang + } + if len(s.params) > 0 { + source["params"] = s.params + } + return source, nil +} + +// rawScriptSource returns an embeddable script. If it uses a short +// script form, e.g. "ctx._source.likes++" (without the quotes), it +// is quoted. Otherwise it returns the raw script that will be directly +// embedded into the JSON data. +func (s *Script) rawScriptSource(script string) interface{} { + v := strings.TrimSpace(script) + if !strings.HasPrefix(v, "{") && !strings.HasPrefix(v, `"`) { + v = fmt.Sprintf("%q", v) + } + raw := json.RawMessage(v) + return &raw +} diff --git a/official/v7/querybuilders/search_queries_bool.go b/official/v7/querybuilders/search_queries_bool.go new file mode 100644 index 0000000..35d9378 --- /dev/null +++ b/official/v7/querybuilders/search_queries_bool.go @@ -0,0 +1,199 @@ +package querybuilders + +import "fmt" + +// A bool query matches documents matching boolean +// combinations of other queries. +// For more details, see: +// https://www.elastic.co/guide/en/elasticsearch/reference/7.0/query-dsl-bool-query.html +type BoolQuery struct { + Query + mustClauses []Query + mustNotClauses []Query + filterClauses []Query + shouldClauses []Query + boost *float64 + minimumShouldMatch string + adjustPureNegative *bool + queryName string +} + +// Creates a new bool query. +func NewBoolQuery() *BoolQuery { + return &BoolQuery{ + mustClauses: make([]Query, 0), + mustNotClauses: make([]Query, 0), + filterClauses: make([]Query, 0), + shouldClauses: make([]Query, 0), + } +} + +func (q *BoolQuery) Must(queries ...Query) *BoolQuery { + q.mustClauses = append(q.mustClauses, queries...) + return q +} + +func (q *BoolQuery) MustNot(queries ...Query) *BoolQuery { + q.mustNotClauses = append(q.mustNotClauses, queries...) + return q +} + +func (q *BoolQuery) Filter(filters ...Query) *BoolQuery { + q.filterClauses = append(q.filterClauses, filters...) + return q +} + +func (q *BoolQuery) Should(queries ...Query) *BoolQuery { + q.shouldClauses = append(q.shouldClauses, queries...) + return q +} + +func (q *BoolQuery) Boost(boost float64) *BoolQuery { + q.boost = &boost + return q +} + +func (q *BoolQuery) MinimumShouldMatch(minimumShouldMatch string) *BoolQuery { + q.minimumShouldMatch = minimumShouldMatch + return q +} + +func (q *BoolQuery) MinimumNumberShouldMatch(minimumNumberShouldMatch int) *BoolQuery { + q.minimumShouldMatch = fmt.Sprintf("%d", minimumNumberShouldMatch) + return q +} + +func (q *BoolQuery) AdjustPureNegative(adjustPureNegative bool) *BoolQuery { + q.adjustPureNegative = &adjustPureNegative + return q +} + +func (q *BoolQuery) QueryName(queryName string) *BoolQuery { + q.queryName = queryName + return q +} + +// Creates the query source for the bool query. +func (q *BoolQuery) Source() (interface{}, error) { + // { + // "bool" : { + // "must" : { + // "term" : { "user" : "kimchy" } + // }, + // "must_not" : { + // "range" : { + // "age" : { "from" : 10, "to" : 20 } + // } + // }, + // "filter" : [ + // ... + // ] + // "should" : [ + // { + // "term" : { "tag" : "wow" } + // }, + // { + // "term" : { "tag" : "elasticsearch" } + // } + // ], + // "minimum_should_match" : 1, + // "boost" : 1.0 + // } + // } + + query := make(map[string]interface{}) + + boolClause := make(map[string]interface{}) + query["bool"] = boolClause + + // must + if len(q.mustClauses) == 1 { + src, err := q.mustClauses[0].Source() + if err != nil { + return nil, err + } + boolClause["must"] = src + } else if len(q.mustClauses) > 1 { + var clauses []interface{} + for _, subQuery := range q.mustClauses { + src, err := subQuery.Source() + if err != nil { + return nil, err + } + clauses = append(clauses, src) + } + boolClause["must"] = clauses + } + + // must_not + if len(q.mustNotClauses) == 1 { + src, err := q.mustNotClauses[0].Source() + if err != nil { + return nil, err + } + boolClause["must_not"] = src + } else if len(q.mustNotClauses) > 1 { + var clauses []interface{} + for _, subQuery := range q.mustNotClauses { + src, err := subQuery.Source() + if err != nil { + return nil, err + } + clauses = append(clauses, src) + } + boolClause["must_not"] = clauses + } + + // filter + if len(q.filterClauses) == 1 { + src, err := q.filterClauses[0].Source() + if err != nil { + return nil, err + } + boolClause["filter"] = src + } else if len(q.filterClauses) > 1 { + var clauses []interface{} + for _, subQuery := range q.filterClauses { + src, err := subQuery.Source() + if err != nil { + return nil, err + } + clauses = append(clauses, src) + } + boolClause["filter"] = clauses + } + + // should + if len(q.shouldClauses) == 1 { + src, err := q.shouldClauses[0].Source() + if err != nil { + return nil, err + } + boolClause["should"] = src + } else if len(q.shouldClauses) > 1 { + var clauses []interface{} + for _, subQuery := range q.shouldClauses { + src, err := subQuery.Source() + if err != nil { + return nil, err + } + clauses = append(clauses, src) + } + boolClause["should"] = clauses + } + + if q.boost != nil { + boolClause["boost"] = *q.boost + } + if q.minimumShouldMatch != "" { + boolClause["minimum_should_match"] = q.minimumShouldMatch + } + if q.adjustPureNegative != nil { + boolClause["adjust_pure_negative"] = *q.adjustPureNegative + } + if q.queryName != "" { + boolClause["_name"] = q.queryName + } + + return query, nil +} diff --git a/official/v7/querybuilders/search_queries_exists.go b/official/v7/querybuilders/search_queries_exists.go new file mode 100644 index 0000000..43736c4 --- /dev/null +++ b/official/v7/querybuilders/search_queries_exists.go @@ -0,0 +1,45 @@ +package querybuilders + +// ExistsQuery is a query that only matches on documents that the field +// has a value in them. +// +// For more details, see: +// https://www.elastic.co/guide/en/elasticsearch/reference/7.0/query-dsl-exists-query.html +type ExistsQuery struct { + name string + queryName string +} + +// NewExistsQuery creates and initializes a new exists query. +func NewExistsQuery(name string) *ExistsQuery { + return &ExistsQuery{ + name: name, + } +} + +// QueryName sets the query name for the filter that can be used +// when searching for matched queries per hit. +func (q *ExistsQuery) QueryName(queryName string) *ExistsQuery { + q.queryName = queryName + return q +} + +// Source returns the JSON serializable content for this query. +func (q *ExistsQuery) Source() (interface{}, error) { + // { + // "exists" : { + // "field" : "user" + // } + // } + + query := make(map[string]interface{}) + params := make(map[string]interface{}) + query["exists"] = params + + params["field"] = q.name + if q.queryName != "" { + params["_name"] = q.queryName + } + + return query, nil +} diff --git a/official/v7/querybuilders/search_queries_match_all.go b/official/v7/querybuilders/search_queries_match_all.go new file mode 100644 index 0000000..278b2ab --- /dev/null +++ b/official/v7/querybuilders/search_queries_match_all.go @@ -0,0 +1,61 @@ +package querybuilders + +import "encoding/json" + +// MatchAllQuery is the most simple query, which matches all documents, +// giving them all a _score of 1.0. +// +// For more details, see +// https://www.elastic.co/guide/en/elasticsearch/reference/7.0/query-dsl-match-all-query.html +type MatchAllQuery struct { + boost *float64 + queryName string +} + +// NewMatchAllQuery creates and initializes a new match all query. +func NewMatchAllQuery() *MatchAllQuery { + return &MatchAllQuery{} +} + +// Boost sets the boost for this query. Documents matching this query will +// (in addition to the normal weightings) have their score multiplied by the +// boost provided. +func (q *MatchAllQuery) Boost(boost float64) *MatchAllQuery { + q.boost = &boost + return q +} + +// QueryName sets the query name. +func (q *MatchAllQuery) QueryName(name string) *MatchAllQuery { + q.queryName = name + return q +} + +// Source returns JSON for the match all query. +func (q *MatchAllQuery) Source() (interface{}, error) { + // { + // "match_all" : { ... } + // } + source := make(map[string]interface{}) + params := make(map[string]interface{}) + source["match_all"] = params + if q.boost != nil { + params["boost"] = *q.boost + } + if q.queryName != "" { + params["_name"] = q.queryName + } + return source, nil +} + +// MarshalJSON enables serializing the type as JSON. +func (q *MatchAllQuery) MarshalJSON() ([]byte, error) { + if q == nil { + return []byte{}, nil + } + src, err := q.Source() + if err != nil { + return nil, err + } + return json.Marshal(src) +} diff --git a/official/v7/querybuilders/search_queries_multi_match.go b/official/v7/querybuilders/search_queries_multi_match.go new file mode 100644 index 0000000..cda73c2 --- /dev/null +++ b/official/v7/querybuilders/search_queries_multi_match.go @@ -0,0 +1,266 @@ +package querybuilders + +import ( + "fmt" + "strings" +) + +// MultiMatchQuery builds on the MatchQuery to allow multi-field queries. +// +// For more details, see +// https://www.elastic.co/guide/en/elasticsearch/reference/7.0/query-dsl-multi-match-query.html +type MultiMatchQuery struct { + text interface{} + fields []string + fieldBoosts map[string]*float64 + typ string // best_fields, boolean, most_fields, cross_fields, phrase, phrase_prefix + operator string // AND or OR + analyzer string + boost *float64 + slop *int + fuzziness string + prefixLength *int + maxExpansions *int + minimumShouldMatch string + rewrite string + fuzzyRewrite string + tieBreaker *float64 + lenient *bool + cutoffFrequency *float64 + zeroTermsQuery string + queryName string +} + +// MultiMatchQuery creates and initializes a new MultiMatchQuery. +func NewMultiMatchQuery(text interface{}, fields ...string) *MultiMatchQuery { + q := &MultiMatchQuery{ + text: text, + fieldBoosts: make(map[string]*float64), + } + q.fields = append(q.fields, fields...) + return q +} + +// Field adds a field to run the multi match against. +func (q *MultiMatchQuery) Field(field string) *MultiMatchQuery { + q.fields = append(q.fields, field) + return q +} + +// FieldWithBoost adds a field to run the multi match against with a specific boost. +func (q *MultiMatchQuery) FieldWithBoost(field string, boost float64) *MultiMatchQuery { + q.fields = append(q.fields, field) + q.fieldBoosts[field] = &boost + return q +} + +// Type can be "best_fields", "boolean", "most_fields", "cross_fields", +// "phrase", "phrase_prefix" or "bool_prefix" +func (q *MultiMatchQuery) Type(typ string) *MultiMatchQuery { + switch strings.ToLower(typ) { + default: // best_fields / boolean + q.typ = "best_fields" + case "most_fields": + q.typ = "most_fields" + case "cross_fields": + q.typ = "cross_fields" + case "phrase": + q.typ = "phrase" + case "phrase_prefix": + q.typ = "phrase_prefix" + case "bool_prefix": + q.typ = "bool_prefix" + } + return q +} + +// Operator sets the operator to use when using boolean query. +// It can be either AND or OR (default). +func (q *MultiMatchQuery) Operator(operator string) *MultiMatchQuery { + q.operator = operator + return q +} + +// Analyzer sets the analyzer to use explicitly. It defaults to use explicit +// mapping config for the field, or, if not set, the default search analyzer. +func (q *MultiMatchQuery) Analyzer(analyzer string) *MultiMatchQuery { + q.analyzer = analyzer + return q +} + +// Boost sets the boost for this query. +func (q *MultiMatchQuery) Boost(boost float64) *MultiMatchQuery { + q.boost = &boost + return q +} + +// Slop sets the phrase slop if evaluated to a phrase query type. +func (q *MultiMatchQuery) Slop(slop int) *MultiMatchQuery { + q.slop = &slop + return q +} + +// Fuzziness sets the fuzziness used when evaluated to a fuzzy query type. +// It defaults to "AUTO". +func (q *MultiMatchQuery) Fuzziness(fuzziness string) *MultiMatchQuery { + q.fuzziness = fuzziness + return q +} + +// PrefixLength for the fuzzy process. +func (q *MultiMatchQuery) PrefixLength(prefixLength int) *MultiMatchQuery { + q.prefixLength = &prefixLength + return q +} + +// MaxExpansions is the number of term expansions to use when using fuzzy +// or prefix type query. It defaults to unbounded so it's recommended +// to set it to a reasonable value for faster execution. +func (q *MultiMatchQuery) MaxExpansions(maxExpansions int) *MultiMatchQuery { + q.maxExpansions = &maxExpansions + return q +} + +// MinimumShouldMatch represents the minimum number of optional should clauses +// to match. +func (q *MultiMatchQuery) MinimumShouldMatch(minimumShouldMatch string) *MultiMatchQuery { + q.minimumShouldMatch = minimumShouldMatch + return q +} + +func (q *MultiMatchQuery) Rewrite(rewrite string) *MultiMatchQuery { + q.rewrite = rewrite + return q +} + +func (q *MultiMatchQuery) FuzzyRewrite(fuzzyRewrite string) *MultiMatchQuery { + q.fuzzyRewrite = fuzzyRewrite + return q +} + +// TieBreaker for "best-match" disjunction queries (OR queries). +// The tie breaker capability allows documents that match more than one +// query clause (in this case on more than one field) to be scored better +// than documents that match only the best of the fields, without confusing +// this with the better case of two distinct matches in the multiple fields. +// +// A tie-breaker value of 1.0 is interpreted as a signal to score queries as +// "most-match" queries where all matching query clauses are considered for scoring. +func (q *MultiMatchQuery) TieBreaker(tieBreaker float64) *MultiMatchQuery { + q.tieBreaker = &tieBreaker + return q +} + +// Lenient indicates whether format based failures will be ignored. +func (q *MultiMatchQuery) Lenient(lenient bool) *MultiMatchQuery { + q.lenient = &lenient + return q +} + +// CutoffFrequency sets a cutoff value in [0..1] (or absolute number >=1) +// representing the maximum threshold of a terms document frequency to be +// considered a low frequency term. +func (q *MultiMatchQuery) CutoffFrequency(cutoff float64) *MultiMatchQuery { + q.cutoffFrequency = &cutoff + return q +} + +// ZeroTermsQuery can be "all" or "none". +func (q *MultiMatchQuery) ZeroTermsQuery(zeroTermsQuery string) *MultiMatchQuery { + q.zeroTermsQuery = zeroTermsQuery + return q +} + +// QueryName sets the query name for the filter that can be used when +// searching for matched filters per hit. +func (q *MultiMatchQuery) QueryName(queryName string) *MultiMatchQuery { + q.queryName = queryName + return q +} + +// Source returns JSON for the query. +func (q *MultiMatchQuery) Source() (interface{}, error) { + // + // { + // "multi_match" : { + // "query" : "this is a test", + // "fields" : [ "subject", "message" ] + // } + // } + + source := make(map[string]interface{}) + + multiMatch := make(map[string]interface{}) + source["multi_match"] = multiMatch + + multiMatch["query"] = q.text + + var fields []string + for _, field := range q.fields { + if boost, found := q.fieldBoosts[field]; found { + if boost != nil { + fields = append(fields, fmt.Sprintf("%s^%f", field, *boost)) + } else { + fields = append(fields, field) + } + } else { + fields = append(fields, field) + } + } + if fields == nil { + multiMatch["fields"] = []string{} + } else { + multiMatch["fields"] = fields + } + + if q.typ != "" { + multiMatch["type"] = q.typ + } + + if q.operator != "" { + multiMatch["operator"] = q.operator + } + if q.analyzer != "" { + multiMatch["analyzer"] = q.analyzer + } + if q.boost != nil { + multiMatch["boost"] = *q.boost + } + if q.slop != nil { + multiMatch["slop"] = *q.slop + } + if q.fuzziness != "" { + multiMatch["fuzziness"] = q.fuzziness + } + if q.prefixLength != nil { + multiMatch["prefix_length"] = *q.prefixLength + } + if q.maxExpansions != nil { + multiMatch["max_expansions"] = *q.maxExpansions + } + if q.minimumShouldMatch != "" { + multiMatch["minimum_should_match"] = q.minimumShouldMatch + } + if q.rewrite != "" { + multiMatch["rewrite"] = q.rewrite + } + if q.fuzzyRewrite != "" { + multiMatch["fuzzy_rewrite"] = q.fuzzyRewrite + } + if q.tieBreaker != nil { + multiMatch["tie_breaker"] = *q.tieBreaker + } + if q.lenient != nil { + multiMatch["lenient"] = *q.lenient + } + if q.cutoffFrequency != nil { + multiMatch["cutoff_frequency"] = *q.cutoffFrequency + } + if q.zeroTermsQuery != "" { + multiMatch["zero_terms_query"] = q.zeroTermsQuery + } + if q.queryName != "" { + multiMatch["_name"] = q.queryName + } + return source, nil +} diff --git a/official/v7/querybuilders/search_queries_nested.go b/official/v7/querybuilders/search_queries_nested.go new file mode 100644 index 0000000..4d4d301 --- /dev/null +++ b/official/v7/querybuilders/search_queries_nested.go @@ -0,0 +1,77 @@ +package querybuilders + +// NestedQuery allows to query nested objects / docs. +// The query is executed against the nested objects / docs as if they were +// indexed as separate docs (they are, internally) and resulting in the +// root parent doc (or parent nested mapping). +// +// For more details, see +// https://www.elastic.co/guide/en/elasticsearch/reference/7.0/query-dsl-nested-query.html +type NestedQuery struct { + query Query + path string + scoreMode string + boost *float64 + queryName string + ignoreUnmapped *bool +} + +// NewNestedQuery creates and initializes a new NestedQuery. +func NewNestedQuery(path string, query Query) *NestedQuery { + return &NestedQuery{path: path, query: query} +} + +// ScoreMode specifies the score mode. +func (q *NestedQuery) ScoreMode(scoreMode string) *NestedQuery { + q.scoreMode = scoreMode + return q +} + +// Boost sets the boost for this query. +func (q *NestedQuery) Boost(boost float64) *NestedQuery { + q.boost = &boost + return q +} + +// QueryName sets the query name for the filter that can be used +// when searching for matched_filters per hit +func (q *NestedQuery) QueryName(queryName string) *NestedQuery { + q.queryName = queryName + return q +} + +// IgnoreUnmapped sets the ignore_unmapped option for the filter that ignores +// unmapped nested fields +func (q *NestedQuery) IgnoreUnmapped(value bool) *NestedQuery { + q.ignoreUnmapped = &value + return q +} + +// Source returns JSON for the query. +func (q *NestedQuery) Source() (interface{}, error) { + query := make(map[string]interface{}) + nq := make(map[string]interface{}) + query["nested"] = nq + + src, err := q.query.Source() + if err != nil { + return nil, err + } + nq["query"] = src + + nq["path"] = q.path + + if q.scoreMode != "" { + nq["score_mode"] = q.scoreMode + } + if q.boost != nil { + nq["boost"] = *q.boost + } + if q.queryName != "" { + nq["_name"] = q.queryName + } + if q.ignoreUnmapped != nil { + nq["ignore_unmapped"] = *q.ignoreUnmapped + } + return query, nil +} diff --git a/official/v7/querybuilders/search_queries_range.go b/official/v7/querybuilders/search_queries_range.go new file mode 100644 index 0000000..53c8402 --- /dev/null +++ b/official/v7/querybuilders/search_queries_range.go @@ -0,0 +1,151 @@ +package querybuilders + +// RangeQuery matches documents with fields that have terms within a certain range. +// +// For details, see +// https://www.elastic.co/guide/en/elasticsearch/reference/7.0/query-dsl-range-query.html +type RangeQuery struct { + name string + from interface{} + to interface{} + timeZone string + includeLower bool + includeUpper bool + boost *float64 + queryName string + format string + relation string +} + +// NewRangeQuery creates and initializes a new RangeQuery. +func NewRangeQuery(name string) *RangeQuery { + return &RangeQuery{name: name, includeLower: true, includeUpper: true} +} + +// From indicates the from part of the RangeQuery. +// Use nil to indicate an unbounded from part. +func (q *RangeQuery) From(from interface{}) *RangeQuery { + q.from = from + return q +} + +// Gt indicates a greater-than value for the from part. +// Use nil to indicate an unbounded from part. +func (q *RangeQuery) Gt(from interface{}) *RangeQuery { + q.from = from + q.includeLower = false + return q +} + +// Gte indicates a greater-than-or-equal value for the from part. +// Use nil to indicate an unbounded from part. +func (q *RangeQuery) Gte(from interface{}) *RangeQuery { + q.from = from + q.includeLower = true + return q +} + +// To indicates the to part of the RangeQuery. +// Use nil to indicate an unbounded to part. +func (q *RangeQuery) To(to interface{}) *RangeQuery { + q.to = to + return q +} + +// Lt indicates a less-than value for the to part. +// Use nil to indicate an unbounded to part. +func (q *RangeQuery) Lt(to interface{}) *RangeQuery { + q.to = to + q.includeUpper = false + return q +} + +// Lte indicates a less-than-or-equal value for the to part. +// Use nil to indicate an unbounded to part. +func (q *RangeQuery) Lte(to interface{}) *RangeQuery { + q.to = to + q.includeUpper = true + return q +} + +// IncludeLower indicates whether the lower bound should be included or not. +// Defaults to true. +func (q *RangeQuery) IncludeLower(includeLower bool) *RangeQuery { + q.includeLower = includeLower + return q +} + +// IncludeUpper indicates whether the upper bound should be included or not. +// Defaults to true. +func (q *RangeQuery) IncludeUpper(includeUpper bool) *RangeQuery { + q.includeUpper = includeUpper + return q +} + +// Boost sets the boost for this query. +func (q *RangeQuery) Boost(boost float64) *RangeQuery { + q.boost = &boost + return q +} + +// QueryName sets the query name for the filter that can be used when +// searching for matched_filters per hit. +func (q *RangeQuery) QueryName(queryName string) *RangeQuery { + q.queryName = queryName + return q +} + +// TimeZone is used for date fields. In that case, we can adjust the +// from/to fields using a timezone. +func (q *RangeQuery) TimeZone(timeZone string) *RangeQuery { + q.timeZone = timeZone + return q +} + +// Format is used for date fields. In that case, we can set the format +// to be used instead of the mapper format. +func (q *RangeQuery) Format(format string) *RangeQuery { + q.format = format + return q +} + +// Relation is used for range fields. which can be one of +// "within", "contains", "intersects" (default) and "disjoint". +func (q *RangeQuery) Relation(relation string) *RangeQuery { + q.relation = relation + return q +} + +// Source returns JSON for the query. +func (q *RangeQuery) Source() (interface{}, error) { + source := make(map[string]interface{}) + + rangeQ := make(map[string]interface{}) + source["range"] = rangeQ + + params := make(map[string]interface{}) + rangeQ[q.name] = params + + params["from"] = q.from + params["to"] = q.to + if q.timeZone != "" { + params["time_zone"] = q.timeZone + } + if q.format != "" { + params["format"] = q.format + } + if q.relation != "" { + params["relation"] = q.relation + } + if q.boost != nil { + params["boost"] = *q.boost + } + params["include_lower"] = q.includeLower + params["include_upper"] = q.includeUpper + + if q.queryName != "" { + rangeQ["_name"] = q.queryName + } + + return source, nil +} diff --git a/official/v7/querybuilders/search_queries_term.go b/official/v7/querybuilders/search_queries_term.go new file mode 100644 index 0000000..bce6e47 --- /dev/null +++ b/official/v7/querybuilders/search_queries_term.go @@ -0,0 +1,63 @@ +package querybuilders + +// TermQuery finds documents that contain the exact term specified +// in the inverted index. +// +// For details, see +// https://www.elastic.co/guide/en/elasticsearch/reference/7.0/query-dsl-term-query.html +type TermQuery struct { + name string + value interface{} + boost *float64 + caseInsensitive *bool + queryName string +} + +// NewTermQuery creates and initializes a new TermQuery. +func NewTermQuery(name string, value interface{}) *TermQuery { + return &TermQuery{name: name, value: value} +} + +// Boost sets the boost for this query. +func (q *TermQuery) Boost(boost float64) *TermQuery { + q.boost = &boost + return q +} + +func (q *TermQuery) CaseInsensitive(caseInsensitive bool) *TermQuery { + q.caseInsensitive = &caseInsensitive + return q +} + +// QueryName sets the query name for the filter that can be used +// when searching for matched_filters per hit +func (q *TermQuery) QueryName(queryName string) *TermQuery { + q.queryName = queryName + return q +} + +// Source returns JSON for the query. +func (q *TermQuery) Source() (interface{}, error) { + // {"term":{"name":"value"}} + source := make(map[string]interface{}) + tq := make(map[string]interface{}) + source["term"] = tq + + if q.boost == nil && q.caseInsensitive == nil && q.queryName == "" { + tq[q.name] = q.value + } else { + subQ := make(map[string]interface{}) + subQ["value"] = q.value + if q.boost != nil { + subQ["boost"] = *q.boost + } + if q.caseInsensitive != nil { + subQ["case_insensitive"] = *q.caseInsensitive + } + if q.queryName != "" { + subQ["_name"] = q.queryName + } + tq[q.name] = subQ + } + return source, nil +} diff --git a/official/v7/querybuilders/search_queries_terms.go b/official/v7/querybuilders/search_queries_terms.go new file mode 100644 index 0000000..e2d5d87 --- /dev/null +++ b/official/v7/querybuilders/search_queries_terms.go @@ -0,0 +1,84 @@ +package querybuilders + +// TermsQuery filters documents that have fields that match any +// of the provided terms (not analyzed). +// +// For more details, see +// https://www.elastic.co/guide/en/elasticsearch/reference/7.0/query-dsl-terms-query.html +type TermsQuery struct { + name string + values []interface{} + termsLookup *TermsLookup + queryName string + boost *float64 +} + +// NewTermsQuery creates and initializes a new TermsQuery. +func NewTermsQuery(name string, values ...interface{}) *TermsQuery { + q := &TermsQuery{ + name: name, + values: make([]interface{}, 0), + } + if len(values) > 0 { + q.values = append(q.values, values...) + } + return q +} + +// NewTermsQueryFromStrings creates and initializes a new TermsQuery +// from strings. +func NewTermsQueryFromStrings(name string, values ...string) *TermsQuery { + q := &TermsQuery{ + name: name, + values: make([]interface{}, 0), + } + for _, v := range values { + q.values = append(q.values, v) + } + return q +} + +// TermsLookup adds terms lookup details to the query. +func (q *TermsQuery) TermsLookup(lookup *TermsLookup) *TermsQuery { + q.termsLookup = lookup + return q +} + +// Boost sets the boost for this query. +func (q *TermsQuery) Boost(boost float64) *TermsQuery { + q.boost = &boost + return q +} + +// QueryName sets the query name for the filter that can be used +// when searching for matched_filters per hit +func (q *TermsQuery) QueryName(queryName string) *TermsQuery { + q.queryName = queryName + return q +} + +// Creates the query source for the term query. +func (q *TermsQuery) Source() (interface{}, error) { + // {"terms":{"name":["value1","value2"]}} + source := make(map[string]interface{}) + params := make(map[string]interface{}) + source["terms"] = params + + if q.termsLookup != nil { + src, err := q.termsLookup.Source() + if err != nil { + return nil, err + } + params[q.name] = src + } else { + params[q.name] = q.values + if q.boost != nil { + params["boost"] = *q.boost + } + if q.queryName != "" { + params["_name"] = q.queryName + } + } + + return source, nil +} diff --git a/official/v7/querybuilders/search_queries_terms_set.go b/official/v7/querybuilders/search_queries_terms_set.go new file mode 100644 index 0000000..38cfb2a --- /dev/null +++ b/official/v7/querybuilders/search_queries_terms_set.go @@ -0,0 +1,93 @@ +package querybuilders + +// TermsSetQuery returns any documents that match with at least +// one or more of the provided terms. The terms are not analyzed +// and thus must match exactly. The number of terms that must +// match varies per document and is either controlled by a +// minimum should match field or computed per document in a +// minimum should match script. +// +// For more details, see +// https://www.elastic.co/guide/en/elasticsearch/reference/7.0/query-dsl-terms-set-query.html +type TermsSetQuery struct { + name string + values []interface{} + minimumShouldMatchField string + minimumShouldMatchScript *Script + queryName string + boost *float64 +} + +// NewTermsSetQuery creates and initializes a new TermsSetQuery. +func NewTermsSetQuery(name string, values ...interface{}) *TermsSetQuery { + q := &TermsSetQuery{ + name: name, + values: make([]interface{}, 0), + } + if len(values) > 0 { + q.values = append(q.values, values...) + } + return q +} + +// MinimumShouldMatchField specifies the field to match. +func (q *TermsSetQuery) MinimumShouldMatchField(minimumShouldMatchField string) *TermsSetQuery { + q.minimumShouldMatchField = minimumShouldMatchField + return q +} + +// MinimumShouldMatchScript specifies the script to match. +func (q *TermsSetQuery) MinimumShouldMatchScript(minimumShouldMatchScript *Script) *TermsSetQuery { + q.minimumShouldMatchScript = minimumShouldMatchScript + return q +} + +// Boost sets the boost for this query. +func (q *TermsSetQuery) Boost(boost float64) *TermsSetQuery { + q.boost = &boost + return q +} + +// QueryName sets the query name for the filter that can be used +// when searching for matched_filters per hit +func (q *TermsSetQuery) QueryName(queryName string) *TermsSetQuery { + q.queryName = queryName + return q +} + +// Source creates the query source for the term query. +func (q *TermsSetQuery) Source() (interface{}, error) { + // {"terms_set":{"codes":{"terms":["abc","def"],"minimum_should_match_field":"required_matches"}}} + source := make(map[string]interface{}) + inner := make(map[string]interface{}) + params := make(map[string]interface{}) + inner[q.name] = params + source["terms_set"] = inner + + // terms + params["terms"] = q.values + + // minimum_should_match_field + if match := q.minimumShouldMatchField; match != "" { + params["minimum_should_match_field"] = match + } + + // minimum_should_match_script + if match := q.minimumShouldMatchScript; match != nil { + src, err := match.Source() + if err != nil { + return nil, err + } + params["minimum_should_match_script"] = src + } + + // Common parameters for all queries + if q.boost != nil { + params["boost"] = *q.boost + } + if q.queryName != "" { + params["_name"] = q.queryName + } + + return source, nil +} diff --git a/official/v7/querybuilders/search_terms_lookup.go b/official/v7/querybuilders/search_terms_lookup.go new file mode 100644 index 0000000..2e02258 --- /dev/null +++ b/official/v7/querybuilders/search_terms_lookup.go @@ -0,0 +1,72 @@ +package querybuilders + +// TermsLookup encapsulates the parameters needed to fetch terms. +// +// For more details, see +// https://www.elastic.co/guide/en/elasticsearch/reference/7.0/query-dsl-terms-query.html#query-dsl-terms-lookup. +type TermsLookup struct { + index string + typ string + id string + path string + routing string +} + +// NewTermsLookup creates and initializes a new TermsLookup. +func NewTermsLookup() *TermsLookup { + t := &TermsLookup{} + return t +} + +// Index name. +func (t *TermsLookup) Index(index string) *TermsLookup { + t.index = index + return t +} + +// Type name. +// +// Deprecated: Types are in the process of being removed. +func (t *TermsLookup) Type(typ string) *TermsLookup { + t.typ = typ + return t +} + +// ID to look up. +func (t *TermsLookup) ID(id string) *TermsLookup { + t.id = id + return t +} + +// Path to use for lookup. +func (t *TermsLookup) Path(path string) *TermsLookup { + t.path = path + return t +} + +// Routing value. +func (t *TermsLookup) Routing(routing string) *TermsLookup { + t.routing = routing + return t +} + +// Source creates the JSON source of the builder. +func (t *TermsLookup) Source() (interface{}, error) { + src := make(map[string]interface{}) + if t.index != "" { + src["index"] = t.index + } + if t.typ != "" { + src["type"] = t.typ + } + if t.id != "" { + src["id"] = t.id + } + if t.path != "" { + src["path"] = t.path + } + if t.routing != "" { + src["routing"] = t.routing + } + return src, nil +} diff --git a/official/v7/search.go b/official/v7/search.go new file mode 100644 index 0000000..9da1e13 --- /dev/null +++ b/official/v7/search.go @@ -0,0 +1,86 @@ +package v7 + +import ( + "context" + "strings" + + "github.com/arquivei/foundationkit/errors" + "github.com/elastic/go-elasticsearch/v7/esapi" +) + +// SearchConfig hold all information to Search method. +type SearchConfig struct { + Indexes []string + Size int + Filter Filter + IgnoreUnavailable bool + AllowNoIndices bool + TrackTotalHits bool + Sort Sorters + SearchAfter string +} + +func (c *esClient) Search(ctx context.Context, config SearchConfig) (SearchResponse, error) { + const op = errors.Op("v7.Client.Search") + + enrichLogWithIndexes(ctx, config.Indexes) + + response, err := c.doSearch(ctx, config) + if err != nil { + return SearchResponse{}, errors.E(op, err) + } + defer response.Body.Close() + + parsedResponse, err := parseResponse(ctx, response) + if err != nil { + if errors.GetCode(err) == errors.CodeEmpty { + err = errors.E(err, ErrCodeUnexpectedResponse) + } + return SearchResponse{}, errors.E(op, err) + } + + return parsedResponse, nil +} + +func (c *esClient) doSearch(ctx context.Context, config SearchConfig) (*esapi.Response, error) { + const op = errors.Op("doSearch") + + queryString, err := getQuery(ctx, config) + if err != nil { + return nil, errors.E(op, err, ErrCodeBadRequest) + } + + response, err := c.client.Search( + c.client.Search.WithIndex(config.Indexes...), + c.client.Search.WithSize(config.Size), + c.client.Search.WithBody(strings.NewReader(queryString)), + c.client.Search.WithIgnoreUnavailable(config.IgnoreUnavailable), + c.client.Search.WithAllowNoIndices(config.AllowNoIndices), + c.client.Search.WithTrackTotalHits(config.TrackTotalHits), + c.client.Search.WithSort(config.Sort.Strings()...), + ) + if err != nil { + return nil, errors.E(op, err, ErrCodeBadGateway) + } + + err = checkErrorFromResponse(response) + if err != nil { + if errors.GetCode(err) == errors.CodeEmpty { + err = errors.E(err, ErrCodeBadGateway) + } + return nil, errors.E(op, err) + } + + return response, nil +} + +func getQuery(ctx context.Context, config SearchConfig) (string, error) { + query, err := buildElasticBoolQuery(config.Filter) + if err != nil { + return "", err + } + + queryString := queryWithSearchAfter(query, config.SearchAfter) + enrichLogWithQuery(ctx, queryString) + return queryString, nil +} diff --git a/official/v7/search_querybuilder.go b/official/v7/search_querybuilder.go new file mode 100644 index 0000000..a3e0bf0 --- /dev/null +++ b/official/v7/search_querybuilder.go @@ -0,0 +1,501 @@ +package v7 + +import ( + "encoding/json" + "fmt" + "reflect" + "strings" + + "github.com/arquivei/foundationkit/errors" + "github.com/rs/zerolog/log" + + "github.com/arquivei/elasticutil/official/v7/querybuilders" +) + +const maxExpansions = 1024 + +func buildElasticBoolQuery(filter Filter) (querybuilders.Query, error) { + const op = errors.Op("buildElasticBoolQuery") + + var mustQueries, mustNotQueries, existsQueries, notExistsQueries []querybuilders.Query + + if filter.Must != nil { + var err error + mustQueries, err = getMustQuery(filter.Must) + if err != nil { + return nil, errors.E(op, err) + } + } + + if filter.MustNot != nil { + var err error + mustNotQueries, err = getMustQuery(filter.MustNot) + if err != nil { + return nil, errors.E(op, err) + } + } + + if filter.Exists != nil { + var err error + existsQueries, notExistsQueries, err = getExistsQuery(filter.Exists) + if err != nil { + return nil, errors.E(op, err) + } + } + + if shouldReturnMatchAllQuery( + mustQueries, + mustNotQueries, + existsQueries, + notExistsQueries, + ) { + return querybuilders.NewMatchAllQuery(), nil + } + + if shouldReturnOnlyMustQuery( + mustQueries, + mustNotQueries, + existsQueries, + notExistsQueries, + ) { + return mustQueries[0], nil + } + + return getBoolQuery( + mustQueries, + mustNotQueries, + existsQueries, + notExistsQueries, + ), nil +} + +func marshalQuery(query querybuilders.Query) string { + if query != nil { + q, e := query.Source() + if e != nil { + return e.Error() + } + qs, e := json.Marshal(q) + if e != nil { + return e.Error() + } + return string(qs) + } + return "" +} + +// nolint: funlen, cyclop, gocognit +func getMustQuery(filter interface{}) ([]querybuilders.Query, error) { + const op = errors.Op("getMustQuery") + + var queries []querybuilders.Query + + rv := reflect.ValueOf(filter) + if rv.Kind() != reflect.Struct { + return nil, errors.E(op, filterMustBeAStructError(rv.Kind().String())) + } + + nFields := rv.Type().NumField() + for i := 0; i < nFields; i++ { + // Get value, type and tags + fvalue := rv.Field(i) + ftype := rv.Type().Field(i) + fnames := parseFieldNames(ftype.Tag.Get("es")) + + // Skip zero values + if fvalue.IsZero() { + continue + } + + // Rename field if specified a new name inside the tag + structFieldName := ftype.Name + names := []string{structFieldName} + if len(fnames) > 0 { + names = fnames + } + + // Fix kind and value if field is a pointer + fkind := ftype.Type.Kind() + if fkind == reflect.Ptr { + fvalue = fvalue.Elem() + fkind = fvalue.Kind() + } + + switch fkind { + case reflect.Slice: + if rv.Field(i).Len() == 0 { + continue + } + switch ftype.Type.Elem().String() { + case "string": + queries = append(queries, querybuilders.NewTermsQuery(names[0], + extractSliceFromInterface[string](fvalue.Interface())...)) + case "uint64": + queries = append(queries, querybuilders.NewTermsQuery(names[0], + extractSliceFromInterface[uint64](fvalue.Interface())...)) + } + case reflect.Bool: + queries = append(queries, querybuilders.NewTermQuery(names[0], + fvalue.Bool())) + case reflect.Struct: + switch v := fvalue.Interface().(type) { + case TimeRange: + queries = append(queries, getRangeQuery(v.From, v.To, names[0])) + case FloatRange: + queries = append(queries, getRangeQuery(v.From, v.To, names[0])) + case IntRange: + queries = append(queries, getRangeQuery(v.From, v.To, names[0])) + case Nested: + var err error + queries, err = getMustNestedQuery(v.payload, names[0], queries) + if err != nil { + return nil, errors.E(op, err) + } + case FullTextSearchShould: + boolQuery, err := getFullTextSearchShouldQuery( + v.payload, + structFieldName, + names, + ) + if err != nil { + return nil, errors.E(op, err) + } + queries = append(queries, boolQuery) + case FullTextSearchMust: + boolQuery, err := getFullTextSearchMustQuery( + v.payload, + structFieldName, + names, + ) + if err != nil { + return nil, errors.E(op, err) + } + queries = append(queries, boolQuery) + case MultiMatchSearchShould: + boolQuery, err := getMultiMatchSearchShouldQuery( + v.payload, + structFieldName, + names, + ) + if err != nil { + return nil, errors.E(op, err) + } + queries = append(queries, boolQuery) + case CustomSearch: + boolQuery, err := v.GetQuery() + if err != nil { + return nil, errors.E(op, err) + } + queries = append(queries, boolQuery) + default: + return nil, errors.E(op, structNotSupportedError(names[0])) + } + default: + return nil, errors.E(op, typeNotSupportedError(structFieldName, fkind.String())) + } + } + + return queries, nil +} + +// nolint: funlen, cyclop +func getExistsQuery( + filter interface{}, +) (existsQueries, notExistsQueries []querybuilders.Query, err error) { + const op = errors.Op("getExistsQuery") + + rv := reflect.ValueOf(filter) + if rv.Kind() != reflect.Struct { + return nil, nil, errors.E(op, filterMustBeAStructError(rv.Kind().String())) + } + + nFields := rv.Type().NumField() + for i := 0; i < nFields; i++ { + // Get value, type and tags + fvalue := rv.Field(i) + ftype := rv.Type().Field(i) + fnames := parseFieldNames(ftype.Tag.Get("es")) + + // Skip zero values + if fvalue.IsZero() { + continue + } + + // Rename field if specified a new name inside the tag + structFieldName := ftype.Name + names := []string{structFieldName} + if len(fnames) > 0 { + names = fnames + } + + // Fix kind and value if field is a pointer + fkind := ftype.Type.Kind() + if fkind == reflect.Ptr { + fvalue = fvalue.Elem() + fkind = fvalue.Kind() + } + + switch fkind { + case reflect.Bool: + if fvalue.Bool() { + existsQueries = append(existsQueries, querybuilders.NewExistsQuery(names[0])) + } else { + notExistsQueries = append(notExistsQueries, querybuilders.NewExistsQuery(names[0])) + } + case reflect.Struct: + switch v := fvalue.Interface().(type) { + case Nested: + var err error + existsQueries, notExistsQueries, err = getExistsNestedQuery( + v.payload, + names[0], + existsQueries, + notExistsQueries, + ) + if err != nil { + return nil, nil, errors.E(op, err) + } + default: + return nil, nil, errors.E(op, structNotSupportedError(names[0])) + } + default: + return nil, nil, errors.E(op, + typeNotSupportedError(structFieldName, fkind.String())) + } + } + + return existsQueries, notExistsQueries, nil +} + +func shouldReturnMatchAllQuery( + mustQueries, + mustNotQueries, + existsQueries, + notExistsQueries []querybuilders.Query, +) bool { + return len(mustNotQueries) == 0 && + len(existsQueries) == 0 && + len(notExistsQueries) == 0 && + len(mustQueries) == 0 +} + +func shouldReturnOnlyMustQuery( + mustQueries, + mustNotQueries, + existsQueries, + notExistsQueries []querybuilders.Query, +) bool { + return len(mustNotQueries) == 0 && + len(existsQueries) == 0 && + len(notExistsQueries) == 0 && + len(mustQueries) == 1 +} + +func getBoolQuery( + mustQueries, + mustNotQueries, + existsQueries, + notExistsQueries []querybuilders.Query, +) *querybuilders.BoolQuery { + boolQuery := querybuilders.NewBoolQuery() + if len(mustQueries) > 0 { + boolQuery.Must(mustQueries...) + } + if len(mustNotQueries) > 0 { + boolQuery.MustNot(mustNotQueries...) + } + if len(existsQueries) > 0 { + boolQuery.Must(existsQueries...) + } + if len(notExistsQueries) > 0 { + boolQuery.MustNot(notExistsQueries...) + } + + return boolQuery +} + +func getFullTextSearchShouldQuery( + payload interface{}, + structName string, + names []string, +) (*querybuilders.BoolQuery, error) { + const op = errors.Op("getFullTextSearchShouldQuery") + contents, ok := payload.([]string) + if !ok { + return nil, errors.E(op, + fullTextSearchTypeNotSupported(structName)) + } + + boolQuery := querybuilders.NewBoolQuery() + for _, content := range contents { + boolQuery.Should( + querybuilders.NewMultiMatchQuery(content, names...). + Type("phrase_prefix"). + MaxExpansions(maxExpansions), + ) + } + return boolQuery, nil +} + +func getFullTextSearchMustQuery( + payload interface{}, + structName string, + names []string, +) (*querybuilders.BoolQuery, error) { + const op = errors.Op("getFullTextSearchMustQuery") + contents, ok := payload.([]string) + if !ok { + return nil, errors.E(op, + fullTextSearchTypeNotSupported(structName)) + } + + boolQuery := querybuilders.NewBoolQuery() + for _, content := range contents { + boolQuery.Must( + querybuilders.NewMultiMatchQuery(content, names...). + Type("phrase_prefix"). + MaxExpansions(maxExpansions), + ) + } + return boolQuery, nil +} + +func getMultiMatchSearchShouldQuery( + payload interface{}, + structName string, + names []string, +) (*querybuilders.BoolQuery, error) { + const op = errors.Op("getMultiMatchSearchShouldQuery") + contents, ok := payload.([]string) + if !ok { + return nil, errors.E(op, + multiMatchSearchTypeNotSupported(structName)) + } + + boolQuery := querybuilders.NewBoolQuery() + for _, content := range contents { + boolQuery.Should( + querybuilders.NewMultiMatchQuery(content, names...). + Type("best_fields"). // default, but we explicitly specify this choice + MaxExpansions(maxExpansions), + ) + } + return boolQuery, nil +} + +func getRangeQuery[T Ranges](from T, to T, name string) *querybuilders.RangeQuery { + var zero T + query := querybuilders.NewRangeQuery(name) + if from != zero { + query = query.From(from) + } + if to != zero { + query = query.To(to) + } + return query +} + +func getMustNestedQuery( + payload interface{}, + name string, + queries []querybuilders.Query, +) ([]querybuilders.Query, error) { + const op = errors.Op("getMustNestedQuery") + nestedQuery, err := getMustQuery(payload) + if err != nil { + return nil, errors.E(op, err) + } + + return appendNestedQuery( + name, + queries, + nestedQuery, + ), nil +} + +func getExistsNestedQuery( + payload interface{}, + name string, + existsQueries []querybuilders.Query, + notExistsQueries []querybuilders.Query, +) ([]querybuilders.Query, []querybuilders.Query, error) { + const op = errors.Op("getExistsNestedQuery") + nestedQueryExists, nestedQueryNotExists, err := getExistsQuery(payload) + if err != nil { + return nil, nil, errors.E(op, err) + } + + existsQueries = appendNestedQuery( + name, + existsQueries, + nestedQueryExists, + ) + + notExistsQueries = appendNestedQuery( + name, + notExistsQueries, + nestedQueryNotExists, + ) + + return existsQueries, notExistsQueries, nil +} + +func appendNestedQuery( + queryName string, + queries []querybuilders.Query, + nestedQuery []querybuilders.Query, +) []querybuilders.Query { + if len(nestedQuery) == 0 { + return queries + } + + if len(nestedQuery) == 1 { + return append(queries, + querybuilders.NewNestedQuery( + queryName, + nestedQuery[0], + ), + ) + } + + return append(queries, + querybuilders.NewNestedQuery( + queryName, + querybuilders.NewBoolQuery().Must(nestedQuery...), + )) +} + +func parseFieldNames(tag string) []string { + if tag == "" { + return nil + } + return strings.Split(tag, ",") +} + +func extractSliceFromInterface[T any](input interface{}) []interface{} { + s, _ := input.([]T) + is := make([]interface{}, len(s)) + for i, v := range s { + is[i] = v + } + return is +} + +func queryWithSearchAfter(q querybuilders.Query, searchAfter string) string { + var b strings.Builder + + b.WriteString(`{"query":`) + + b.WriteString(marshalQuery(q)) + + if len(searchAfter) > 0 { + b.WriteString(", ") + b.WriteString(fmt.Sprintf(` "search_after": %s`, searchAfter)) + } + + b.WriteString("}") + + log.Debug().Str("elastic_query", b.String()).Msg("Elastic query") + + return b.String() +} diff --git a/official/v7/search_test.go b/official/v7/search_test.go new file mode 100644 index 0000000..32fe5f3 --- /dev/null +++ b/official/v7/search_test.go @@ -0,0 +1,289 @@ +package v7 + +import ( + "context" + "io" + "net/http" + "strings" + "testing" + "time" + + "github.com/arquivei/foundationkit/errors" + "github.com/arquivei/foundationkit/ref" + es "github.com/elastic/go-elasticsearch/v7" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + + "github.com/arquivei/elasticutil/official/v7/querybuilders" +) + +func Test_Search(t *testing.T) { + t.Parallel() + tests := []struct { + name string + config SearchConfig + expectedResponse SearchResponse + transport *mockTransport + expectedError string + expectedErrorCode errors.Code + }{ + { + name: "success", + config: SearchConfig{ + Indexes: []string{"index1", "index2"}, + Size: 19, + Filter: getMockFilter(), + IgnoreUnavailable: true, + AllowNoIndices: true, + TrackTotalHits: true, + Sort: Sorters{ + Sorters: []Sorter{ + { + Field: "Date", + Ascending: true, + }, + { + Field: "ID", + Ascending: false, + }, + }, + }, + SearchAfter: `{"paginator"}`, + }, + transport: func() *mockTransport { + server := new(mockTransport) + server.On( + "RoundTrip", + "http://localhost:9200/index1,index2/_search?allow_no_indices=true&ignore_unavailable=true&size=19&sort=Date%3Aasc%2CID%3Adesc&track_total_hits=true", + `{"query":{"bool":{"must":[{"terms":{"Name":["John","Mary"]}},{"terms":{"Age":[16,17,18,25,26]}},{"term":{"HasCovid":true}},{"range":{"CreatedAt":{"from":"2020-11-28T15:27:39.000000049Z","include_lower":true,"include_upper":true,"to":"2021-11-28T15:27:39.000000049Z"}}},{"range":{"Age":{"from":15,"include_lower":true,"include_upper":true,"to":30}}},{"range":{"Age":{"from":0.5,"include_lower":true,"include_upper":true,"to":1.9}}},{"nested":{"path":"Covid","query":{"bool":{"must":[{"terms":{"Covid.Symptom":["cough"]}},{"range":{"Covid.Date":{"from":"2019-11-28T15:27:39.000000049Z","include_lower":true,"include_upper":true,"to":"2020-11-28T15:27:39.000000049Z"}}}]}}}},{"bool":{"should":[{"multi_match":{"fields":["Name","SocialName"],"max_expansions":1024,"query":"John","type":"phrase_prefix"}},{"multi_match":{"fields":["Name","SocialName"],"max_expansions":1024,"query":"Mary","type":"phrase_prefix"}},{"multi_match":{"fields":["Name","SocialName"],"max_expansions":1024,"query":"Rebecca","type":"phrase_prefix"}}]}},{"bool":{"must":[{"multi_match":{"fields":["Name","SocialName"],"max_expansions":1024,"query":"Lennon","type":"phrase_prefix"}},{"multi_match":{"fields":["Name","SocialName"],"max_expansions":1024,"query":"McCartney","type":"phrase_prefix"}}]}},{"bool":{"should":[{"multi_match":{"fields":["Any"],"max_expansions":1024,"query":"Beatles","type":"best_fields"}},{"multi_match":{"fields":["Any"],"max_expansions":1024,"query":"Stones","type":"best_fields"}}]}},{"bool":{"must":{"term":{"Name":"John"}}}},{"nested":{"path":"Covid","query":{"exists":{"field":"Covid"}}}},{"exists":{"field":"Age"}}],"must_not":[{"terms":{"Name":["Lary"]}},{"range":{"Age":{"from":29,"include_lower":true,"include_upper":true,"to":30}}}]}}, "search_after": {"paginator"}}`, + ).Once().Return( + `{"took":10,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},"hits":{"total":{"value":2},"max_score":null,"hits":[{"_index":"tiramisu_cte-2022101","_id":"elastic-id-1","_score":null,"sort":["pag2"]},{"_index":"tiramisu_cte-2019","_id":"elastic-id-2","_score":null,"sort":["pag3"]}]}}`, + 200, + nil, + ) + + return server + }(), + expectedResponse: SearchResponse{ + IDs: []string{"elastic-id-1", "elastic-id-2"}, + Paginator: `["pag3"]`, + Total: 2, + Took: 10, + }, + }, + { + name: "1 shard failed", + config: SearchConfig{ + Indexes: []string{"index1", "index2"}, + Size: 19, + Filter: getMockFilter(), + IgnoreUnavailable: true, + AllowNoIndices: true, + TrackTotalHits: true, + Sort: Sorters{ + Sorters: []Sorter{ + { + Field: "Date", + Ascending: true, + }, + { + Field: "ID", + Ascending: false, + }, + }, + }, + SearchAfter: `{"paginator"}`, + }, + transport: func() *mockTransport { + server := new(mockTransport) + server.On( + "RoundTrip", + "http://localhost:9200/index1,index2/_search?allow_no_indices=true&ignore_unavailable=true&size=19&sort=Date%3Aasc%2CID%3Adesc&track_total_hits=true", + `{"query":{"bool":{"must":[{"terms":{"Name":["John","Mary"]}},{"terms":{"Age":[16,17,18,25,26]}},{"term":{"HasCovid":true}},{"range":{"CreatedAt":{"from":"2020-11-28T15:27:39.000000049Z","include_lower":true,"include_upper":true,"to":"2021-11-28T15:27:39.000000049Z"}}},{"range":{"Age":{"from":15,"include_lower":true,"include_upper":true,"to":30}}},{"range":{"Age":{"from":0.5,"include_lower":true,"include_upper":true,"to":1.9}}},{"nested":{"path":"Covid","query":{"bool":{"must":[{"terms":{"Covid.Symptom":["cough"]}},{"range":{"Covid.Date":{"from":"2019-11-28T15:27:39.000000049Z","include_lower":true,"include_upper":true,"to":"2020-11-28T15:27:39.000000049Z"}}}]}}}},{"bool":{"should":[{"multi_match":{"fields":["Name","SocialName"],"max_expansions":1024,"query":"John","type":"phrase_prefix"}},{"multi_match":{"fields":["Name","SocialName"],"max_expansions":1024,"query":"Mary","type":"phrase_prefix"}},{"multi_match":{"fields":["Name","SocialName"],"max_expansions":1024,"query":"Rebecca","type":"phrase_prefix"}}]}},{"bool":{"must":[{"multi_match":{"fields":["Name","SocialName"],"max_expansions":1024,"query":"Lennon","type":"phrase_prefix"}},{"multi_match":{"fields":["Name","SocialName"],"max_expansions":1024,"query":"McCartney","type":"phrase_prefix"}}]}},{"bool":{"should":[{"multi_match":{"fields":["Any"],"max_expansions":1024,"query":"Beatles","type":"best_fields"}},{"multi_match":{"fields":["Any"],"max_expansions":1024,"query":"Stones","type":"best_fields"}}]}},{"bool":{"must":{"term":{"Name":"John"}}}},{"nested":{"path":"Covid","query":{"exists":{"field":"Covid"}}}},{"exists":{"field":"Age"}}],"must_not":[{"terms":{"Name":["Lary"]}},{"range":{"Age":{"from":29,"include_lower":true,"include_upper":true,"to":30}}}]}}, "search_after": {"paginator"}}`, + ).Once().Return( + `{"took":10,"_shards":{"total":1,"successful":0,"skipped":0,"failed":1},"hits":{"total":{"value":2},"max_score":null,"hits":[{"_index":"tiramisu_cte-2022101","_id":"elastic-id-1","_score":null,"sort":["pag2"]},{"_index":"tiramisu_cte-2019","_id":"elastic-id-2","_score":null,"sort":["pag3"]}]}}`, + 200, + nil, + ) + + return server + }(), + expectedError: "v7.Client.Search: parseResponse: not all shards replied [replied=0,failed=1,total=1]", + expectedErrorCode: ErrCodeBadGateway, + }, + { + name: "elastic return an error", + config: SearchConfig{ + Indexes: []string{"index1", "index2"}, + Size: 19, + Filter: getMockFilter(), + IgnoreUnavailable: true, + AllowNoIndices: true, + TrackTotalHits: true, + Sort: Sorters{ + Sorters: []Sorter{ + { + Field: "Date", + Ascending: true, + }, + { + Field: "ID", + Ascending: false, + }, + }, + }, + SearchAfter: `{"paginator"}`, + }, + transport: func() *mockTransport { + server := new(mockTransport) + server.On( + "RoundTrip", + "http://localhost:9200/index1,index2/_search?allow_no_indices=true&ignore_unavailable=true&size=19&sort=Date%3Aasc%2CID%3Adesc&track_total_hits=true", + `{"query":{"bool":{"must":[{"terms":{"Name":["John","Mary"]}},{"terms":{"Age":[16,17,18,25,26]}},{"term":{"HasCovid":true}},{"range":{"CreatedAt":{"from":"2020-11-28T15:27:39.000000049Z","include_lower":true,"include_upper":true,"to":"2021-11-28T15:27:39.000000049Z"}}},{"range":{"Age":{"from":15,"include_lower":true,"include_upper":true,"to":30}}},{"range":{"Age":{"from":0.5,"include_lower":true,"include_upper":true,"to":1.9}}},{"nested":{"path":"Covid","query":{"bool":{"must":[{"terms":{"Covid.Symptom":["cough"]}},{"range":{"Covid.Date":{"from":"2019-11-28T15:27:39.000000049Z","include_lower":true,"include_upper":true,"to":"2020-11-28T15:27:39.000000049Z"}}}]}}}},{"bool":{"should":[{"multi_match":{"fields":["Name","SocialName"],"max_expansions":1024,"query":"John","type":"phrase_prefix"}},{"multi_match":{"fields":["Name","SocialName"],"max_expansions":1024,"query":"Mary","type":"phrase_prefix"}},{"multi_match":{"fields":["Name","SocialName"],"max_expansions":1024,"query":"Rebecca","type":"phrase_prefix"}}]}},{"bool":{"must":[{"multi_match":{"fields":["Name","SocialName"],"max_expansions":1024,"query":"Lennon","type":"phrase_prefix"}},{"multi_match":{"fields":["Name","SocialName"],"max_expansions":1024,"query":"McCartney","type":"phrase_prefix"}}]}},{"bool":{"should":[{"multi_match":{"fields":["Any"],"max_expansions":1024,"query":"Beatles","type":"best_fields"}},{"multi_match":{"fields":["Any"],"max_expansions":1024,"query":"Stones","type":"best_fields"}}]}},{"bool":{"must":{"term":{"Name":"John"}}}},{"nested":{"path":"Covid","query":{"exists":{"field":"Covid"}}}},{"exists":{"field":"Age"}}],"must_not":[{"terms":{"Name":["Lary"]}},{"range":{"Age":{"from":29,"include_lower":true,"include_upper":true,"to":30}}}]}}, "search_after": {"paginator"}}`, + ).Once().Return( + `{"took":10,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},"hits":{"total":{"value":2},"max_score":null,"hits":[{"_index":"tiramisu_cte-2022101","_id":"elastic-id-1","_score":null,"sort":["pag2"]},{"_index":"tiramisu_cte-2019","_id":"elastic-id-2","_score":null,"sort":["pag3"]}]}}`, + 200, + errors.New("elastic error"), + ) + + return server + }(), + expectedError: "v7.Client.Search: doSearch: elastic error", + expectedErrorCode: ErrCodeBadGateway, + }, + } + + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + assert.NotPanics(t, func() { + client := mustNewClientTest(test.transport) + response, err := client.Search(context.Background(), test.config) + if test.expectedError == "" { + assert.NoError(t, err) + } else { + assert.EqualError(t, err, test.expectedError) + assert.Equal(t, test.expectedErrorCode, errors.GetCode(err)) + } + assert.Equal(t, test.expectedResponse, response) + }) + }) + } +} + +type mockTransport struct { + mock.Mock +} + +func (m *mockTransport) RoundTrip(r *http.Request) (*http.Response, error) { + buf := new(strings.Builder) + io.Copy(buf, r.Body) + request := buf.String() + + args := m.Called(r.URL.String(), request) + bodyContent := args.String(0) + httpCode := args.Int(1) + + response := &http.Response{ + Header: make(http.Header), + StatusCode: httpCode, + Body: io.NopCloser(strings.NewReader(bodyContent)), + } + + response.Header.Add("X-Elastic-Product", "Elasticsearch") + + return response, args.Error(2) +} + +func getMockFilter() Filter { + type ExampleFilterMust struct { + Names []string `es:"Name"` + Ages []uint64 `es:"Age"` + HasCovid *bool + CreatedAt *TimeRange + AgeRange *IntRange `es:"Age"` + ValueRange *FloatRange `es:"Age"` + CovidInfo Nested `es:"Covid"` + NameOrSocialName FullTextSearchShould `es:"Name,SocialName"` + NameAndSocialName FullTextSearchMust `es:"Name,SocialName"` + Any MultiMatchSearchShould `es:"Any"` + MyCustomSearch CustomSearch + } + + type ExampleFilterExists struct { + HasCovidInfo Nested `es:"Covid"` + HasAge *bool `es:"Age"` + } + + type ExampleCovidInfo struct { + HasCovidInfo *bool `es:"Covid"` + Symptoms []string `es:"Covid.Symptom"` + FirstSymptomDate *TimeRange `es:"Covid.Date"` + } + + return Filter{ + Must: ExampleFilterMust{ + Names: []string{"John", "Mary"}, + Ages: []uint64{16, 17, 18, 25, 26}, + HasCovid: ref.Bool(true), + CovidInfo: NewNested( + ExampleCovidInfo{ + Symptoms: []string{"cough"}, + FirstSymptomDate: &TimeRange{ + From: time.Date(2019, time.November, 28, 15, 27, 39, 49, time.UTC), + To: time.Date(2020, time.November, 28, 15, 27, 39, 49, time.UTC), + }, + }, + ), + CreatedAt: &TimeRange{ + From: time.Date(2020, time.November, 28, 15, 27, 39, 49, time.UTC), + To: time.Date(2021, time.November, 28, 15, 27, 39, 49, time.UTC), + }, + AgeRange: &IntRange{ + From: 15, + To: 30, + }, + NameOrSocialName: NewFullTextSearchShould([]string{"John", "Mary", "Rebecca"}), + ValueRange: &FloatRange{ + From: 0.5, + To: 1.9, + }, + NameAndSocialName: NewFullTextSearchMust([]string{"Lennon", "McCartney"}), + Any: NewMultiMatchSearchShould([]string{"Beatles", "Stones"}), + MyCustomSearch: NewCustomSearch(func() (querybuilders.Query, error) { + return querybuilders.NewBoolQuery().Must(querybuilders.NewTermQuery("Name", "John")), nil + }), + }, + MustNot: ExampleFilterMust{ + Names: []string{"Lary"}, + AgeRange: &IntRange{ + From: 29, + To: 30, + }, + }, + Exists: ExampleFilterExists{ + HasCovidInfo: NewNested( + ExampleCovidInfo{ + HasCovidInfo: ref.Bool(true), + }, + ), + HasAge: ref.Bool(true), + }, + } +} + +func mustNewClientTest(roundTripper http.RoundTripper) Client { + client, err := es.NewClient( + es.Config{ + Transport: roundTripper, + UseResponseCheckOnly: true, + }, + ) + if err != nil { + panic(err) + } + return &esClient{ + client: client, + } +}