From f2925c67a5ca0b46374b4796e7a44c33b2f9face Mon Sep 17 00:00:00 2001 From: Artem Date: Mon, 10 Jul 2023 17:17:07 +0100 Subject: [PATCH] Fix: opg by partitions --- internal/helpers/quarters.go | 38 ++++++++++++++++++++++++++ internal/postgres/operation/storage.go | 37 +++++++++++++++++++++---- internal/postgres/partition_manager.go | 36 ++---------------------- 3 files changed, 73 insertions(+), 38 deletions(-) create mode 100644 internal/helpers/quarters.go diff --git a/internal/helpers/quarters.go b/internal/helpers/quarters.go new file mode 100644 index 000000000..07575b317 --- /dev/null +++ b/internal/helpers/quarters.go @@ -0,0 +1,38 @@ +package helpers + +import ( + "errors" + "time" +) + +// QuarterOf - +func QuarterOf(month time.Month) int { + return (int(month) + 2) / 3 +} + +// QuarterBoundaries - +func QuarterBoundaries(current time.Time) (time.Time, time.Time, error) { + year := current.Year() + quarter := QuarterOf(current.Month()) + + switch quarter { + case 1: + start := time.Date(year, time.January, 1, 0, 0, 0, 0, time.UTC) + end := start.AddDate(0, 3, 0) + return start, end, nil + case 2: + start := time.Date(year, time.April, 1, 0, 0, 0, 0, time.UTC) + end := start.AddDate(0, 3, 0) + return start, end, nil + case 3: + start := time.Date(year, time.July, 1, 0, 0, 0, 0, time.UTC) + end := start.AddDate(0, 3, 0) + return start, end, nil + case 4: + start := time.Date(year, time.October, 1, 0, 0, 0, 0, time.UTC) + end := start.AddDate(0, 3, 0) + return start, end, nil + } + + return time.Now(), time.Now(), errors.New("invalid quarter") +} diff --git a/internal/postgres/operation/storage.go b/internal/postgres/operation/storage.go index d2db97a54..19947335d 100644 --- a/internal/postgres/operation/storage.go +++ b/internal/postgres/operation/storage.go @@ -5,8 +5,12 @@ import ( "fmt" "time" + "github.com/baking-bad/bcdhub/internal/bcd" + "github.com/baking-bad/bcdhub/internal/bcd/consts" + "github.com/baking-bad/bcdhub/internal/helpers" "github.com/baking-bad/bcdhub/internal/models" "github.com/baking-bad/bcdhub/internal/models/account" + "github.com/baking-bad/bcdhub/internal/models/contract" "github.com/baking-bad/bcdhub/internal/models/operation" "github.com/baking-bad/bcdhub/internal/models/types" "github.com/baking-bad/bcdhub/internal/postgres/core" @@ -251,15 +255,35 @@ func (storage *Storage) OPG(address string, size, lastID int64) ([]operation.OPG result = make([]operation.OPG, 0) currentOpg operation.OPG isAdded bool + lastAction = time.Now().UTC() + limit = 1000 ) + + if bcd.IsContractLazy(address) { + if err := storage.DB.Model((*contract.Contract)(nil)). + Column("last_action"). + Where("account_id = ?", accountID). + Select(&lastAction); err != nil { + return nil, err + } + } + for !end { + startTime, endTime, err := helpers.QuarterBoundaries(lastAction) + if err != nil { + return nil, err + } + subQuery := storage.DB.Model((*operation.Operation)(nil)). + Column("id", "hash", "counter", "entrypoint", "amount", "fee", "burned", "level", "content_index", "timestamp", "kind", "status", "internal", "destination_id", "source_id"). WhereGroup( func(q *orm.Query) (*orm.Query, error) { return q.Where("destination_id = ?", accountID).WhereOr("source_id = ?", accountID), nil }, ). - Limit(1000). + Where("timestamp < ?", endTime). + Where("timestamp >= ?", startTime). + Limit(limit). Order("id desc") if lastID > 0 { @@ -271,10 +295,6 @@ func (storage *Storage) OPG(address string, size, lastID int64) ([]operation.OPG return nil, err } - if len(ops) == 0 { - break - } - for i := range ops { if !bytes.Equal(currentOpg.Hash, ops[i].Hash) && currentOpg.Counter != ops[i].Counter { if len(currentOpg.Hash) > 0 && currentOpg.Counter > 0 { @@ -322,6 +342,13 @@ func (storage *Storage) OPG(address string, size, lastID int64) ([]operation.OPG } lastID = currentOpg.LastID + + if len(ops) < limit { + lastAction = lastAction.AddDate(0, -3, 0) + if lastAction.Before(consts.BeginningOfTime) { + break + } + } } if len(currentOpg.Hash) > 0 && currentOpg.Counter > 0 && !isAdded { diff --git a/internal/postgres/partition_manager.go b/internal/postgres/partition_manager.go index 3994d0894..ebf6ef5d9 100644 --- a/internal/postgres/partition_manager.go +++ b/internal/postgres/partition_manager.go @@ -2,10 +2,10 @@ package postgres import ( "context" - "errors" "fmt" "time" + "github.com/baking-bad/bcdhub/internal/helpers" "github.com/baking-bad/bcdhub/internal/models" "github.com/baking-bad/bcdhub/internal/models/bigmapdiff" "github.com/baking-bad/bcdhub/internal/models/operation" @@ -29,38 +29,8 @@ func NewPartitionManager(conn *core.Postgres) *PartitionManager { const createPartitionTemplate = `CREATE TABLE IF NOT EXISTS ? PARTITION OF ? FOR VALUES FROM (?) TO (?);` -func quarterOf(month time.Month) int { - return (int(month) + 2) / 3 -} - -func quarterBoundaries(current time.Time) (time.Time, time.Time, error) { - year := current.Year() - quarter := quarterOf(current.Month()) - - switch quarter { - case 1: - start := time.Date(year, time.January, 1, 0, 0, 0, 0, time.UTC) - end := start.AddDate(0, 3, 0) - return start, end, nil - case 2: - start := time.Date(year, time.April, 1, 0, 0, 0, 0, time.UTC) - end := start.AddDate(0, 3, 0) - return start, end, nil - case 3: - start := time.Date(year, time.July, 1, 0, 0, 0, 0, time.UTC) - end := start.AddDate(0, 3, 0) - return start, end, nil - case 4: - start := time.Date(year, time.October, 1, 0, 0, 0, 0, time.UTC) - end := start.AddDate(0, 3, 0) - return start, end, nil - } - - return time.Now(), time.Now(), errors.New("invalid quarter") -} - func (pm *PartitionManager) partitionId(currentTime time.Time) string { - return fmt.Sprintf("%dQ%d", currentTime.Year(), quarterOf(currentTime.Month())) + return fmt.Sprintf("%dQ%d", currentTime.Year(), helpers.QuarterOf(currentTime.Month())) } // CreatePartitions - @@ -70,7 +40,7 @@ func (pm *PartitionManager) CreatePartitions(ctx context.Context, currentTime ti return nil } - start, end, err := quarterBoundaries(currentTime) + start, end, err := helpers.QuarterBoundaries(currentTime) if err != nil { return err }