From 266d35e71c2493d291b1bdb8710d73b02c841780 Mon Sep 17 00:00:00 2001 From: Vilsol Date: Fri, 1 Dec 2023 20:31:50 +0200 Subject: [PATCH] fix: retry metadata extraction --- go.mod | 3 +- go.sum | 2 + validation/validation.go | 87 +++++++++++++++++++++++----------------- 3 files changed, 55 insertions(+), 37 deletions(-) diff --git a/go.mod b/go.mod index c5a614e..6307e24 100755 --- a/go.mod +++ b/go.mod @@ -10,11 +10,13 @@ require ( github.com/Masterminds/semver/v3 v3.2.1 github.com/Vilsol/slox v0.0.1 github.com/VirusTotal/vt-go v0.0.0-20230717142150-8431ff2cc00f + github.com/avast/retry-go v3.0.0+incompatible github.com/avast/retry-go/v3 v3.1.1 github.com/aws/aws-sdk-go v1.46.1 github.com/btcsuite/btcd/btcutil v1.1.3 github.com/cespare/xxhash v1.1.0 github.com/chai2010/webp v1.1.1 + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/dgraph-io/ristretto v0.1.1 github.com/felixge/fgprof v0.9.3 github.com/finnbear/moderation v0.11.4 @@ -83,7 +85,6 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/containerd/console v1.0.3 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dustin/go-humanize v1.0.1 // indirect diff --git a/go.sum b/go.sum index 6b46a4c..8abf08c 100644 --- a/go.sum +++ b/go.sum @@ -103,6 +103,8 @@ github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkE github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q= github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE= github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk= +github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= +github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= github.com/avast/retry-go/v3 v3.1.1 h1:49Scxf4v8PmiQ/nY0aY3p0hDueqSmc7++cBbtiDGu2g= github.com/avast/retry-go/v3 v3.1.1/go.mod h1:6cXRK369RpzFL3UQGqIUp9Q7GDrams+KsYWrfNA1/nQ= github.com/aws/aws-sdk-go v1.46.1 h1:U26quvBWFZMQuultLw5tloW4GnmWaChEwMZNq8uYatw= diff --git a/validation/validation.go b/validation/validation.go index 90638d6..54d3717 100644 --- a/validation/validation.go +++ b/validation/validation.go @@ -19,6 +19,7 @@ import ( "github.com/Masterminds/semver/v3" "github.com/Vilsol/slox" + "github.com/avast/retry-go" "github.com/davecgh/go-spew/spew" "github.com/pkg/errors" "github.com/spf13/viper" @@ -171,52 +172,66 @@ func ExtractModInfo(ctx context.Context, body []byte, withMetadata bool, withVal slox.Info(ctx, "decided engine version", slog.String("version", engineVersion)) - parserClient := parser.NewParserClient(conn) - stream, err := parserClient.Parse(ctx, &parser.ParseRequest{ - ZipData: body, - EngineVersion: engineVersion, - }, - grpc.MaxCallSendMsgSize(1024*1024*1024), // 1GB - grpc.MaxCallRecvMsgSize(1024*1024*1024), // 1GB - ) - if err != nil { - return nil, fmt.Errorf("failed to parse mod: %w", err) - } - - defer func(stream parser.Parser_ParseClient) { - err := stream.CloseSend() + if err := retry.Do(func() error { + parserClient := parser.NewParserClient(conn) + stream, err := parserClient.Parse(ctx, &parser.ParseRequest{ + ZipData: body, + EngineVersion: engineVersion, + }, + grpc.MaxCallSendMsgSize(1024*1024*1024), // 1GB + grpc.MaxCallRecvMsgSize(1024*1024*1024), // 1GB + ) if err != nil { - slox.Error(ctx, "failed closing parser stream", slog.Any("err", err)) + return fmt.Errorf("failed to parse mod: %w", err) } - }(stream) - beforeUpload := time.Now().Add(-time.Minute) - for { - asset, err := stream.Recv() - if err != nil { - //nolint - if errors.Is(err, io.EOF) || err == io.EOF { - break + defer func(stream parser.Parser_ParseClient) { + err := stream.CloseSend() + if err != nil { + slox.Error(ctx, "failed closing parser stream", slog.Any("err", err)) } - spew.Config.DisablePointerMethods = false - spew.Dump(err) - return nil, fmt.Errorf("failed reading parser stream: %w", err) - } + }(stream) - slox.Info(ctx, "received asset from parser", slog.String("path", asset.GetPath())) - - if asset.Path == "metadata.json" { - out, err := ExtractMetadata(asset.Data) + beforeUpload := time.Now().Add(-time.Minute) + for { + asset, err := stream.Recv() if err != nil { - return nil, err + //nolint + if errors.Is(err, io.EOF) || err == io.EOF { + break + } + spew.Config.DisablePointerMethods = false + spew.Dump(err) + return fmt.Errorf("failed reading parser stream: %w", err) + } + + slox.Info(ctx, "received asset from parser", slog.String("path", asset.GetPath())) + + if asset.Path == "metadata.json" { + out, err := ExtractMetadata(asset.Data) + if err != nil { + return err + } + modInfo.Metadata = append(modInfo.Metadata, out) } - modInfo.Metadata = append(modInfo.Metadata, out) + + storage.UploadModAsset(ctx, modInfo.ModReference, asset.GetPath(), asset.GetData()) } - storage.UploadModAsset(ctx, modInfo.ModReference, asset.GetPath(), asset.GetData()) - } + storage.DeleteOldModAssets(ctx, modInfo.ModReference, beforeUpload) - storage.DeleteOldModAssets(ctx, modInfo.ModReference, beforeUpload) + return nil + }, + retry.Attempts(10), + retry.Delay(time.Second*10), + retry.DelayType(retry.FixedDelay), + retry.OnRetry(func(n uint, err error) { + if n > 0 { + slox.Info(ctx, "retrying to extract metadata", slog.Uint64("n", uint64(n))) + } + })); err != nil { + return nil, err //nolint + } } modInfo.Size = int64(len(body))