Skip to content

Commit

Permalink
free ourselves of all aws-sdk-go v1
Browse files Browse the repository at this point in the history
  • Loading branch information
Hongyu Zhou committed Aug 21, 2023
1 parent 636c6c6 commit c410558
Show file tree
Hide file tree
Showing 3 changed files with 25 additions and 15 deletions.
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ require (
github.com/aws/aws-sdk-go-v2/config v1.18.29
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.73
github.com/aws/aws-sdk-go-v2/service/s3 v1.37.0
github.com/aws/smithy-go v1.14.0
github.com/fsnotify/fsnotify v1.5.1
github.com/go-sql-driver/mysql v1.4.1
github.com/google/go-cmp v0.5.8
Expand Down Expand Up @@ -42,7 +43,6 @@ require (
github.com/aws/aws-sdk-go-v2/service/sso v1.13.1 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.1 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.21.1 // indirect
github.com/aws/smithy-go v1.14.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/klauspost/compress v1.16.7 // indirect
Expand Down
18 changes: 11 additions & 7 deletions pkg/reflector/download.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@ package reflector
import (
"bytes"
"context"
er "errors"
"fmt"
"io"
"net/http"
"os"
"strings"
"time"
Expand All @@ -14,7 +14,8 @@ import (
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/aws/smithy-go"
gzip "github.com/klauspost/pgzip"

"github.com/segmentio/ctlstore/pkg/errs"
Expand Down Expand Up @@ -60,11 +61,14 @@ func (d *S3Downloader) DownloadTo(w io.Writer) (n int64, err error) {
stats.Observe("snapshot_download_time", time.Now().Sub(start))

if err != nil {
switch err := err.(type) {
case awserr.RequestFailure:
if d.StartOverOnNotFound && err.StatusCode() == http.StatusNotFound {
// don't bother retrying. we'll start with a fresh ldb.
return -1, errors.WithTypes(errors.Wrap(err, "get s3 data"), errs.ErrTypePermanent)
var ae smithy.APIError
if er.As(err, &ae) {
switch ae.(type) {
case *types.NotFound:
if d.StartOverOnNotFound {
// don't bother retrying. we'll start with a fresh ldb.
return -1, errors.WithTypes(errors.Wrap(err, "get s3 data"), errs.ErrTypePermanent)
}
}
}
// retry
Expand Down
20 changes: 13 additions & 7 deletions pkg/reflector/download_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,10 @@ import (
"strings"
"testing"

"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/aws/smithy-go"
gzip "github.com/klauspost/pgzip"
"github.com/stretchr/testify/require"

Expand Down Expand Up @@ -62,8 +64,9 @@ func TestS3DownloadErrors(t *testing.T) {
isSupervisor: true,
s3Client: func() reflector.S3Client {
f := &fakes.FakeS3Client{}
f.GetObjectReturns(nil, awserr.NewRequestFailure(
awserr.New("error-code", "error-message", errors.New("failure")), http.StatusNotFound, ""))
f.GetObjectReturns(nil, &types.NotFound{
Message: aws.String("failure"),
})
return f
},
err: errors.New("failure"),
Expand All @@ -74,8 +77,9 @@ func TestS3DownloadErrors(t *testing.T) {
name: "temporary failure on 404 if not-supervisor",
s3Client: func() reflector.S3Client {
f := &fakes.FakeS3Client{}
f.GetObjectReturns(nil, awserr.NewRequestFailure(
awserr.New("error-code", "error-message", errors.New("failure")), http.StatusNotFound, ""))
f.GetObjectReturns(nil, &types.NotFound{
Message: aws.String("failure"),
})
return f
},
err: errors.New("failure"),
Expand All @@ -86,8 +90,10 @@ func TestS3DownloadErrors(t *testing.T) {
name: "temporary failure",
s3Client: func() reflector.S3Client {
f := &fakes.FakeS3Client{}
f.GetObjectReturns(nil, awserr.NewRequestFailure(
awserr.New("error-code", "error-message", errors.New("failure")), http.StatusInternalServerError, ""))
f.GetObjectReturns(nil, &smithy.GenericAPIError{
Code: string(rune(http.StatusInternalServerError)),
Message: "failure",
})
return f
},
err: errors.New("failure"),
Expand Down

0 comments on commit c410558

Please sign in to comment.