Skip to content

Commit

Permalink
Merge pull request devfile#174 from maysunfaisal/option-download-1
Browse files Browse the repository at this point in the history 
Gate downloading of git resources
  • Loading branch information
@maysunfaisal
maysunfaisal committed Jul 7, 2023
2 parents 04a8b3f + cff5c26 commit 6c4a5b8
Show file tree
Hide file tree
Showing 3 changed files with 85 additions and 42 deletions.
8 changes: 8 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,14 @@ The function documentation can be accessed via [pkg.go.dev](https://pkg.go.dev/g
...
```
10. By default, the library downloads the Git repository resources associated with the Git URL that is mentioned in a devfile uri field. To turn off the download, pass in the `DownloadGitResources` property in the parser argument
```go
downloadGitResources := false
parserArgs := parser.ParserArgs{
DownloadGitResources: &downloadGitResources,
}
```
## Projects using devfile/library
The following projects are consuming this library as a Golang dependency
Expand Down
35 changes: 24 additions & 11 deletions pkg/devfile/parser/parse.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,16 @@ import (
"context"
"encoding/json"
"fmt"
"github.com/devfile/library/v2/pkg/git"
"github.com/hashicorp/go-multierror"
"io/ioutil"
"net/url"
"os"
"path"
"reflect"
"strings"

"github.com/devfile/library/v2/pkg/git"
"github.com/hashicorp/go-multierror"

"github.com/devfile/api/v2/pkg/attributes"
devfileCtx "github.com/devfile/library/v2/pkg/devfile/parser/context"
"github.com/devfile/library/v2/pkg/devfile/parser/data"
Expand Down Expand Up @@ -167,6 +168,8 @@ type ParserArgs struct {
// ImageNamesAsSelector sets the information that will be used to handle image names as selectors when parsing the Devfile.
// Not setting this field or setting it to nil disables the logic of handling image names as selectors.
ImageNamesAsSelector *ImageSelectorArgs
// DownloadGitResources downloads the resources from Git repository if true
DownloadGitResources *bool
}

// ImageSelectorArgs defines the structure to leverage for using image names as selectors after parsing the Devfile.
Expand Down Expand Up @@ -214,12 +217,18 @@ func ParseDevfile(args ParserArgs) (d DevfileObj, err error) {
d.Ctx.SetToken(args.Token)
}

downloadGitResources := true
if args.DownloadGitResources != nil {
downloadGitResources = *args.DownloadGitResources
}

tool := resolverTools{
defaultNamespace: args.DefaultNamespace,
registryURLs: args.RegistryURLs,
context: args.Context,
k8sClient: args.K8sClient,
httpTimeout: args.HTTPTimeout,
defaultNamespace: args.DefaultNamespace,
registryURLs: args.RegistryURLs,
context: args.Context,
k8sClient: args.K8sClient,
httpTimeout: args.HTTPTimeout,
downloadGitResources: downloadGitResources,
}

flattenedDevfile := true
Expand Down Expand Up @@ -273,6 +282,8 @@ type resolverTools struct {
k8sClient client.Client
// httpTimeout is the timeout value in seconds passed in from the client.
httpTimeout *int
// downloadGitResources downloads the resources from Git repository if true
downloadGitResources bool
}

func populateAndParseDevfile(d DevfileObj, resolveCtx *resolutionContextTree, tool resolverTools, flattenedDevfile bool) (DevfileObj, error) {
Expand Down Expand Up @@ -522,10 +533,12 @@ func parseFromURI(importReference v1.ImportReference, curDevfileCtx devfileCtx.D
d.Ctx.SetToken(token)
}

destDir := path.Dir(curDevfileCtx.GetAbsPath())
err = downloadGitRepoResources(newUri, destDir, tool.httpTimeout, token)
if err != nil {
return DevfileObj{}, err
if tool.downloadGitResources {
destDir := path.Dir(curDevfileCtx.GetAbsPath())
err = downloadGitRepoResources(newUri, destDir, tool.httpTimeout, token)
if err != nil {
return DevfileObj{}, err
}
}
}
importReference.Uri = newUri
Expand Down
84 changes: 53 additions & 31 deletions pkg/devfile/parser/parse_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ import (
"bytes"
"context"
"fmt"
"github.com/devfile/library/v2/pkg/util"
"io/ioutil"
"net"
"net/http"
Expand All @@ -31,6 +30,8 @@ import (
"strings"
"testing"

"github.com/devfile/library/v2/pkg/util"

v1 "github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2"
"github.com/devfile/library/v2/pkg/git"

Expand Down Expand Up @@ -4235,16 +4236,17 @@ func Test_parseFromURI_GitProviders(t *testing.T) {
invalidDevfilePathError := "error getting devfile from url: failed to retrieve*"

tests := []struct {
name string
url string
gitUrl *git.GitUrl
token string
destDir string
importReference v1.ImportReference
wantDevFile DevfileObj
wantError *string
wantResources []string
wantResourceContent []byte
name string
url string
gitUrl *git.GitUrl
token string
destDir string
importReference v1.ImportReference
wantDevFile DevfileObj
wantError *string
wantResources []string
wantResourceContent []byte
downloadGitResources bool
}{
{
name: "private parent devfile",
Expand All @@ -4256,9 +4258,10 @@ func Test_parseFromURI_GitProviders(t *testing.T) {
Uri: server.URL,
},
},
wantDevFile: minimalDevfile,
wantResources: []string{"resource.file"},
wantResourceContent: []byte("private repo\ngit switched"),
wantDevFile: minimalDevfile,
wantResources: []string{"resource.file"},
wantResourceContent: []byte("private repo\ngit switched"),
downloadGitResources: true,
},
{
name: "public parent devfile",
Expand All @@ -4270,9 +4273,22 @@ func Test_parseFromURI_GitProviders(t *testing.T) {
Uri: server.URL,
},
},
wantDevFile: minimalDevfile,
wantResources: []string{"resource.file"},
wantResourceContent: []byte("public repo\ngit switched"),
wantDevFile: minimalDevfile,
wantResources: []string{"resource.file"},
wantResourceContent: []byte("public repo\ngit switched"),
downloadGitResources: true,
},
{
name: "public parent devfile with download turned off",
url: validUrl,
gitUrl: validGitUrl,
importReference: v1.ImportReference{
ImportReferenceUnion: v1.ImportReferenceUnion{
Uri: server.URL,
},
},
wantDevFile: minimalDevfile,
downloadGitResources: false,
},
{
// a valid parent url must contain a revision
Expand All @@ -4293,8 +4309,9 @@ func Test_parseFromURI_GitProviders(t *testing.T) {
Uri: server.URL,
},
},
wantError: &invalidDevfilePathError,
wantResources: []string{},
wantError: &invalidDevfilePathError,
wantResources: []string{},
downloadGitResources: true,
},
{
name: "public parent devfile that is not from a git provider",
Expand All @@ -4306,8 +4323,9 @@ func Test_parseFromURI_GitProviders(t *testing.T) {
Uri: server.URL,
},
},
wantDevFile: minimalDevfile,
wantResources: []string{},
wantDevFile: minimalDevfile,
wantResources: []string{},
downloadGitResources: true,
},
{
name: "public parent devfile with no devfile path",
Expand All @@ -4325,8 +4343,9 @@ func Test_parseFromURI_GitProviders(t *testing.T) {
Uri: server.URL,
},
},
wantError: &invalidDevfilePathError,
wantResources: []string{},
wantError: &invalidDevfilePathError,
wantResources: []string{},
downloadGitResources: true,
},
{
name: "public parent devfile with invalid devfile path",
Expand All @@ -4346,8 +4365,9 @@ func Test_parseFromURI_GitProviders(t *testing.T) {
Uri: server.URL,
},
},
wantError: &invalidDevfilePathError,
wantResources: []string{},
wantError: &invalidDevfilePathError,
wantResources: []string{},
downloadGitResources: true,
},
{
name: "private parent devfile with invalid token",
Expand All @@ -4359,8 +4379,9 @@ func Test_parseFromURI_GitProviders(t *testing.T) {
Uri: server.URL,
},
},
wantError: &invalidTokenError,
wantResources: []string{},
wantError: &invalidTokenError,
wantResources: []string{},
downloadGitResources: true,
},
{
name: "private parent devfile with invalid revision",
Expand All @@ -4380,8 +4401,9 @@ func Test_parseFromURI_GitProviders(t *testing.T) {
Uri: server.URL,
},
},
wantError: &invalidGitSwitchError,
wantResources: []string{},
wantError: &invalidGitSwitchError,
wantResources: []string{},
downloadGitResources: true,
},
}

Expand All @@ -4396,13 +4418,13 @@ func Test_parseFromURI_GitProviders(t *testing.T) {

// tt.gitUrl is the parent devfile URL
downloadGitRepoResources = mockDownloadGitRepoResources(tt.url, tt.gitUrl, tt.token)
got, err := parseFromURI(tt.importReference, curDevfileContext, &resolutionContextTree{}, resolverTools{})
got, err := parseFromURI(tt.importReference, curDevfileContext, &resolutionContextTree{}, resolverTools{downloadGitResources: tt.downloadGitResources})

// validate even if we want an error; check that no files are copied to destDir
validateGitResourceFunctions(t, tt.wantResources, tt.wantResourceContent, destDir)

if (err != nil) != (tt.wantError != nil) {
t.Errorf("Unexpected error: %v, wantErr %v", err, tt.wantError)
t.Errorf("Unexpected error: %v, wantErr %v", err, *tt.wantError)
} else if err == nil && !reflect.DeepEqual(got.Data, tt.wantDevFile.Data) {
t.Errorf("Wanted: %v, got: %v, difference at %v", tt.wantDevFile, got, pretty.Compare(tt.wantDevFile, got))
} else if err != nil {
Expand Down

0 comments on commit 6c4a5b8

Please sign in to comment.