Skip to content

Commit

Permalink
Support various target and resource with go-getter
Browse files Browse the repository at this point in the history
  • Loading branch information
k8s-ci-robot authored and yujunz committed Mar 24, 2020
1 parent 3a2635b commit 51b29d7
Show file tree
Hide file tree
Showing 9 changed files with 228 additions and 45 deletions.
1 change: 1 addition & 0 deletions api/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ require (
github.com/golangci/golangci-lint v1.21.0
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
github.com/pkg/errors v0.8.1
github.com/yujunz/go-getter v1.4.1-lite
golang.org/x/tools v0.0.0-20191010075000-0337d82405ff
gopkg.in/yaml.v2 v2.2.4
k8s.io/api v0.17.0
Expand Down
15 changes: 15 additions & 0 deletions api/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1UJrqV3uuy861HCTo708pDMbjHHdCas=
github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4=
github.com/bombsimon/wsl v1.2.5 h1:9gTOkIwVtoDZywvX802SDHokeX4kW1cKnV8ZTVAPkRs=
github.com/bombsimon/wsl v1.2.5/go.mod h1:43lEF/i0kpXbLCeDXL9LMT8c92HyBywXb0AsgMHYngM=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
Expand Down Expand Up @@ -176,6 +178,12 @@ github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:Fecb
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
github.com/hashicorp/go-cleanhttp v0.5.0 h1:wvCrVc9TjDls6+YGAF2hAifE1E5U1+b4tH6KdvN3Gig=
github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo=
github.com/hashicorp/go-safetemp v1.0.0/go.mod h1:oaerMy3BhqiTbVye6QuFhFtIceqFoDHxNAB65b+Rj1I=
github.com/hashicorp/go-version v1.1.0 h1:bPIoEKD27tNdebFGGxxYwcL4nepeY4j1QP23PFRGzg0=
github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
Expand Down Expand Up @@ -225,9 +233,12 @@ github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk=
github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0=
github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
Expand Down Expand Up @@ -318,6 +329,8 @@ github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e/go.mod h1:Qimiff
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/ulikunitz/xz v0.5.5 h1:pFrO0lVpTBXLpYw+pnLj6TbvHuyjXMfjGeCwSqCVwok=
github.com/ulikunitz/xz v0.5.5/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
github.com/ultraware/funlen v0.0.2 h1:Av96YVBwwNSe4MLR7iI/BIa3VyI7/djnto/pK3Uxbdo=
github.com/ultraware/funlen v0.0.2/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA=
github.com/ultraware/whitespace v0.0.4 h1:If7Va4cM03mpgrNH9k49/VOicWpGoG70XPBFFODYDsg=
Expand All @@ -330,6 +343,8 @@ github.com/valyala/quicktemplate v1.2.0/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOV
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
github.com/yujunz/go-getter v1.4.1-lite h1:FhvNc94AXMZkfqUwfMKhnQEC9phkphSGdPTL7tIdhOM=
github.com/yujunz/go-getter v1.4.1-lite/go.mod h1:sbmqxXjyLunH1PkF3n7zSlnVeMvmYUuIl9ZVs/7NyCc=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
Expand Down
3 changes: 2 additions & 1 deletion api/krusty/kustomizer_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
package krusty_test

import (
"strings"
"testing"

"sigs.k8s.io/kustomize/api/filesys"
Expand All @@ -21,7 +22,7 @@ func TestEmptyFileSystem(t *testing.T) {
if err == nil {
t.Fatalf("expected error")
}
if err.Error() != "'noSuchThing' doesn't exist" {
if !strings.Contains(err.Error(), "'noSuchThing' doesn't exist") {
t.Fatalf("unexpected error: %v", err)
}
}
48 changes: 32 additions & 16 deletions api/loader/fileloader.go
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,13 @@ type fileLoader struct {
// Used to clone repositories.
cloner git.Cloner

// If this is non-nil, the files were
// obtained from the given resource
rscSpec *resourceSpec

// Used to get resources
getter resourceGetter

// Used to clean up, as needed.
cleaner func() error
}
Expand Down Expand Up @@ -127,20 +134,21 @@ func newLoaderOrDie(
log.Fatalf("unable to make loader at '%s'; %v", path, err)
}
return newLoaderAtConfirmedDir(
lr, root, fSys, nil, git.ClonerUsingGitExec)
lr, root, fSys, nil, git.ClonerUsingGitExec, getResource)
}

// newLoaderAtConfirmedDir returns a new fileLoader with given root.
func newLoaderAtConfirmedDir(
lr LoadRestrictorFunc,
root filesys.ConfirmedDir, fSys filesys.FileSystem,
referrer *fileLoader, cloner git.Cloner) *fileLoader {
referrer *fileLoader, cloner git.Cloner, getter resourceGetter) *fileLoader {
return &fileLoader{
loadRestrictor: lr,
root: root,
referrer: referrer,
fSys: fSys,
cloner: cloner,
getter: getter,
cleaner: func() error { return nil },
}
}
Expand Down Expand Up @@ -170,37 +178,44 @@ func (fl *fileLoader) New(path string) (ifc.Loader, error) {
if path == "" {
return nil, fmt.Errorf("new root cannot be empty")
}
repoSpec, err := git.NewRepoSpecFromUrl(path)
if err == nil {

ldr, errGet := newResourceGetter(path, fl.fSys, nil, fl.cloner, fl.getter)
if errGet == nil {
return ldr, nil
}

repoSpec, errGit := git.NewRepoSpecFromUrl(path)
if errGit == nil {
// Treat this as git repo clone request.
if err := fl.errIfRepoCycle(repoSpec); err != nil {
return nil, err
if errGit := fl.errIfRepoCycle(repoSpec); errGit != nil {
return nil, errGit
}
return newLoaderAtGitClone(
repoSpec, fl.fSys, fl, fl.cloner)
repoSpec, fl.fSys, fl, fl.cloner, fl.getter)
}

if filepath.IsAbs(path) {
return nil, fmt.Errorf("new root '%s' cannot be absolute", path)
}
root, err := demandDirectoryRoot(fl.fSys, fl.root.Join(path))
if err != nil {
return nil, err
root, errDir := demandDirectoryRoot(fl.fSys, fl.root.Join(path))
if errDir != nil {
return nil, fmt.Errorf("Error loading %s with git: %v, dir: %v, get: %v", path, errGit, errDir, errGet)
}
if err := fl.errIfGitContainmentViolation(root); err != nil {
return nil, err
if errDir := fl.errIfGitContainmentViolation(root); errDir != nil {
return nil, errDir
}
if err := fl.errIfArgEqualOrHigher(root); err != nil {
return nil, err
if errDir := fl.errIfArgEqualOrHigher(root); errDir != nil {
return nil, errDir
}
return newLoaderAtConfirmedDir(
fl.loadRestrictor, root, fl.fSys, fl, fl.cloner), nil
fl.loadRestrictor, root, fl.fSys, fl, fl.cloner, fl.getter), nil
}

// newLoaderAtGitClone returns a new Loader pinned to a temporary
// directory holding a cloned git repo.
func newLoaderAtGitClone(
repoSpec *git.RepoSpec, fSys filesys.FileSystem,
referrer *fileLoader, cloner git.Cloner) (ifc.Loader, error) {
referrer *fileLoader, cloner git.Cloner, getter resourceGetter) (ifc.Loader, error) {
cleaner := repoSpec.Cleaner(fSys)
err := cloner(repoSpec)
if err != nil {
Expand Down Expand Up @@ -230,6 +245,7 @@ func newLoaderAtGitClone(
repoSpec: repoSpec,
fSys: fSys,
cloner: cloner,
getter: getter,
cleaner: cleaner,
}, nil
}
Expand Down
10 changes: 5 additions & 5 deletions api/loader/fileloader_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -388,7 +388,7 @@ whatever
}
l, err := newLoaderAtGitClone(
repoSpec, fSys, nil,
git.DoNothingCloner(filesys.ConfirmedDir(coRoot)))
git.DoNothingCloner(filesys.ConfirmedDir(coRoot)), getNothing)
if err != nil {
t.Fatalf("unexpected err: %v\n", err)
}
Expand Down Expand Up @@ -467,7 +467,7 @@ func TestLoaderDisallowsLocalBaseFromRemoteOverlay(t *testing.T) {
}
l1, err = newLoaderAtGitClone(
repoSpec, fSys, nil,
git.DoNothingCloner(filesys.ConfirmedDir(cloneRoot)))
git.DoNothingCloner(filesys.ConfirmedDir(cloneRoot)), getNothing)
if err != nil {
t.Fatalf("unexpected err: %v\n", err)
}
Expand Down Expand Up @@ -506,7 +506,7 @@ func TestLocalLoaderReferencingGitBase(t *testing.T) {
}
l1 := newLoaderAtConfirmedDir(
RestrictionRootOnly, root, fSys, nil,
git.DoNothingCloner(filesys.ConfirmedDir(cloneRoot)))
git.DoNothingCloner(filesys.ConfirmedDir(cloneRoot)), getNothing)
if l1.Root() != topDir {
t.Fatalf("unexpected root %s", l1.Root())
}
Expand All @@ -532,7 +532,7 @@ func TestRepoDirectCycleDetection(t *testing.T) {
}
l1 := newLoaderAtConfirmedDir(
RestrictionRootOnly, root, fSys, nil,
git.DoNothingCloner(filesys.ConfirmedDir(cloneRoot)))
git.DoNothingCloner(filesys.ConfirmedDir(cloneRoot)), getNothing)
p1 := "github.com/someOrg/someRepo/foo"
rs1, err := git.NewRepoSpecFromUrl(p1)
if err != nil {
Expand Down Expand Up @@ -561,7 +561,7 @@ func TestRepoIndirectCycleDetection(t *testing.T) {
}
l0 := newLoaderAtConfirmedDir(
RestrictionRootOnly, root, fSys, nil,
git.DoNothingCloner(filesys.ConfirmedDir(cloneRoot)))
git.DoNothingCloner(filesys.ConfirmedDir(cloneRoot)), getNothing)

p1 := "github.com/someOrg/someRepo1"
p2 := "github.com/someOrg/someRepo2"
Expand Down
25 changes: 17 additions & 8 deletions api/loader/loader.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
package loader

import (
"fmt"

"sigs.k8s.io/kustomize/api/filesys"
"sigs.k8s.io/kustomize/api/ifc"
"sigs.k8s.io/kustomize/api/internal/git"
Expand All @@ -19,16 +21,23 @@ import (
func NewLoader(
lr LoadRestrictorFunc,
target string, fSys filesys.FileSystem) (ifc.Loader, error) {
repoSpec, err := git.NewRepoSpecFromUrl(target)
if err == nil {

ldr, errGet := newResourceGetter(target, fSys, nil, git.ClonerUsingGitExec, getResource)
if errGet == nil {
return ldr, nil
}

repoSpec, errGit := git.NewRepoSpecFromUrl(target)
if errGit == nil {
// The target qualifies as a remote git target.
return newLoaderAtGitClone(
repoSpec, fSys, nil, git.ClonerUsingGitExec)
repoSpec, fSys, nil, git.ClonerUsingGitExec, getResource)
}
root, err := demandDirectoryRoot(fSys, target)
if err != nil {
return nil, err

root, errDir := demandDirectoryRoot(fSys, target)
if errDir == nil {
return newLoaderAtConfirmedDir(lr, root, fSys, nil, git.ClonerUsingGitExec, getResource), nil
}
return newLoaderAtConfirmedDir(
lr, root, fSys, nil, git.ClonerUsingGitExec), nil

return nil, fmt.Errorf("Error creating new loader with git: %v, dir: %v, get: %v", errGit, errDir, errGet)
}
103 changes: 103 additions & 0 deletions api/loader/resource.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
// Copyright 2020 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0

package loader

import (
"context"
"log"
"os"

"github.com/yujunz/go-getter"
"sigs.k8s.io/kustomize/api/filesys"
"sigs.k8s.io/kustomize/api/ifc"
"sigs.k8s.io/kustomize/api/internal/git"
)

type resourceSpec struct {
// Raw is the original resource in kustomization.yaml
Raw string

// Dir is where the resource is saved
Dir filesys.ConfirmedDir
}

// Getter is a function that can gets resource
type resourceGetter func(rs *resourceSpec) error

func newResourceGetter(raw string, fSys filesys.FileSystem, referrer *fileLoader, cloner git.Cloner, getter resourceGetter) (ifc.Loader, error) {
rs := &resourceSpec{
Raw: raw,
}

cleaner := func() error {
return fSys.RemoveAll(rs.Dir.String())
}

if err := getter(rs); err != nil {
cleaner()
return nil, err
}

// TODO(yujunz): check file or directory

return &fileLoader{
loadRestrictor: RestrictionRootOnly,
// TODO(yujunz): limit to getter root
root: rs.Dir,
referrer: referrer,
fSys: fSys,
cloner: cloner,
rscSpec: rs,
getter: getter,
cleaner: cleaner,
}, nil
}

func getResource(rs *resourceSpec) error {
var err error

rs.Dir, err = filesys.NewTmpConfirmedDir()
if err != nil {
return err
}

// Get the pwd
pwd, err := os.Getwd()
if err != nil {
log.Fatalf("Error getting wd: %s", err)
}

opts := []getter.ClientOption{}
client := &getter.Client{
Ctx: context.TODO(),
Src: rs.Raw,
Dst: rs.Dir.String(),
Pwd: pwd,
Mode: getter.ClientModeAny,
Detectors: []getter.Detector{
new(getter.GitHubDetector),
new(getter.GitDetector),
new(getter.BitBucketDetector),
},
Options: opts,
}
return client.Get()
}

func getNothing(rs *resourceSpec) error {
var err error
rs.Dir, err = filesys.NewTmpConfirmedDir()
if err != nil {
return err
}

// Get the pwd
pwd, err := os.Getwd()
if err != nil {
log.Fatalf("Error getting wd: %s", err)
}

_, err = getter.Detect(rs.Raw, pwd, []getter.Detector{})
return err
}
Loading

0 comments on commit 51b29d7

Please sign in to comment.