Merge pull request #2169 from yujunz/loader/go-getter

Replace git cloner with go getter to support various target

api/go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/golangci/golangci-lint v1.21.0
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
 	github.com/pkg/errors v0.8.1
+	github.com/yujunz/go-getter v1.5.3-lite
 	golang.org/x/tools v0.0.0-20191010075000-0337d82405ff
 	gopkg.in/yaml.v2 v2.2.4
 	k8s.io/api v0.17.0

api/go.sum

@@ -26,6 +26,8 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1UJrqV3uuy861HCTo708pDMbjHHdCas=
+github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4=
 github.com/bombsimon/wsl v1.2.5 h1:9gTOkIwVtoDZywvX802SDHokeX4kW1cKnV8ZTVAPkRs=
 github.com/bombsimon/wsl v1.2.5/go.mod h1:43lEF/i0kpXbLCeDXL9LMT8c92HyBywXb0AsgMHYngM=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
@@ -176,6 +178,12 @@ github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:Fecb
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/hashicorp/go-cleanhttp v0.5.0 h1:wvCrVc9TjDls6+YGAF2hAifE1E5U1+b4tH6KdvN3Gig=
+github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo=
+github.com/hashicorp/go-safetemp v1.0.0/go.mod h1:oaerMy3BhqiTbVye6QuFhFtIceqFoDHxNAB65b+Rj1I=
+github.com/hashicorp/go-version v1.1.0 h1:bPIoEKD27tNdebFGGxxYwcL4nepeY4j1QP23PFRGzg0=
+github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
@@ -225,9 +233,12 @@ github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk=
+github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0=
+github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
 github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -318,6 +329,8 @@ github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e/go.mod h1:Qimiff
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
+github.com/ulikunitz/xz v0.5.5 h1:pFrO0lVpTBXLpYw+pnLj6TbvHuyjXMfjGeCwSqCVwok=
+github.com/ulikunitz/xz v0.5.5/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
 github.com/ultraware/funlen v0.0.2 h1:Av96YVBwwNSe4MLR7iI/BIa3VyI7/djnto/pK3Uxbdo=
 github.com/ultraware/funlen v0.0.2/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA=
 github.com/ultraware/whitespace v0.0.4 h1:If7Va4cM03mpgrNH9k49/VOicWpGoG70XPBFFODYDsg=
@@ -330,6 +343,8 @@ github.com/valyala/quicktemplate v1.2.0/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOV
 github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+github.com/yujunz/go-getter v1.5.3-lite h1:0gGDu+HGEehDwcHdo3AJbykqFCjlyFoQSWKrefF5wmE=
+github.com/yujunz/go-getter v1.5.3-lite/go.mod h1:Ej4QvrukA8HF7q/OODH4s60MbreJMmdhIrCbVRfRj+Y=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=

api/internal/git/repospec.go

@@ -62,6 +62,11 @@ func (x *RepoSpec) Raw() string {
 	return x.raw
 }
 
+func (x *RepoSpec) WithRaw(s string) *RepoSpec {
+	x.raw = s
+	return x
+}
+
 func (x *RepoSpec) AbsPath() string {
 	return x.Dir.Join(x.Path)
 }

api/krusty/kustomizer_test.go

@@ -4,6 +4,7 @@
 package krusty_test
 
 import (
+	"strings"
 	"testing"
 
 	"sigs.k8s.io/kustomize/api/filesys"
@@ -21,7 +22,7 @@ func TestEmptyFileSystem(t *testing.T) {
 	if err == nil {
 		t.Fatalf("expected error")
 	}
-	if err.Error() != "'noSuchThing' doesn't exist" {
+	if !strings.Contains(err.Error(), "'noSuchThing' doesn't exist") {
 		t.Fatalf("unexpected error: %v", err)
 	}
 }

api/loader/fileloader.go

@@ -170,30 +170,49 @@ func (fl *fileLoader) New(path string) (ifc.Loader, error) {
 	if path == "" {
 		return nil, fmt.Errorf("new root cannot be empty")
 	}
-	repoSpec, err := git.NewRepoSpecFromUrl(path)
-	if err == nil {
-		// Treat this as git repo clone request.
+
+	root, errD := func() (filesys.ConfirmedDir, error) {
+		if filepath.IsAbs(path) {
+			return "", fmt.Errorf("new root '%s' cannot be absolute", path)
+		}
+		root, err := demandDirectoryRoot(fl.fSys, fl.root.Join(path))
+		if err != nil {
+			return "", err
+		}
+		if err := fl.errIfGitContainmentViolation(root); err != nil {
+			return "", err
+		}
+		if err := fl.errIfArgEqualOrHigher(root); err != nil {
+			return "", err
+		}
+		return root, nil
+	}()
+
+	if errD == nil {
+		return newLoaderAtConfirmedDir(
+			fl.loadRestrictor, root, fl.fSys, fl, fl.cloner), nil
+	}
+
+	ldr, errL := func() (ifc.Loader, error) {
+		repoSpec, err := git.NewRepoSpecFromUrl(path)
+		if err != nil {
+			return nil, err
+		}
 		if err := fl.errIfRepoCycle(repoSpec); err != nil {
 			return nil, err
 		}
-		return newLoaderAtGitClone(
-			repoSpec, fl.fSys, fl, fl.cloner)
-	}
-	if filepath.IsAbs(path) {
-		return nil, fmt.Errorf("new root '%s' cannot be absolute", path)
-	}
-	root, err := demandDirectoryRoot(fl.fSys, fl.root.Join(path))
-	if err != nil {
-		return nil, err
-	}
-	if err := fl.errIfGitContainmentViolation(root); err != nil {
-		return nil, err
-	}
-	if err := fl.errIfArgEqualOrHigher(root); err != nil {
-		return nil, err
+		ldr, err := newLoaderAtGitClone(repoSpec, fl.fSys, fl, fl.cloner)
+
+		if err != nil {
+			return nil, fmt.Errorf("Error create loader: %s", err)
+		}
+		return ldr, nil
+	}()
+
+	if errL == nil {
+		return ldr, nil
 	}
-	return newLoaderAtConfirmedDir(
-		fl.loadRestrictor, root, fl.fSys, fl, fl.cloner), nil
+	return nil, fmt.Errorf("cannot load at directory %q nor load at git clone %q", errD, errL)
 }
 
 // newLoaderAtGitClone returns a new Loader pinned to a temporary

api/loader/loader.go

@@ -5,6 +5,13 @@
 package loader
 
 import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+
+	getter "github.com/yujunz/go-getter"
+
 	"sigs.k8s.io/kustomize/api/filesys"
 	"sigs.k8s.io/kustomize/api/ifc"
 	"sigs.k8s.io/kustomize/api/internal/git"
@@ -19,16 +26,43 @@ import (
 func NewLoader(
 	lr LoadRestrictorFunc,
 	target string, fSys filesys.FileSystem) (ifc.Loader, error) {
-	repoSpec, err := git.NewRepoSpecFromUrl(target)
-	if err == nil {
-		// The target qualifies as a remote git target.
-		return newLoaderAtGitClone(
-			repoSpec, fSys, nil, git.ClonerUsingGitExec)
+
+	root, errD := demandDirectoryRoot(fSys, target)
+	if errD == nil {
+		return newLoaderAtConfirmedDir(lr, root, fSys, nil, getRepo), nil
+	}
+
+	ldr, errL := newLoaderAtGitClone(
+		(&git.RepoSpec{}).WithRaw(target), fSys, nil, getRepo)
+
+	if errL != nil {
+		return nil, fmt.Errorf("Error demand directory %q and create loader %q", errD, errL)
+	}
+
+	return ldr, nil
+}
+
+func getRepo(repoSpec *git.RepoSpec) error {
+	var err error
+	repoSpec.Dir, err = filesys.NewTmpConfirmedDir()
+	if err != nil {
+		return err
 	}
-	root, err := demandDirectoryRoot(fSys, target)
+
+	// Get the pwd
+	pwd, err := os.Getwd()
 	if err != nil {
-		return nil, err
+		log.Fatalf("Error getting wd: %s", err)
+	}
+
+	opts := []getter.ClientOption{}
+	client := &getter.Client{
+		Ctx:     context.TODO(),
+		Src:     repoSpec.Raw(),
+		Dst:     repoSpec.Dir.String(),
+		Pwd:     pwd,
+		Mode:    getter.ClientModeAny,
+		Options: opts,
 	}
-	return newLoaderAtConfirmedDir(
-		lr, root, fSys, nil, git.ClonerUsingGitExec), nil
+	return client.Get()
 }