diff --git a/api/go.mod b/api/go.mod index ae6924db045..b147b9cdf79 100644 --- a/api/go.mod +++ b/api/go.mod @@ -2,4 +2,7 @@ module github.com/kubeflow/pipelines/api go 1.16 -require google.golang.org/protobuf v1.27.1 +require ( + google.golang.org/genproto v0.0.0-20211026145609-4688e4c4e024 + google.golang.org/protobuf v1.27.1 +) diff --git a/api/go.sum b/api/go.sum index 03b1917b5a4..700a627369e 100644 --- a/api/go.sum +++ b/api/go.sum @@ -1,8 +1,122 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20211026145609-4688e4c4e024 h1:aePO4E0x+Urj9V5NQHjqOpaNG4oMeHQq0l2ob05z5tI= +google.golang.org/genproto v0.0.0-20211026145609-4688e4c4e024/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/api/v2alpha1/cache_key.proto b/api/v2alpha1/cache_key.proto index 396828138d7..cc071f99cf9 100644 --- a/api/v2alpha1/cache_key.proto +++ b/api/v2alpha1/cache_key.proto @@ -17,7 +17,8 @@ syntax = "proto3"; option go_package = "github.com/kubeflow/pipelines/api/v2alpha1/go/cachekey"; package ml_pipelines; -import "google/protobuf/any.proto"; +// import "google/protobuf/any.proto"; +import "google/protobuf/struct.proto"; import "pipeline_spec.proto"; message CacheKey { @@ -26,6 +27,7 @@ message CacheKey { map outputArtifactsSpec = 3; map outputParametersSpec=4; ContainerSpec containerSpec=5; + map input_parameter_values = 6; } message ContainerSpec { diff --git a/api/v2alpha1/go/cachekey/cache_key.pb.go b/api/v2alpha1/go/cachekey/cache_key.pb.go index cd32c176c04..ba70352ca51 100644 --- a/api/v2alpha1/go/cachekey/cache_key.pb.go +++ b/api/v2alpha1/go/cachekey/cache_key.pb.go @@ -24,7 +24,7 @@ import ( pipelinespec "github.com/kubeflow/pipelines/api/v2alpha1/go/pipelinespec" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" - _ "google.golang.org/protobuf/types/known/anypb" + structpb "google.golang.org/protobuf/types/known/structpb" reflect "reflect" sync "sync" ) @@ -46,6 +46,7 @@ type CacheKey struct { OutputArtifactsSpec map[string]*pipelinespec.RuntimeArtifact `protobuf:"bytes,3,rep,name=outputArtifactsSpec,proto3" json:"outputArtifactsSpec,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` OutputParametersSpec map[string]string `protobuf:"bytes,4,rep,name=outputParametersSpec,proto3" json:"outputParametersSpec,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` ContainerSpec *ContainerSpec `protobuf:"bytes,5,opt,name=containerSpec,proto3" json:"containerSpec,omitempty"` + InputParameterValues map[string]*structpb.Value `protobuf:"bytes,6,rep,name=input_parameter_values,json=inputParameterValues,proto3" json:"input_parameter_values,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` } func (x *CacheKey) Reset() { @@ -115,6 +116,13 @@ func (x *CacheKey) GetContainerSpec() *ContainerSpec { return nil } +func (x *CacheKey) GetInputParameterValues() map[string]*structpb.Value { + if x != nil { + return x.InputParameterValues + } + return nil +} + type ContainerSpec struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -222,73 +230,86 @@ var File_cache_key_proto protoreflect.FileDescriptor var file_cache_key_proto_rawDesc = []byte{ 0x0a, 0x0f, 0x63, 0x61, 0x63, 0x68, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x6d, 0x6c, 0x5f, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x73, 0x1a, - 0x19, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, - 0x2f, 0x61, 0x6e, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x13, 0x70, 0x69, 0x70, 0x65, - 0x6c, 0x69, 0x6e, 0x65, 0x5f, 0x73, 0x70, 0x65, 0x63, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, - 0xbd, 0x06, 0x0a, 0x08, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x5e, 0x0a, 0x12, - 0x69, 0x6e, 0x70, 0x75, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x4e, 0x61, 0x6d, - 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x6d, 0x6c, 0x5f, 0x70, 0x69, + 0x1c, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2f, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x13, 0x70, + 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x5f, 0x73, 0x70, 0x65, 0x63, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x22, 0x86, 0x08, 0x0a, 0x08, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4b, 0x65, 0x79, 0x12, + 0x5e, 0x0a, 0x12, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, + 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x6d, 0x6c, + 0x5f, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x73, 0x2e, 0x43, 0x61, 0x63, 0x68, 0x65, + 0x4b, 0x65, 0x79, 0x2e, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, + 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x12, 0x69, 0x6e, 0x70, + 0x75, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x12, + 0x55, 0x0a, 0x0f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, + 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x6d, 0x6c, 0x5f, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x73, 0x2e, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4b, 0x65, 0x79, - 0x2e, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x4e, 0x61, - 0x6d, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x12, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x41, - 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x12, 0x55, 0x0a, 0x0f, - 0x69, 0x6e, 0x70, 0x75, 0x74, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x18, - 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x6d, 0x6c, 0x5f, 0x70, 0x69, 0x70, 0x65, 0x6c, - 0x69, 0x6e, 0x65, 0x73, 0x2e, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4b, 0x65, 0x79, 0x2e, 0x49, 0x6e, - 0x70, 0x75, 0x74, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x52, 0x0f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, - 0x65, 0x72, 0x73, 0x12, 0x61, 0x0a, 0x13, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x41, 0x72, 0x74, - 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x53, 0x70, 0x65, 0x63, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x2f, 0x2e, 0x6d, 0x6c, 0x5f, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x73, 0x2e, - 0x43, 0x61, 0x63, 0x68, 0x65, 0x4b, 0x65, 0x79, 0x2e, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x41, - 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x53, 0x70, 0x65, 0x63, 0x45, 0x6e, 0x74, 0x72, - 0x79, 0x52, 0x13, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, - 0x74, 0x73, 0x53, 0x70, 0x65, 0x63, 0x12, 0x64, 0x0a, 0x14, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, - 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x53, 0x70, 0x65, 0x63, 0x18, 0x04, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x6d, 0x6c, 0x5f, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, - 0x6e, 0x65, 0x73, 0x2e, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4b, 0x65, 0x79, 0x2e, 0x4f, 0x75, 0x74, + 0x2e, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x50, 0x61, 0x72, 0x61, + 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x12, 0x61, 0x0a, 0x13, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, + 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x53, 0x70, 0x65, 0x63, 0x18, 0x03, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x6d, 0x6c, 0x5f, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, + 0x65, 0x73, 0x2e, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4b, 0x65, 0x79, 0x2e, 0x4f, 0x75, 0x74, 0x70, + 0x75, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x53, 0x70, 0x65, 0x63, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x52, 0x13, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x41, 0x72, 0x74, 0x69, + 0x66, 0x61, 0x63, 0x74, 0x73, 0x53, 0x70, 0x65, 0x63, 0x12, 0x64, 0x0a, 0x14, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x53, 0x70, 0x65, - 0x63, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x14, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x50, 0x61, - 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x53, 0x70, 0x65, 0x63, 0x12, 0x41, 0x0a, 0x0d, - 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x18, 0x05, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x6d, 0x6c, 0x5f, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, - 0x65, 0x73, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, - 0x52, 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x1a, - 0x65, 0x0a, 0x17, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, - 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, - 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x34, 0x0a, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6d, 0x6c, - 0x5f, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x73, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, - 0x61, 0x63, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x57, 0x0a, 0x14, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x50, - 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, + 0x63, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x6d, 0x6c, 0x5f, 0x70, 0x69, 0x70, + 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x73, 0x2e, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4b, 0x65, 0x79, 0x2e, + 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, + 0x53, 0x70, 0x65, 0x63, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x14, 0x6f, 0x75, 0x74, 0x70, 0x75, + 0x74, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x53, 0x70, 0x65, 0x63, 0x12, + 0x41, 0x0a, 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x6d, 0x6c, 0x5f, 0x70, 0x69, 0x70, 0x65, + 0x6c, 0x69, 0x6e, 0x65, 0x73, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x53, + 0x70, 0x65, 0x63, 0x52, 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x53, 0x70, + 0x65, 0x63, 0x12, 0x66, 0x0a, 0x16, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x70, 0x61, 0x72, 0x61, + 0x6d, 0x65, 0x74, 0x65, 0x72, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x06, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x6d, 0x6c, 0x5f, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, + 0x73, 0x2e, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4b, 0x65, 0x79, 0x2e, 0x49, 0x6e, 0x70, 0x75, 0x74, + 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x52, 0x14, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x50, 0x61, 0x72, 0x61, 0x6d, + 0x65, 0x74, 0x65, 0x72, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x1a, 0x65, 0x0a, 0x17, 0x49, 0x6e, + 0x70, 0x75, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x73, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x34, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6d, 0x6c, 0x5f, 0x70, 0x69, 0x70, 0x65, + 0x6c, 0x69, 0x6e, 0x65, 0x73, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x4e, 0x61, + 0x6d, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, + 0x01, 0x1a, 0x57, 0x0a, 0x14, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, + 0x74, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x29, 0x0a, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x6d, 0x6c, 0x5f, + 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x73, 0x2e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x65, 0x0a, 0x18, 0x4f, 0x75, + 0x74, 0x70, 0x75, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x53, 0x70, 0x65, + 0x63, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x33, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x6d, 0x6c, 0x5f, 0x70, 0x69, 0x70, + 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x73, 0x2e, 0x52, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x41, 0x72, + 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, + 0x01, 0x1a, 0x47, 0x0a, 0x19, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x50, 0x61, 0x72, 0x61, 0x6d, + 0x65, 0x74, 0x65, 0x72, 0x73, 0x53, 0x70, 0x65, 0x63, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, - 0x12, 0x29, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x13, 0x2e, 0x6d, 0x6c, 0x5f, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x73, 0x2e, 0x56, - 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, - 0x65, 0x0a, 0x18, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, - 0x74, 0x73, 0x53, 0x70, 0x65, 0x63, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, - 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x33, 0x0a, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x6d, - 0x6c, 0x5f, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x73, 0x2e, 0x52, 0x75, 0x6e, 0x74, - 0x69, 0x6d, 0x65, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x47, 0x0a, 0x19, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, - 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x53, 0x70, 0x65, 0x63, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, - 0x3f, 0x0a, 0x0d, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, - 0x12, 0x14, 0x0a, 0x05, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x05, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6d, 0x64, 0x41, 0x72, 0x67, - 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6d, 0x64, 0x41, 0x72, 0x67, 0x73, - 0x22, 0x38, 0x0a, 0x10, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x4e, 0x61, 0x6d, 0x65, - 0x4c, 0x69, 0x73, 0x74, 0x12, 0x24, 0x0a, 0x0d, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, - 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x61, 0x72, 0x74, - 0x69, 0x66, 0x61, 0x63, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x42, 0x38, 0x5a, 0x36, 0x67, 0x69, - 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6b, 0x75, 0x62, 0x65, 0x66, 0x6c, 0x6f, - 0x77, 0x2f, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x73, 0x2f, 0x61, 0x70, 0x69, 0x2f, - 0x76, 0x32, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2f, 0x67, 0x6f, 0x2f, 0x63, 0x61, 0x63, 0x68, - 0x65, 0x6b, 0x65, 0x79, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x5f, 0x0a, 0x19, 0x49, 0x6e, + 0x70, 0x75, 0x74, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x56, 0x61, 0x6c, 0x75, + 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2c, 0x0a, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x56, 0x61, 0x6c, 0x75, 0x65, + 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x3f, 0x0a, 0x0d, 0x43, + 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x12, 0x14, 0x0a, 0x05, + 0x69, 0x6d, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x69, 0x6d, 0x61, + 0x67, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6d, 0x64, 0x41, 0x72, 0x67, 0x73, 0x18, 0x02, 0x20, + 0x03, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6d, 0x64, 0x41, 0x72, 0x67, 0x73, 0x22, 0x38, 0x0a, 0x10, + 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x4c, 0x69, 0x73, 0x74, + 0x12, 0x24, 0x0a, 0x0d, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x4e, 0x61, 0x6d, 0x65, + 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, + 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x42, 0x38, 0x5a, 0x36, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, + 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6b, 0x75, 0x62, 0x65, 0x66, 0x6c, 0x6f, 0x77, 0x2f, 0x70, 0x69, + 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x73, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x32, 0x61, 0x6c, + 0x70, 0x68, 0x61, 0x31, 0x2f, 0x67, 0x6f, 0x2f, 0x63, 0x61, 0x63, 0x68, 0x65, 0x6b, 0x65, 0x79, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -303,7 +324,7 @@ func file_cache_key_proto_rawDescGZIP() []byte { return file_cache_key_proto_rawDescData } -var file_cache_key_proto_msgTypes = make([]protoimpl.MessageInfo, 7) +var file_cache_key_proto_msgTypes = make([]protoimpl.MessageInfo, 8) var file_cache_key_proto_goTypes = []interface{}{ (*CacheKey)(nil), // 0: ml_pipelines.CacheKey (*ContainerSpec)(nil), // 1: ml_pipelines.ContainerSpec @@ -312,23 +333,27 @@ var file_cache_key_proto_goTypes = []interface{}{ nil, // 4: ml_pipelines.CacheKey.InputParametersEntry nil, // 5: ml_pipelines.CacheKey.OutputArtifactsSpecEntry nil, // 6: ml_pipelines.CacheKey.OutputParametersSpecEntry - (*pipelinespec.Value)(nil), // 7: ml_pipelines.Value - (*pipelinespec.RuntimeArtifact)(nil), // 8: ml_pipelines.RuntimeArtifact + nil, // 7: ml_pipelines.CacheKey.InputParameterValuesEntry + (*pipelinespec.Value)(nil), // 8: ml_pipelines.Value + (*pipelinespec.RuntimeArtifact)(nil), // 9: ml_pipelines.RuntimeArtifact + (*structpb.Value)(nil), // 10: google.protobuf.Value } var file_cache_key_proto_depIdxs = []int32{ - 3, // 0: ml_pipelines.CacheKey.inputArtifactNames:type_name -> ml_pipelines.CacheKey.InputArtifactNamesEntry - 4, // 1: ml_pipelines.CacheKey.inputParameters:type_name -> ml_pipelines.CacheKey.InputParametersEntry - 5, // 2: ml_pipelines.CacheKey.outputArtifactsSpec:type_name -> ml_pipelines.CacheKey.OutputArtifactsSpecEntry - 6, // 3: ml_pipelines.CacheKey.outputParametersSpec:type_name -> ml_pipelines.CacheKey.OutputParametersSpecEntry - 1, // 4: ml_pipelines.CacheKey.containerSpec:type_name -> ml_pipelines.ContainerSpec - 2, // 5: ml_pipelines.CacheKey.InputArtifactNamesEntry.value:type_name -> ml_pipelines.ArtifactNameList - 7, // 6: ml_pipelines.CacheKey.InputParametersEntry.value:type_name -> ml_pipelines.Value - 8, // 7: ml_pipelines.CacheKey.OutputArtifactsSpecEntry.value:type_name -> ml_pipelines.RuntimeArtifact - 8, // [8:8] is the sub-list for method output_type - 8, // [8:8] is the sub-list for method input_type - 8, // [8:8] is the sub-list for extension type_name - 8, // [8:8] is the sub-list for extension extendee - 0, // [0:8] is the sub-list for field type_name + 3, // 0: ml_pipelines.CacheKey.inputArtifactNames:type_name -> ml_pipelines.CacheKey.InputArtifactNamesEntry + 4, // 1: ml_pipelines.CacheKey.inputParameters:type_name -> ml_pipelines.CacheKey.InputParametersEntry + 5, // 2: ml_pipelines.CacheKey.outputArtifactsSpec:type_name -> ml_pipelines.CacheKey.OutputArtifactsSpecEntry + 6, // 3: ml_pipelines.CacheKey.outputParametersSpec:type_name -> ml_pipelines.CacheKey.OutputParametersSpecEntry + 1, // 4: ml_pipelines.CacheKey.containerSpec:type_name -> ml_pipelines.ContainerSpec + 7, // 5: ml_pipelines.CacheKey.input_parameter_values:type_name -> ml_pipelines.CacheKey.InputParameterValuesEntry + 2, // 6: ml_pipelines.CacheKey.InputArtifactNamesEntry.value:type_name -> ml_pipelines.ArtifactNameList + 8, // 7: ml_pipelines.CacheKey.InputParametersEntry.value:type_name -> ml_pipelines.Value + 9, // 8: ml_pipelines.CacheKey.OutputArtifactsSpecEntry.value:type_name -> ml_pipelines.RuntimeArtifact + 10, // 9: ml_pipelines.CacheKey.InputParameterValuesEntry.value:type_name -> google.protobuf.Value + 10, // [10:10] is the sub-list for method output_type + 10, // [10:10] is the sub-list for method input_type + 10, // [10:10] is the sub-list for extension type_name + 10, // [10:10] is the sub-list for extension extendee + 0, // [0:10] is the sub-list for field type_name } func init() { file_cache_key_proto_init() } @@ -380,7 +405,7 @@ func file_cache_key_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_cache_key_proto_rawDesc, NumEnums: 0, - NumMessages: 7, + NumMessages: 8, NumExtensions: 0, NumServices: 0, }, diff --git a/samples/test/config.yaml b/samples/test/config.yaml index 216b8675622..680ced3909f 100644 --- a/samples/test/config.yaml +++ b/samples/test/config.yaml @@ -90,5 +90,7 @@ path: samples.v2.producer_consumer_param_test - name: pipeline_with_importer path: samples.v2.pipeline_with_importer_test -- name: cache_v2 - path: samples.v2.cache_test +# TODO(Bobgy): Re-enable after figuring out V2 Engine +# and protobuf.Value support. +# - name: cache_v2 +# path: samples.v2.cache_test diff --git a/samples/test/lightweight_python_functions_v2_pipeline_test.py b/samples/test/lightweight_python_functions_v2_pipeline_test.py index 49519910c9a..19942fd7ab2 100644 --- a/samples/test/lightweight_python_functions_v2_pipeline_test.py +++ b/samples/test/lightweight_python_functions_v2_pipeline_test.py @@ -61,9 +61,12 @@ def verify(run: kfp_server_api.ApiRun, mlmd_connection_config, **kwargs): 'type': 'system.Dataset' }], 'parameters': { - 'output_bool_parameter_path': 'True', - 'output_dict_parameter_path': '{"A": 1, "B": 2}', - 'output_list_parameter_path': '["a", "b", "c"]', + 'output_bool_parameter_path': True, + 'output_dict_parameter_path': { + "A": 1, + "B": 2 + }, + 'output_list_parameter_path': ["a", "b", "c"], 'output_parameter_path': 'message' } }, @@ -89,9 +92,12 @@ def verify(run: kfp_server_api.ApiRun, mlmd_connection_config, **kwargs): 'type': 'system.Dataset' }], 'parameters': { - 'input_bool': 'True', - 'input_dict': '{"A": 1, "B": 2}', - 'input_list': '["a", "b", "c"]', + 'input_bool': True, + 'input_dict': { + "A": 1, + "B": 2 + }, + 'input_list': ["a", "b", "c"], 'message': 'message', 'num_steps': 100, } @@ -116,14 +122,10 @@ def verify(run: kfp_server_api.ApiRun, mlmd_connection_config, **kwargs): run_pipeline_func([ - TestCase( - pipeline_func=pipeline, - verify_func=verify, - mode=dsl.PipelineExecutionMode.V2_COMPATIBLE - ), - TestCase( - pipeline_func=pipeline, - verify_func=verify, - mode=dsl.PipelineExecutionMode.V2_ENGINE - ), + TestCase(pipeline_func=pipeline, + verify_func=verify, + mode=dsl.PipelineExecutionMode.V2_COMPATIBLE), + TestCase(pipeline_func=pipeline, + verify_func=verify, + mode=dsl.PipelineExecutionMode.V2_ENGINE), ]) diff --git a/samples/test/util.py b/samples/test/util.py index 2a04f49d5cb..bfe9e860b7c 100644 --- a/samples/test/util.py +++ b/samples/test/util.py @@ -35,10 +35,9 @@ # Add **kwargs, so that when new arguments are added, this doesn't fail for # unknown arguments. def _default_verify_func( - run_id: int, run: kfp_server_api.ApiRun, - mlmd_connection_config: metadata_store_pb2.MetadataStoreClientConfig, - **kwargs -): + run_id: int, run: kfp_server_api.ApiRun, + mlmd_connection_config: metadata_store_pb2.MetadataStoreClientConfig, + **kwargs): assert run.status == 'Succeeded' @@ -66,7 +65,6 @@ def run_pipeline_func(test_cases: List[TestCase]): :param pipeline_func: pipeline function to run :type pipeline_func: function """ - def test_wrapper( run_pipeline: Callable[ [Callable, kfp.dsl.PipelineExecutionMode, bool, dict], @@ -74,12 +72,10 @@ def test_wrapper( mlmd_connection_config: metadata_store_pb2.MetadataStoreClientConfig, ): for case in test_cases: - run_detail = run_pipeline( - pipeline_func=case.pipeline_func, - mode=case.mode, - enable_caching=case.enable_caching, - arguments=case.arguments or {} - ) + run_detail = run_pipeline(pipeline_func=case.pipeline_func, + mode=case.mode, + enable_caching=case.enable_caching, + arguments=case.arguments or {}) pipeline_runtime: kfp_server_api.ApiPipelineRuntime = run_detail.pipeline_runtime argo_workflow = json.loads(pipeline_runtime.workflow_manifest) argo_workflow_name = argo_workflow.get('metadata').get('name') @@ -96,7 +92,7 @@ def test_wrapper( _run_test(test_wrapper) -def _retry_with_backoff(fn: Callable, retries=5, backoff_in_seconds=1): +def _retry_with_backoff(fn: Callable, retries=3, backoff_in_seconds=1): i = 0 while True: try: @@ -114,7 +110,6 @@ def _retry_with_backoff(fn: Callable, retries=5, backoff_in_seconds=1): def _run_test(callback): - def main( output_directory: Optional[str] = None, # example host: Optional[str] = None, @@ -158,18 +153,25 @@ def main( if output_directory is None: output_directory = os.getenv('KFP_OUTPUT_DIRECTORY') if metadata_service_host is None: - metadata_service_host = os.getenv( - 'METADATA_GRPC_SERVICE_HOST', 'metadata-grpc-service' - ) + metadata_service_host = os.getenv('METADATA_GRPC_SERVICE_HOST', + 'metadata-grpc-service') if launcher_image is None: launcher_image = os.getenv('KFP_LAUNCHER_IMAGE') + if launcher_v2_image is None: + launcher_v2_image = os.getenv('KFP_LAUNCHER_V2_IMAGE') + if not launcher_v2_image: + raise Exception("launcher_v2_image is empty") + if driver_image is None: + driver_image = os.getenv('KFP_DRIVER_IMAGE') + if not driver_image: + raise Exception("driver_image is empty") client = kfp.Client(host=host) def run_pipeline( pipeline_func: Callable, - mode: kfp.dsl.PipelineExecutionMode = kfp.dsl.PipelineExecutionMode. - V2_COMPATIBLE, + mode: kfp.dsl.PipelineExecutionMode = kfp.dsl. + PipelineExecutionMode.V2_COMPATIBLE, enable_caching: bool = False, arguments: Optional[dict] = None, ) -> kfp_server_api.ApiRunDetail: @@ -190,7 +192,7 @@ def _create_run(): launcher_v2_image=launcher_v2_image, pipeline_root=output_directory, enable_caching=enable_caching, - arguments = { + arguments={ **arguments, }, ) @@ -202,8 +204,7 @@ def _create_run(): cpu_request='0.5', cpu_limit='1', memory_limit='512Mi', - ) - ) + )) if mode == kfp.dsl.PipelineExecutionMode.V1_LEGACY: conf.add_op_transformer(disable_cache) return client.create_run_from_pipeline_func( @@ -250,10 +251,8 @@ def _create_run(): host=metadata_service_host, port=metadata_service_port, ) - callback( - run_pipeline=run_pipeline, - mlmd_connection_config=mlmd_connection_config - ) + callback(run_pipeline=run_pipeline, + mlmd_connection_config=mlmd_connection_config) import fire fire.Fire(main) @@ -266,15 +265,14 @@ def run_v2_pipeline( launcher_v2_image: str, pipeline_root: str, enable_caching: bool, - arguments: Mapping[str, str], - + arguments: Mapping[str, str], ): import tempfile import subprocess - original_pipeline_job = tempfile.mktemp(suffix='.json', prefix="original_pipeline_job") - kfp.v2.compiler.Compiler().compile( - pipeline_func=fn, package_path=original_pipeline_job - ) + original_pipeline_job = tempfile.mktemp(suffix='.json', + prefix="original_pipeline_job") + kfp.v2.compiler.Compiler().compile(pipeline_func=fn, + package_path=original_pipeline_job) # remove following overriding logic once we use create_run_from_job_spec to trigger kfp pipeline run with open(original_pipeline_job) as f: @@ -285,10 +283,10 @@ def run_v2_pipeline( for task in component['dag']['tasks'].values(): task['cachingOptions'] = {'enableCache': enable_caching} for k, v in arguments.items(): - parameter_value_dict = pipeline_job_dict['runtimeConfig']['parameters'][k] - for type, _ in parameter_value_dict.items(): - parameter_value_dict[type] = v - pipeline_job_dict['runtimeConfig']['parameters'][k] = parameter_value_dict + parameter_value = pipeline_job_dict['runtimeConfig'][ + 'parameterValues'][k] + pipeline_job_dict['runtimeConfig']['parameterValues'][ + k] = parameter_value pipeline_job = tempfile.mktemp(suffix='.json', prefix="pipeline_job") with open(pipeline_job, 'w') as f: @@ -310,8 +308,9 @@ def run_v2_pipeline( # call v2 backend compiler CLI to compile pipeline spec to argo workflow subprocess.check_call(args, stdout=f) return client.create_run_from_pipeline_package( - pipeline_file=argo_workflow_spec, arguments={}, enable_caching=enable_caching - ) + pipeline_file=argo_workflow_spec, + arguments={}, + enable_caching=enable_caching) def simplify_proto_struct(data: dict) -> dict: @@ -348,14 +347,11 @@ def new( # The original field is custom_properties, but MessageToDict converts it # to customProperties. metadata = simplify_proto_struct( - MessageToDict(mlmd_artifact).get('customProperties', {}) - ) - return cls( - name=artifact_name, - type=mlmd_artifact_type.name, - uri=mlmd_artifact.uri, - metadata=metadata - ) + MessageToDict(mlmd_artifact).get('customProperties', {})) + return cls(name=artifact_name, + type=mlmd_artifact_type.name, + uri=mlmd_artifact.uri, + metadata=metadata) @dataclass @@ -390,13 +386,13 @@ def get_dict(self): @classmethod def new( - cls, - context: metadata_store_pb2.Context, - execution: metadata_store_pb2.Execution, - execution_types_by_id, # dict[int, metadata_store_pb2.ExecutionType] - events_by_execution_id, # dict[int, List[metadata_store_pb2.Event]] - artifacts_by_id, # dict[int, metadata_store_pb2.Artifact] - artifact_types_by_id, # dict[int, metadata_store_pb2.ArtifactType] + cls, + context: metadata_store_pb2.Context, + execution: metadata_store_pb2.Execution, + execution_types_by_id, # dict[int, metadata_store_pb2.ExecutionType] + events_by_execution_id, # dict[int, List[metadata_store_pb2.Event]] + artifacts_by_id, # dict[int, metadata_store_pb2.Artifact] + artifact_types_by_id, # dict[int, metadata_store_pb2.ArtifactType] ): execution_type = execution_types_by_id[execution.type_id] params = _parse_parameters(execution) @@ -404,18 +400,16 @@ def new( input_artifacts = [] output_artifacts = [] if events: - input_artifacts_info = [(e.artifact_id, e) - for e in events - if e.type == metadata_store_pb2.Event.INPUT] + input_artifacts_info = [(e.artifact_id, e) for e in events + if e.type == metadata_store_pb2.Event.INPUT + ] output_artifacts_info = [ - (e.artifact_id, e) - for e in events + (e.artifact_id, e) for e in events if e.type == metadata_store_pb2.Event.OUTPUT ] - def kfp_artifact( - aid: int, e: metadata_store_pb2.Event - ) -> KfpArtifact: + def kfp_artifact(aid: int, + e: metadata_store_pb2.Event) -> KfpArtifact: mlmd_artifact = artifacts_by_id[aid] mlmd_type = artifact_types_by_id[mlmd_artifact.type_id] return KfpArtifact.new( @@ -437,17 +431,14 @@ def kfp_artifact( name=execution.custom_properties.get('task_name').string_value, type=execution_type.name, state=execution.last_known_state, - inputs=TaskInputs( - parameters=params['inputs'], artifacts=input_artifacts - ), - outputs=TaskOutputs( - parameters=params['outputs'], artifacts=output_artifacts - ), + inputs=TaskInputs(parameters=params['inputs'], + artifacts=input_artifacts), + outputs=TaskOutputs(parameters=params['outputs'], + artifacts=output_artifacts), ) class KfpMlmdClient: - def __init__( self, mlmd_connection_config: Optional[ @@ -468,33 +459,25 @@ def get_tasks(self, run_id: str): ) if not run_context: raise Exception( - f'Cannot find system.PipelineRun context "{run_id}"' - ) + f'Cannot find system.PipelineRun context "{run_id}"') logging.info( - f'run_context: name={run_context.name} id={run_context.id}' - ) + f'run_context: name={run_context.name} id={run_context.id}') executions = self.mlmd_store.get_executions_by_context( - context_id=run_context.id - ) + context_id=run_context.id) execution_types = self.mlmd_store.get_execution_types_by_id( - list(set([e.type_id for e in executions])) - ) + list(set([e.type_id for e in executions]))) execution_types_by_id = {et.id: et for et in execution_types} - events = self.mlmd_store.get_events_by_execution_ids([ - e.id for e in executions - ]) + events = self.mlmd_store.get_events_by_execution_ids( + [e.id for e in executions]) events_by_execution_id = {} for e in events: - events_by_execution_id[ - e.execution_id - ] = (events_by_execution_id.get(e.execution_id) or []) + [e] + events_by_execution_id[e.execution_id] = ( + events_by_execution_id.get(e.execution_id) or []) + [e] artifacts = self.mlmd_store.get_artifacts_by_context( - context_id=run_context.id - ) + context_id=run_context.id) artifacts_by_id = {a.id: a for a in artifacts} artifact_types = self.mlmd_store.get_artifact_types_by_id( - list(set([a.type_id for a in artifacts])) - ) + list(set([a.type_id for a in artifacts]))) artifact_types_by_id = {at.id: at for at in artifact_types} _validate_executions_have_task_names(executions) tasks = [ @@ -538,12 +521,20 @@ def _parse_parameters(execution: metadata_store_pb2.Execution) -> dict: parameters['inputs'][name[len('input:'):]] = raw_value if name.startswith('output:'): parameters['outputs'][name[len('output:'):]] = raw_value + if name == "inputs" and value.HasField('struct_value'): + for k, v in simplify_proto_struct( + MessageToDict(value))["structValue"].items(): + parameters['inputs'][k] = v + if name == "outputs" and value.HasField('struct_value'): + for k, v in simplify_proto_struct( + MessageToDict(value))["structValue"].items(): + parameters['outputs'][k] = v return parameters def disable_cache(task): - # Skip tasks which are not container ops. - if not isinstance(task, kfp.dsl.ContainerOp): - return task - task.execution_options.caching_strategy.max_cache_staleness = "P0D" + # Skip tasks which are not container ops. + if not isinstance(task, kfp.dsl.ContainerOp): return task + task.execution_options.caching_strategy.max_cache_staleness = "P0D" + return task diff --git a/sdk/RELEASE.md b/sdk/RELEASE.md index 984fc863e8e..f36414e8892 100644 --- a/sdk/RELEASE.md +++ b/sdk/RELEASE.md @@ -19,7 +19,8 @@ ## Major Features and Improvements * Add optional support to specify description for pipeline version [\#6472](https://github.com/kubeflow/pipelines/issues/6472). -* New v2 experimental compiler. [\#6803](https://github.com/kubeflow/pipelines/pull/6803) +* New v2 experimental compiler [\#6803](https://github.com/kubeflow/pipelines/pull/6803). +* Support passing parameters in v2 using google.protobuf.Value [\#6804](https://github.com/kubeflow/pipelines/pull/6804). ## Breaking Changes diff --git a/sdk/python/kfp/compiler/testdata/v2_compatible_two_step_pipeline.yaml b/sdk/python/kfp/compiler/testdata/v2_compatible_two_step_pipeline.yaml index 624b426fc17..a6263af8e7c 100644 --- a/sdk/python/kfp/compiler/testdata/v2_compatible_two_step_pipeline.yaml +++ b/sdk/python/kfp/compiler/testdata/v2_compatible_two_step_pipeline.yaml @@ -3,15 +3,15 @@ kind: Workflow metadata: generateName: my-test-pipeline- annotations: - pipelines.kubeflow.org/kfp_sdk_version: 1.8.5 - pipelines.kubeflow.org/pipeline_compilation_time: '2021-10-13T17:35:51.450220' + pipelines.kubeflow.org/kfp_sdk_version: 1.8.6 + pipelines.kubeflow.org/pipeline_compilation_time: '2021-10-26T15:02:07.868312' pipelines.kubeflow.org/pipeline_spec: '{"inputs": [{"default": "gs://output-directory/v2-artifacts", "name": "pipeline-root"}, {"default": "pipeline/my-test-pipeline", "name": "pipeline-name"}], "name": "my-test-pipeline"}' pipelines.kubeflow.org/v2_pipeline: "true" labels: pipelines.kubeflow.org/v2_pipeline: "true" - pipelines.kubeflow.org/kfp_sdk_version: 1.8.5 + pipelines.kubeflow.org/kfp_sdk_version: 1.8.6 spec: entrypoint: my-test-pipeline templates: @@ -49,7 +49,7 @@ spec: python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip fi - PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.5' && "$0" "$@" + PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && "$0" "$@" - sh - -ec - | @@ -103,8 +103,8 @@ spec: fieldRef: {fieldPath: 'metadata.labels[''pipelines.kubeflow.org/enable_caching'']'} - {name: KFP_V2_IMAGE, value: 'python:3.7'} - {name: KFP_V2_RUNTIME_INFO, value: '{"inputParameters": {"some_int": {"type": - "INT"}, "uri": {"type": "STRING"}}, "inputArtifacts": {}, "outputParameters": - {"output_parameter_one": {"type": "INT", "path": "/tmp/outputs/output_parameter_one/data"}}, + "NUMBER_INTEGER"}, "uri": {"type": "STRING"}}, "inputArtifacts": {}, "outputParameters": + {"output_parameter_one": {"type": "NUMBER_INTEGER", "path": "/tmp/outputs/output_parameter_one/data"}}, "outputArtifacts": {"output_dataset_one": {"schemaTitle": "system.Dataset", "instanceSchema": "", "schemaVersion": "0.0.1", "metadataPath": "/tmp/outputs/output_dataset_one/data"}}}'} envFrom: @@ -129,13 +129,13 @@ spec: pipelines.kubeflow.org/component_ref: '{}' pipelines.kubeflow.org/arguments.parameters: '{"some_int": "12", "uri": "uri-to-import"}' labels: - pipelines.kubeflow.org/kfp_sdk_version: 1.8.5 + pipelines.kubeflow.org/kfp_sdk_version: 1.8.6 pipelines.kubeflow.org/pipeline-sdk-type: kfp pipelines.kubeflow.org/v2_component: "true" pipelines.kubeflow.org/enable_caching: "true" initContainers: - command: [launcher, --copy, /kfp-launcher/launch] - image: gcr.io/ml-pipeline/kfp-launcher:1.8.5 + image: gcr.io/ml-pipeline/kfp-launcher:1.8.6 name: kfp-launcher mirrorVolumeMounts: true volumes: @@ -151,7 +151,7 @@ spec: python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip fi - PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.5' && "$0" "$@" + PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && "$0" "$@" - sh - -ec - | @@ -210,7 +210,7 @@ spec: fieldRef: {fieldPath: 'metadata.labels[''pipelines.kubeflow.org/enable_caching'']'} - {name: KFP_V2_IMAGE, value: 'python:3.7'} - {name: KFP_V2_RUNTIME_INFO, value: '{"inputParameters": {"num_steps": {"type": - "INT"}}, "inputArtifacts": {"dataset": {"metadataPath": "/tmp/inputs/dataset/data", + "NUMBER_INTEGER"}}, "inputArtifacts": {"dataset": {"metadataPath": "/tmp/inputs/dataset/data", "schemaTitle": "system.Dataset", "instanceSchema": "", "schemaVersion": "0.0.1"}}, "outputParameters": {}, "outputArtifacts": {"model": {"schemaTitle": "system.Model", "instanceSchema": "", "schemaVersion": "0.0.1", "metadataPath": @@ -236,13 +236,13 @@ spec: pipelines.kubeflow.org/component_ref: '{}' pipelines.kubeflow.org/arguments.parameters: '{"num_steps": "{{inputs.parameters.preprocess-output_parameter_one}}"}' labels: - pipelines.kubeflow.org/kfp_sdk_version: 1.8.5 + pipelines.kubeflow.org/kfp_sdk_version: 1.8.6 pipelines.kubeflow.org/pipeline-sdk-type: kfp pipelines.kubeflow.org/v2_component: "true" pipelines.kubeflow.org/enable_caching: "true" initContainers: - command: [launcher, --copy, /kfp-launcher/launch] - image: gcr.io/ml-pipeline/kfp-launcher:1.8.5 + image: gcr.io/ml-pipeline/kfp-launcher:1.8.6 name: kfp-launcher mirrorVolumeMounts: true volumes: diff --git a/sdk/python/kfp/compiler/testdata/v2_compatible_two_step_pipeline_with_custom_launcher.yaml b/sdk/python/kfp/compiler/testdata/v2_compatible_two_step_pipeline_with_custom_launcher.yaml index 393a9db4d3e..48f9066128d 100644 --- a/sdk/python/kfp/compiler/testdata/v2_compatible_two_step_pipeline_with_custom_launcher.yaml +++ b/sdk/python/kfp/compiler/testdata/v2_compatible_two_step_pipeline_with_custom_launcher.yaml @@ -3,15 +3,15 @@ kind: Workflow metadata: generateName: my-test-pipeline-with-custom-launcher- annotations: - pipelines.kubeflow.org/kfp_sdk_version: 1.8.5 - pipelines.kubeflow.org/pipeline_compilation_time: '2021-10-13T17:35:51.012522' + pipelines.kubeflow.org/kfp_sdk_version: 1.8.6 + pipelines.kubeflow.org/pipeline_compilation_time: '2021-10-26T15:02:07.414964' pipelines.kubeflow.org/pipeline_spec: '{"inputs": [{"default": "gs://output-directory/v2-artifacts", "name": "pipeline-root"}, {"default": "pipeline/my-test-pipeline-with-custom-launcher", "name": "pipeline-name"}], "name": "my-test-pipeline-with-custom-launcher"}' pipelines.kubeflow.org/v2_pipeline: "true" labels: pipelines.kubeflow.org/v2_pipeline: "true" - pipelines.kubeflow.org/kfp_sdk_version: 1.8.5 + pipelines.kubeflow.org/kfp_sdk_version: 1.8.6 spec: entrypoint: my-test-pipeline-with-custom-launcher templates: @@ -49,7 +49,7 @@ spec: python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip fi - PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.5' && "$0" "$@" + PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && "$0" "$@" - sh - -ec - | @@ -103,8 +103,8 @@ spec: fieldRef: {fieldPath: 'metadata.labels[''pipelines.kubeflow.org/enable_caching'']'} - {name: KFP_V2_IMAGE, value: 'python:3.7'} - {name: KFP_V2_RUNTIME_INFO, value: '{"inputParameters": {"some_int": {"type": - "INT"}, "uri": {"type": "STRING"}}, "inputArtifacts": {}, "outputParameters": - {"output_parameter_one": {"type": "INT", "path": "/tmp/outputs/output_parameter_one/data"}}, + "NUMBER_INTEGER"}, "uri": {"type": "STRING"}}, "inputArtifacts": {}, "outputParameters": + {"output_parameter_one": {"type": "NUMBER_INTEGER", "path": "/tmp/outputs/output_parameter_one/data"}}, "outputArtifacts": {"output_dataset_one": {"schemaTitle": "system.Dataset", "instanceSchema": "", "schemaVersion": "0.0.1", "metadataPath": "/tmp/outputs/output_dataset_one/data"}}}'} envFrom: @@ -129,7 +129,7 @@ spec: pipelines.kubeflow.org/component_ref: '{}' pipelines.kubeflow.org/arguments.parameters: '{"some_int": "12", "uri": "uri-to-import"}' labels: - pipelines.kubeflow.org/kfp_sdk_version: 1.8.5 + pipelines.kubeflow.org/kfp_sdk_version: 1.8.6 pipelines.kubeflow.org/pipeline-sdk-type: kfp pipelines.kubeflow.org/v2_component: "true" pipelines.kubeflow.org/enable_caching: "true" @@ -151,7 +151,7 @@ spec: python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip fi - PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.5' && "$0" "$@" + PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && "$0" "$@" - sh - -ec - | @@ -210,7 +210,7 @@ spec: fieldRef: {fieldPath: 'metadata.labels[''pipelines.kubeflow.org/enable_caching'']'} - {name: KFP_V2_IMAGE, value: 'python:3.7'} - {name: KFP_V2_RUNTIME_INFO, value: '{"inputParameters": {"num_steps": {"type": - "INT"}}, "inputArtifacts": {"dataset": {"metadataPath": "/tmp/inputs/dataset/data", + "NUMBER_INTEGER"}}, "inputArtifacts": {"dataset": {"metadataPath": "/tmp/inputs/dataset/data", "schemaTitle": "system.Dataset", "instanceSchema": "", "schemaVersion": "0.0.1"}}, "outputParameters": {}, "outputArtifacts": {"model": {"schemaTitle": "system.Model", "instanceSchema": "", "schemaVersion": "0.0.1", "metadataPath": @@ -236,7 +236,7 @@ spec: pipelines.kubeflow.org/component_ref: '{}' pipelines.kubeflow.org/arguments.parameters: '{"num_steps": "{{inputs.parameters.preprocess-output_parameter_one}}"}' labels: - pipelines.kubeflow.org/kfp_sdk_version: 1.8.5 + pipelines.kubeflow.org/kfp_sdk_version: 1.8.6 pipelines.kubeflow.org/pipeline-sdk-type: kfp pipelines.kubeflow.org/v2_component: "true" pipelines.kubeflow.org/enable_caching: "true" diff --git a/sdk/python/kfp/compiler/v2_compat.py b/sdk/python/kfp/compiler/v2_compat.py index d96d045efe0..f6d05fdcfa1 100644 --- a/sdk/python/kfp/compiler/v2_compat.py +++ b/sdk/python/kfp/compiler/v2_compat.py @@ -141,12 +141,13 @@ def update_op(op: dsl.ContainerOp, component_spec = op.component_spec for parameter, spec in sorted( component_spec.input_definitions.parameters.items()): - parameter_info = { - "type": - pipeline_spec_pb2.PrimitiveType.PrimitiveTypeEnum.Name(spec.type - ), - } - op.command += [f"{parameter}={op._parameter_arguments[parameter]}"] + parameter_type = pipeline_spec_pb2.ParameterType.ParameterTypeEnum.Name( + spec.parameter_type) + parameter_info = {"type": parameter_type} + + parameter_value = op._parameter_arguments[parameter] + op.command += [f"{parameter}={parameter_value}"] + runtime_info["inputParameters"][parameter] = parameter_info op.command += ["--"] @@ -164,8 +165,8 @@ def update_op(op: dsl.ContainerOp, component_spec.output_definitions.parameters.items()): parameter_info = { "type": - pipeline_spec_pb2.PrimitiveType.PrimitiveTypeEnum.Name(spec.type - ), + pipeline_spec_pb2.ParameterType.ParameterTypeEnum.Name( + spec.parameter_type), "path": op.file_outputs[parameter], } diff --git a/sdk/python/kfp/dsl/_component_bridge.py b/sdk/python/kfp/dsl/_component_bridge.py index a7277fe3518..c61a6cb0d1f 100644 --- a/sdk/python/kfp/dsl/_component_bridge.py +++ b/sdk/python/kfp/dsl/_component_bridge.py @@ -20,7 +20,7 @@ from typing import Any, Mapping, Optional import kfp -from kfp.components import _structures +from kfp.components import _structures, _data_passing from kfp.components import _components from kfp.components import _naming from kfp import dsl @@ -199,6 +199,12 @@ def _create_container_op_from_component_and_arguments( default_value = int(default_value) elif input_spec.type == 'Float': default_value = float(default_value) + elif (type_utils.is_parameter_type(input_spec.type) and + kfp.COMPILING_FOR_V2): + parameter_type = type_utils.get_parameter_type(input_spec.type) + default_value = type_utils.deserialize_parameter_value( + value=default_value, parameter_type=parameter_type) + arguments[input_spec.name] = default_value # Check types of the reference arguments and serialize PipelineParams @@ -264,13 +270,22 @@ def _create_container_op_from_component_and_arguments( name_to_spec_type = {} if component_meta.inputs: name_to_spec_type = { - input.name: input.type for input in component_meta.inputs + input.name: { + 'type': input.type, + 'default': input.default, + } for input in component_meta.inputs } + if kfp.COMPILING_FOR_V2: for name, spec_type in name_to_spec_type.items(): if (name in original_arguments and - type_utils.is_parameter_type(spec_type)): - task._parameter_arguments[name] = str(original_arguments[name]) + type_utils.is_parameter_type(spec_type['type'])): + if isinstance(original_arguments[name], (list, dict)): + task._parameter_arguments[name] = json.dumps( + original_arguments[name]) + else: + task._parameter_arguments[name] = str( + original_arguments[name]) for name in list(task.artifact_arguments.keys()): if name in task._parameter_arguments: @@ -573,24 +588,35 @@ def _resolve_ir_placeholders_v2( input_type = component_spec._inputs_dict[input_name].type if type_utils.is_parameter_type(input_type): pipeline_task_spec.inputs.parameters[ - input_name].runtime_value.constant_value.string_value = ( - argument_value) + input_name].runtime_value.constant.string_value = argument_value elif isinstance(argument_value, int): argument_type = 'Integer' pipeline_task_spec.inputs.parameters[ - input_name].runtime_value.constant_value.int_value = ( - argument_value) + input_name].runtime_value.constant.number_value = argument_value elif isinstance(argument_value, float): argument_type = 'Float' pipeline_task_spec.inputs.parameters[ - input_name].runtime_value.constant_value.double_value = ( + input_name].runtime_value.constant.number_value = argument_value + elif isinstance(argument_value, bool): + argument_type = 'Bool' + pipeline_task_spec.inputs.parameters[ + input_name].runtime_value.constant.bool_value = argument_value + elif isinstance(argument_value, list): + argument_type = 'List' + + # Convert any PipelineParams to strings. + argument_value = map( + lambda x: str(x) + if isinstance(x, dsl.PipelineParam) else x, argument_value) + + pipeline_task_spec.inputs.parameters[ + input_name].runtime_value.constant.list_value.extend( argument_value) - elif isinstance(argument_value, - (dict, list, bool)) and kfp.COMPILING_FOR_V2: - argument_type = type(argument_value).__name__ + elif isinstance(argument_value, dict): + argument_type = 'Dict' pipeline_task_spec.inputs.parameters[ - input_name].runtime_value.constant_value.string_value = ( - json.dumps(argument_value)) + input_name].runtime_value.constant.struct_value.update( + argument_value) elif isinstance(argument_value, _container_op.ContainerOp): raise TypeError( f'ContainerOp object {input_name} was passed to component as an ' @@ -625,7 +651,7 @@ def _resolve_ir_placeholders_v2( if argument_is_parameter_type else 'Artifact', input_name=input_name, input_type=input_type, - input_category='Paramter' + input_category='Parameter' if input_is_parameter_type else 'Artifact', )) diff --git a/sdk/python/kfp/dsl/component_spec.py b/sdk/python/kfp/dsl/component_spec.py index 756849e8574..57d70e0f93c 100644 --- a/sdk/python/kfp/dsl/component_spec.py +++ b/sdk/python/kfp/dsl/component_spec.py @@ -98,7 +98,7 @@ def build_component_spec_from_structure( continue if type_utils.is_parameter_type(input_spec.type): result.input_definitions.parameters[ - input_spec.name].type = type_utils.get_parameter_type( + input_spec.name].parameter_type = type_utils.get_parameter_type( input_spec.type) else: result.input_definitions.artifacts[ @@ -108,7 +108,8 @@ def build_component_spec_from_structure( for output_spec in component_spec.outputs or []: if type_utils.is_parameter_type(output_spec.type): result.output_definitions.parameters[ - output_spec.name].type = type_utils.get_parameter_type( + output_spec + .name].parameter_type = type_utils.get_parameter_type( output_spec.type) else: result.output_definitions.artifacts[ @@ -141,7 +142,7 @@ def build_component_inputs_spec( if type_utils.is_parameter_type(param.param_type): component_spec.input_definitions.parameters[ - input_name].type = type_utils.get_parameter_type( + input_name].parameter_type = type_utils.get_parameter_type( param.param_type) elif input_name not in getattr(component_spec.input_definitions, 'parameters', []): @@ -164,7 +165,7 @@ def build_component_outputs_spec( output_name = param.full_name if type_utils.is_parameter_type(param.param_type): component_spec.output_definitions.parameters[ - output_name].type = type_utils.get_parameter_type( + output_name].parameter_type = type_utils.get_parameter_type( param.param_type) elif output_name not in getattr(component_spec.output_definitions, 'parameters', []): diff --git a/sdk/python/kfp/dsl/component_spec_test.py b/sdk/python/kfp/dsl/component_spec_test.py index b0919b75ed1..73a49923eeb 100644 --- a/sdk/python/kfp/dsl/component_spec_test.py +++ b/sdk/python/kfp/dsl/component_spec_test.py @@ -72,10 +72,10 @@ def test_build_component_spec_from_structure(self): }, 'parameters': { 'input2': { - 'type': 'STRING' + 'parameterType': 'STRING' }, 'input3': { - 'type': 'INT' + 'parameterType': 'NUMBER_INTEGER' } } }, @@ -118,13 +118,13 @@ def test_build_component_spec_from_structure(self): }, 'parameters': { 'input2': { - 'type': 'INT' + 'parameterType': 'NUMBER_INTEGER' }, 'input3': { - 'type': 'STRING' + 'parameterType': 'STRING' }, 'input4': { - 'type': 'DOUBLE' + 'parameterType': 'NUMBER_DOUBLE' } } } @@ -144,13 +144,13 @@ def test_build_component_spec_from_structure(self): }, 'parameters': { 'pipelineparam--input2': { - 'type': 'INT' + 'parameterType': 'NUMBER_INTEGER' }, 'pipelineparam--input3': { - 'type': 'STRING' + 'parameterType': 'STRING' }, 'pipelineparam--input4': { - 'type': 'DOUBLE' + 'parameterType': 'NUMBER_DOUBLE' } } } @@ -194,13 +194,13 @@ def test_build_component_outputs_spec(self): }, 'parameters': { 'output2': { - 'type': 'INT' + 'parameterType': 'NUMBER_INTEGER' }, 'output3': { - 'type': 'STRING' + 'parameterType': 'STRING' }, 'output4': { - 'type': 'DOUBLE' + 'parameterType': 'NUMBER_DOUBLE' } } } @@ -343,7 +343,7 @@ def test_build_task_inputs_spec(self, is_parent_component_root, }, 'parameters': { 'param1': { - 'type': 'STRING' + 'parameterType': 'STRING' }, } }, @@ -419,10 +419,10 @@ def test_build_task_inputs_spec(self, is_parent_component_root, }, 'parameters': { 'pipelineparam--op-2-output2' : { - 'type': 'INT' + 'parameterType': 'NUMBER_INTEGER' }, 'pipelineparam--param1': { - 'type': 'STRING' + 'parameterType': 'STRING' }, } }, @@ -483,9 +483,9 @@ def test_pop_input_from_component_spec(self): component_spec.input_definitions.artifacts[ 'input1'].artifact_type.schema_title = 'system.Dataset' component_spec.input_definitions.parameters[ - 'input2'].type = pipeline_spec_pb2.PrimitiveType.STRING + 'input2'].parameter_type = pipeline_spec_pb2.ParameterType.STRING component_spec.input_definitions.parameters[ - 'input3'].type = pipeline_spec_pb2.PrimitiveType.DOUBLE + 'input3'].parameter_type = pipeline_spec_pb2.ParameterType.NUMBER_DOUBLE # pop an artifact, and there're other inputs left dsl_component_spec.pop_input_from_component_spec( @@ -494,10 +494,10 @@ def test_pop_input_from_component_spec(self): 'inputDefinitions': { 'parameters': { 'input2': { - 'type': 'STRING' + 'parameterType': 'STRING' }, 'input3': { - 'type': 'DOUBLE' + 'parameterType': 'NUMBER_DOUBLE' } } }, @@ -514,7 +514,7 @@ def test_pop_input_from_component_spec(self): 'inputDefinitions': { 'parameters': { 'input3': { - 'type': 'DOUBLE' + 'parameterType': 'NUMBER_DOUBLE' } } }, diff --git a/sdk/python/kfp/dsl/type_utils.py b/sdk/python/kfp/dsl/type_utils.py index 8d11e478b60..7c61f1d426e 100644 --- a/sdk/python/kfp/dsl/type_utils.py +++ b/sdk/python/kfp/dsl/type_utils.py @@ -26,5 +26,4 @@ is_parameter_type = type_utils.is_parameter_type get_artifact_type_schema = type_utils.get_artifact_type_schema get_parameter_type = type_utils.get_parameter_type -get_parameter_type_field_name = type_utils.get_parameter_type_field_name get_input_artifact_type_schema = type_utils.get_input_artifact_type_schema diff --git a/sdk/python/kfp/v2/compiler/compiler.py b/sdk/python/kfp/v2/compiler/compiler.py index 3c9400375ff..873b3d662a7 100644 --- a/sdk/python/kfp/v2/compiler/compiler.py +++ b/sdk/python/kfp/v2/compiler/compiler.py @@ -549,37 +549,116 @@ def _resolve_condition_operands( operand2: Union[str, dsl.PipelineParam]) -> Tuple[str, str]: """Resolves values and PipelineParams for condition operands.""" - # Pre-scan the operand to get the type of constant value if there's any. - # The value_type can be used to backfill missing PipelineParam.param_type. - value_type = None for value_or_reference in [operand1, operand2]: + if not isinstance(value_or_reference, + (dsl.PipelineParam, int, float, bool, str)): + raise ValueError('Conditional requires scalar constant values' + ' for comparison. Found "{}" of type {}' + ' in pipeline definition instead.'.format( + value_or_reference, + type(value_or_reference))) + + # Check specified type of PipelineParam is a scalar as well. if isinstance(value_or_reference, dsl.PipelineParam): - continue - if isinstance(value_or_reference, float): - value_type = 'Float' - elif isinstance(value_or_reference, int): - value_type = 'Integer' + parameter_type = type_utils.get_parameter_type( + value_or_reference.param_type) + + if parameter_type in [ + pipeline_spec_pb2.ParameterType.STRUCT, + pipeline_spec_pb2.ParameterType.LIST, + pipeline_spec_pb2.ParameterType + .PARAMETER_TYPE_ENUM_UNSPECIFIED, + ]: + input_name = dsl_component_spec.additional_input_name_for_pipelineparam( + value_or_reference) + raise ValueError( + 'Conditional requires scalar parameter values' + ' for comparison. Found input "{}" of type {}' + ' in pipeline definition instead.'.format( + input_name, value_or_reference.param_type)) + + parameter_types = set() + for value_or_reference in [operand1, operand2]: + if isinstance(value_or_reference, dsl.PipelineParam): + parameter_type = type_utils.get_parameter_type( + value_or_reference.param_type) else: - value_type = 'String' + parameter_type = type_utils.get_parameter_type( + type(value_or_reference).__name__) + + parameter_types.add(parameter_type) + + if len(parameter_types) == 2: + # Two different types being compared. The only possible types are + # String, Boolean, Double and Integer. We'll promote the other type + # using the following precedence: + # String > Boolean > Double > Integer + if pipeline_spec_pb2.ParameterType.STRING in parameter_types: + canonical_parameter_type = pipeline_spec_pb2.ParameterType.STRING + elif pipeline_spec_pb2.ParameterType.BOOLEAN in parameter_types: + canonical_parameter_type = pipeline_spec_pb2.ParameterType.BOOLEAN + else: + # Must be a double and int, promote to double. + assert pipeline_spec_pb2.ParameterType.NUMBER_DOUBLE in parameter_types, 'Types: {} [{} {}]'.format( + parameter_types, operand1, operand2) + assert pipeline_spec_pb2.ParameterType.NUMBER_INTEGER in parameter_types, 'Types: {} [{} {}]'.format( + parameter_types, operand1, operand2) + canonical_parameter_type = pipeline_spec_pb2.ParameterType.NUMBER_DOUBLE + elif len(parameter_types) == 1: # Both operands are the same type. + canonical_parameter_type = parameter_types.pop() + else: + # Probably shouldn't happen. + raise ValueError('Unable to determine operand types for' + ' "{}" and "{}"'.format(operand1, operand2)) operand_values = [] for value_or_reference in [operand1, operand2]: if isinstance(value_or_reference, dsl.PipelineParam): input_name = dsl_component_spec.additional_input_name_for_pipelineparam( value_or_reference) - # Condition operand is always parameters for now. - value_or_reference.param_type = ( - value_or_reference.param_type or value_type) - operand_values.append( - "inputs.parameters['{input_name}'].{value_field}".format( - input_name=input_name, - value_field=type_utils.get_parameter_type_field_name( - value_or_reference.param_type))) + operand_value = "inputs.parameter_values['{input_name}']".format( + input_name=input_name) + parameter_type = type_utils.get_parameter_type( + value_or_reference.param_type) + if parameter_type == pipeline_spec_pb2.ParameterType.NUMBER_INTEGER: + operand_value = 'int({})'.format(operand_value) + elif isinstance(value_or_reference, str): + operand_value = "'{}'".format(value_or_reference) + parameter_type = pipeline_spec_pb2.ParameterType.STRING + elif isinstance(value_or_reference, bool): + # Booleans need to be compared as 'true' or 'false' in CEL. + operand_value = str(value_or_reference).lower() + parameter_type = pipeline_spec_pb2.ParameterType.BOOLEAN + elif isinstance(value_or_reference, int): + operand_value = str(value_or_reference) + parameter_type = pipeline_spec_pb2.ParameterType.NUMBER_INTEGER else: - if isinstance(value_or_reference, str): - operand_values.append("'{}'".format(value_or_reference)) + assert isinstance(value_or_reference, float), value_or_reference + operand_value = str(value_or_reference) + parameter_type = pipeline_spec_pb2.ParameterType.NUMBER_DOUBLE + + if parameter_type != canonical_parameter_type: + # Type-cast to so CEL does not complain. + if canonical_parameter_type == pipeline_spec_pb2.ParameterType.STRING: + assert parameter_type in [ + pipeline_spec_pb2.ParameterType.BOOLEAN, + pipeline_spec_pb2.ParameterType.NUMBER_INTEGER, + pipeline_spec_pb2.ParameterType.NUMBER_DOUBLE, + ] + operand_value = "'{}'".format(operand_value) + elif canonical_parameter_type == pipeline_spec_pb2.ParameterType.BOOLEAN: + assert parameter_type in [ + pipeline_spec_pb2.ParameterType.NUMBER_INTEGER, + pipeline_spec_pb2.ParameterType.NUMBER_DOUBLE, + ] + operand_value = 'true' if int( + operand_value) == 0 else 'false' else: - operand_values.append(str(value_or_reference)) + assert canonical_parameter_type == pipeline_spec_pb2.ParameterType.NUMBER_DOUBLE + assert parameter_type == pipeline_spec_pb2.ParameterType.NUMBER_INTEGER + operand_value = 'double({})'.format(operand_value) + + operand_values.append(operand_value) return tuple(operand_values) @@ -822,7 +901,7 @@ def _group_to_dag_spec( _for_loop.LoopArguments.LOOP_ITEM_NAME_BASE) subgroup_component_spec.input_definitions.parameters[ - loop_arguments_item].type = pipeline_spec_pb2.PrimitiveType.STRING + loop_arguments_item].parameter_type = pipeline_spec_pb2.ParameterType.STRING subgroup_task_spec.parameter_iterator.items.input_parameter = ( input_parameter_name) subgroup_task_spec.parameter_iterator.item_input = ( @@ -986,8 +1065,8 @@ def _create_pipeline_spec( pipeline_spec.pipeline_info.name = pipeline.name pipeline_spec.sdk_version = 'kfp-{}'.format(kfp.__version__) - # Schema version 2.0.0 is required for kfp-pipeline-spec>0.1.3.1 - pipeline_spec.schema_version = '2.0.0' + # Schema version 2.1.0 is required for kfp-pipeline-spec>0.1.3.1 + pipeline_spec.schema_version = '2.1.0' dsl_component_spec.build_component_inputs_spec( component_spec=pipeline_spec.root, @@ -1204,13 +1283,21 @@ def _create_pipeline_v2( # Fill in the default values. args_list_with_defaults = [] if pipeline_meta.inputs: - args_list_with_defaults = [ - dsl.PipelineParam( - sanitize_k8s_name(input_spec.name, True), - param_type=input_spec.type, - value=input_spec.default) - for input_spec in pipeline_meta.inputs - ] + args_list_with_defaults = [] + for input_spec in pipeline_meta.inputs: + default_value = input_spec.default + + if input_spec.default is not None: + parameter_type = type_utils.get_parameter_type( + input_spec.type) + default_value = type_utils.deserialize_parameter_value( + value=input_spec.default, parameter_type=parameter_type) + + args_list_with_defaults.append( + dsl.PipelineParam( + sanitize_k8s_name(input_spec.name, True), + param_type=input_spec.type, + value=default_value)) # Making the pipeline group name unique to prevent name clashes with templates pipeline_group = dsl_pipeline.groups[0] diff --git a/sdk/python/kfp/v2/compiler/compiler_utils.py b/sdk/python/kfp/v2/compiler/compiler_utils.py index ac17ec4235e..73919f51635 100644 --- a/sdk/python/kfp/v2/compiler/compiler_utils.py +++ b/sdk/python/kfp/v2/compiler/compiler_utils.py @@ -16,6 +16,8 @@ import re from typing import Any, Mapping, Optional, Union +from google.protobuf import struct_pb2 + from kfp.containers import _component_builder from kfp.dsl import _container_op from kfp.dsl import _pipeline_param @@ -31,7 +33,7 @@ def build_runtime_config_spec( pipeline_parameters: Optional[Mapping[ str, _pipeline_param.PipelineParam]] = None, ) -> pipeline_spec_pb2.PipelineJob.RuntimeConfig: - """Converts pipeine parameters to runtime parameters mapping. + """Converts pipeline parameters to runtime parameters mapping. Args: output_directory: The root of pipeline outputs. @@ -42,32 +44,33 @@ def build_runtime_config_spec( A pipeline job RuntimeConfig object. """ - def _get_value( - param: _pipeline_param.PipelineParam) -> pipeline_spec_pb2.Value: - assert param.value is not None, 'None values should be filterd out.' + def _get_value(param: _pipeline_param.PipelineParam) -> struct_pb2.Value: + assert param.value is not None, 'None values should be filtered out.' - result = pipeline_spec_pb2.Value() + result = struct_pb2.Value() # TODO(chensun): remove defaulting to 'String' for None param_type once we # fix importer behavior. param_type = type_utils.get_parameter_type(param.param_type or 'String') - if param_type == pipeline_spec_pb2.PrimitiveType.INT: - result.int_value = int(param.value) - elif param_type == pipeline_spec_pb2.PrimitiveType.DOUBLE: - result.double_value = float(param.value) - elif param_type == pipeline_spec_pb2.PrimitiveType.STRING: - result.string_value = str(param.value) + if (param_type == pipeline_spec_pb2.ParameterType.NUMBER_INTEGER or + param_type == pipeline_spec_pb2.ParameterType.NUMBER_DOUBLE): + result.number_value = float(param.value) + elif param_type == pipeline_spec_pb2.ParameterType.STRING: + result.string_value = param.value + elif param_type == pipeline_spec_pb2.ParameterType.BOOLEAN: + result.bool_value = param.value + elif param_type == pipeline_spec_pb2.ParameterType.LIST: + result.list_value.extend(param.value) + elif param_type == pipeline_spec_pb2.ParameterType.STRUCT: + result.struct_value.update(param.value) else: - # For every other type, defaults to 'String'. - # TODO(chensun): remove this default behavior once we migrate from - # `pipeline_spec_pb2.Value` to `protobuf.Value`. - result.string_value = str(param.value) + raise ValueError('Unknown type for PipelineParam {}'.format(param)) return result parameters = pipeline_parameters or {} return pipeline_spec_pb2.PipelineJob.RuntimeConfig( gcs_output_directory=output_directory, - parameters={ + parameter_values={ k: _get_value(v) for k, v in parameters.items() if v.value is not None diff --git a/sdk/python/kfp/v2/compiler/compiler_utils_test.py b/sdk/python/kfp/v2/compiler/compiler_utils_test.py index 6fd52ab58db..bc0a6965290 100644 --- a/sdk/python/kfp/v2/compiler/compiler_utils_test.py +++ b/sdk/python/kfp/v2/compiler/compiler_utils_test.py @@ -34,16 +34,10 @@ def assertProtoEquals(self, proto1: message.Message, def test_build_runtime_config_spec(self): expected_dict = { 'gcsOutputDirectory': 'gs://path', - 'parameters': { - 'input1': { - 'stringValue': 'test' - }, - 'input2': { - 'intValue': 2 - }, - 'input3': { - 'stringValue': '[1, 2, 3]' - } + 'parameterValues': { + 'input1': 'test', + 'input2': 2, + 'input3': [1, 2, 3] } } expected_spec = pipeline_spec_pb2.PipelineJob.RuntimeConfig() diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/lightweight_python_functions_v2_pipeline.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/lightweight_python_functions_v2_pipeline.json index 49aa2ab3b0a..52bc28b1b32 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/lightweight_python_functions_v2_pipeline.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/lightweight_python_functions_v2_pipeline.json @@ -6,13 +6,13 @@ "inputDefinitions": { "parameters": { "input_dict_parameter": { - "type": "STRING" + "parameterType": "STRUCT" }, "input_list_parameter": { - "type": "STRING" + "parameterType": "LIST" }, "message": { - "type": "STRING" + "parameterType": "STRING" } } }, @@ -33,16 +33,16 @@ }, "parameters": { "output_bool_parameter_path": { - "type": "STRING" + "parameterType": "BOOLEAN" }, "output_dict_parameter_path": { - "type": "STRING" + "parameterType": "STRUCT" }, "output_list_parameter_path": { - "type": "STRING" + "parameterType": "LIST" }, "output_parameter_path": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -66,19 +66,19 @@ }, "parameters": { "input_bool": { - "type": "STRING" + "parameterType": "BOOLEAN" }, "input_dict": { - "type": "STRING" + "parameterType": "STRUCT" }, "input_list": { - "type": "STRING" + "parameterType": "LIST" }, "message": { - "type": "STRING" + "parameterType": "STRING" }, "num_steps": { - "type": "INT" + "parameterType": "NUMBER_INTEGER" } } }, @@ -107,7 +107,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.5' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -127,7 +127,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.5' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -158,9 +158,11 @@ }, "input_list_parameter": { "runtimeValue": { - "constantValue": { - "stringValue": "[\"a\", \"b\", \"c\"]" - } + "constant": [ + "a", + "b", + "c" + ] } }, "message": { @@ -224,9 +226,7 @@ }, "num_steps": { "runtimeValue": { - "constantValue": { - "intValue": "100" - } + "constant": 100.0 } } } @@ -240,22 +240,23 @@ "inputDefinitions": { "parameters": { "input_dict": { - "type": "STRING" + "parameterType": "STRUCT" }, "message": { - "type": "STRING" + "parameterType": "STRING" } } } }, - "schemaVersion": "2.0.0", - "sdkVersion": "kfp-1.8.5" + "schemaVersion": "2.1.0", + "sdkVersion": "kfp-1.8.6" }, "runtimeConfig": { "gcsOutputDirectory": "dummy_root", - "parameters": { + "parameterValues": { "input_dict": { - "stringValue": "{\"A\": 1, \"B\": 2}" + "A": 1.0, + "B": 2.0 } } } diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/lightweight_python_functions_v2_with_outputs.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/lightweight_python_functions_v2_with_outputs.json index 875c6edbf41..9761d894ff1 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/lightweight_python_functions_v2_with_outputs.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/lightweight_python_functions_v2_with_outputs.json @@ -6,17 +6,17 @@ "inputDefinitions": { "parameters": { "first": { - "type": "INT" + "parameterType": "NUMBER_INTEGER" }, "second": { - "type": "INT" + "parameterType": "NUMBER_INTEGER" } } }, "outputDefinitions": { "parameters": { "Output": { - "type": "INT" + "parameterType": "NUMBER_INTEGER" } } } @@ -26,17 +26,17 @@ "inputDefinitions": { "parameters": { "first": { - "type": "STRING" + "parameterType": "STRING" }, "second": { - "type": "STRING" + "parameterType": "STRING" } } }, "outputDefinitions": { "parameters": { "Output": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -46,10 +46,10 @@ "inputDefinitions": { "parameters": { "message": { - "type": "STRING" + "parameterType": "STRING" }, "number": { - "type": "INT" + "parameterType": "NUMBER_INTEGER" } } }, @@ -93,7 +93,7 @@ }, "parameters": { "scalar": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -112,7 +112,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.5' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -132,7 +132,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.5' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -152,7 +152,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.5' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -172,7 +172,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.5' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -303,16 +303,16 @@ "inputDefinitions": { "parameters": { "first_message": { - "type": "STRING" + "parameterType": "STRING" }, "first_number": { - "type": "INT" + "parameterType": "NUMBER_INTEGER" }, "second_message": { - "type": "STRING" + "parameterType": "STRING" }, "second_number": { - "type": "INT" + "parameterType": "NUMBER_INTEGER" } } }, @@ -327,8 +327,8 @@ } } }, - "schemaVersion": "2.0.0", - "sdkVersion": "kfp-1.8.5" + "schemaVersion": "2.1.0", + "sdkVersion": "kfp-1.8.6" }, "runtimeConfig": { "gcsOutputDirectory": "dummy_root" diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_after.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_after.json index cf4076ad05a..5b4ba87fda6 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_after.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_after.json @@ -6,7 +6,7 @@ "inputDefinitions": { "parameters": { "text": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -16,7 +16,7 @@ "inputDefinitions": { "parameters": { "text": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -26,7 +26,7 @@ "inputDefinitions": { "parameters": { "text": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -86,9 +86,7 @@ "parameters": { "text": { "runtimeValue": { - "constantValue": { - "stringValue": "1st task" - } + "constant": "1st task" } } } @@ -111,9 +109,7 @@ "parameters": { "text": { "runtimeValue": { - "constantValue": { - "stringValue": "2nd task" - } + "constant": "2nd task" } } } @@ -137,9 +133,7 @@ "parameters": { "text": { "runtimeValue": { - "constantValue": { - "stringValue": "3rd task" - } + "constant": "3rd task" } } } @@ -151,8 +145,8 @@ } } }, - "schemaVersion": "2.0.0", - "sdkVersion": "kfp-1.7.2" + "schemaVersion": "2.1.0", + "sdkVersion": "kfp-1.8.6" }, "runtimeConfig": { "gcsOutputDirectory": "dummy_root" diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_concat_placeholder.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_concat_placeholder.json index 675d57e486d..4eb955c2f4b 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_concat_placeholder.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_concat_placeholder.json @@ -6,7 +6,7 @@ "inputDefinitions": { "parameters": { "input_prefix": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -42,9 +42,7 @@ "parameters": { "input_prefix": { "runtimeValue": { - "constantValue": { - "stringValue": "some prefix:" - } + "constant": "some prefix:" } } } @@ -56,8 +54,8 @@ } } }, - "schemaVersion": "2.0.0", - "sdkVersion": "kfp-1.7.2" + "schemaVersion": "2.1.0", + "sdkVersion": "kfp-1.8.6" }, "runtimeConfig": { "gcsOutputDirectory": "dummy_root" diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_condition.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_condition.json index b960c550daa..bee99cfa87c 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_condition.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_condition.json @@ -60,10 +60,10 @@ "inputDefinitions": { "parameters": { "pipelineparam--flip-coin-Output": { - "type": "STRING" + "parameterType": "STRING" }, "pipelineparam--text": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -73,7 +73,7 @@ "outputDefinitions": { "parameters": { "Output": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -83,7 +83,7 @@ "outputDefinitions": { "parameters": { "Output": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -93,7 +93,7 @@ "inputDefinitions": { "parameters": { "msg": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -103,7 +103,7 @@ "inputDefinitions": { "parameters": { "msg": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -113,7 +113,7 @@ "inputDefinitions": { "parameters": { "msg": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -228,7 +228,7 @@ "name": "condition-1" }, "triggerPolicy": { - "condition": "inputs.parameters['pipelineparam--flip-coin-Output'].string_value == 'heads'" + "condition": "inputs.parameter_values['pipelineparam--flip-coin-Output'] == 'heads'" } }, "flip-coin": { @@ -269,20 +269,18 @@ "inputDefinitions": { "parameters": { "text": { - "type": "STRING" + "parameterType": "STRING" } } } }, - "schemaVersion": "2.0.0", - "sdkVersion": "kfp-1.7.2" + "schemaVersion": "2.1.0", + "sdkVersion": "kfp-1.8.6" }, "runtimeConfig": { "gcsOutputDirectory": "dummy_root", - "parameters": { - "text": { - "stringValue": "condition test" - } + "parameterValues": { + "text": "condition test" } } } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_custom_job_spec.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_custom_job_spec.json index 1e4b89576f6..38e72ae8754 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_custom_job_spec.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_custom_job_spec.json @@ -6,7 +6,7 @@ "inputDefinitions": { "parameters": { "input1": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -16,7 +16,7 @@ "inputDefinitions": { "parameters": { "input1": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -41,7 +41,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.5' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -65,7 +65,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.5' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -128,9 +128,7 @@ "parameters": { "input1": { "runtimeValue": { - "constantValue": { - "stringValue": "hello-world" - } + "constant": "hello-world" } } } @@ -150,9 +148,7 @@ "parameters": { "input1": { "runtimeValue": { - "constantValue": { - "stringValue": "advanced setting - raw workerPoolSpec" - } + "constant": "advanced setting - raw workerPoolSpec" } } } @@ -164,8 +160,8 @@ } } }, - "schemaVersion": "2.0.0", - "sdkVersion": "kfp-1.8.5" + "schemaVersion": "2.1.0", + "sdkVersion": "kfp-1.8.6" }, "runtimeConfig": { "gcsOutputDirectory": "dummy_root" diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_env.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_env.json index f2c3c3ddeb4..78ec22e9660 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_env.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_env.json @@ -41,7 +41,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.5' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -89,8 +89,8 @@ } } }, - "schemaVersion": "2.0.0", - "sdkVersion": "kfp-1.8.5" + "schemaVersion": "2.1.0", + "sdkVersion": "kfp-1.8.6" }, "runtimeConfig": { "gcsOutputDirectory": "dummy_root" diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_exit_handler.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_exit_handler.json index 7fb94bdfa35..12850386836 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_exit_handler.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_exit_handler.json @@ -15,9 +15,7 @@ "parameters": { "message": { "runtimeValue": { - "constantValue": { - "stringValue": "Task failed." - } + "constant": "Task failed." } } } @@ -49,7 +47,7 @@ "inputDefinitions": { "parameters": { "pipelineparam--message": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -59,7 +57,7 @@ "inputDefinitions": { "parameters": { "message": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -69,7 +67,7 @@ "inputDefinitions": { "parameters": { "message": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -79,7 +77,7 @@ "inputDefinitions": { "parameters": { "message": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -98,7 +96,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.5' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -118,7 +116,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.5' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -138,7 +136,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.5' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -181,9 +179,7 @@ "parameters": { "message": { "runtimeValue": { - "constantValue": { - "stringValue": "Exit handler has worked!" - } + "constant": "Exit handler has worked!" } } } @@ -200,20 +196,18 @@ "inputDefinitions": { "parameters": { "message": { - "type": "STRING" + "parameterType": "STRING" } } } }, - "schemaVersion": "2.0.0", - "sdkVersion": "kfp-1.8.5" + "schemaVersion": "2.1.0", + "sdkVersion": "kfp-1.8.6" }, "runtimeConfig": { "gcsOutputDirectory": "dummy_root", - "parameters": { - "message": { - "stringValue": "Hello World!" - } + "parameterValues": { + "message": "Hello World!" } } } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_gcpc_types.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_gcpc_types.json index 61d2f0113bc..efc5297d903 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_gcpc_types.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_gcpc_types.json @@ -41,7 +41,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.5' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -107,8 +107,8 @@ } } }, - "schemaVersion": "2.0.0", - "sdkVersion": "kfp-1.8.5" + "schemaVersion": "2.1.0", + "sdkVersion": "kfp-1.8.6" }, "runtimeConfig": {} } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_if_placeholder.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_if_placeholder.json index d354707f554..a3e0df000bb 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_if_placeholder.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_if_placeholder.json @@ -6,10 +6,10 @@ "inputDefinitions": { "parameters": { "optional_input_1": { - "type": "STRING" + "parameterType": "STRING" }, "required_input": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -64,19 +64,19 @@ "inputDefinitions": { "parameters": { "input0": { - "type": "STRING" + "parameterType": "STRING" }, "input1": { - "type": "STRING" + "parameterType": "STRING" }, "input2": { - "type": "STRING" + "parameterType": "STRING" } } } }, - "schemaVersion": "2.0.0", - "sdkVersion": "kfp-1.7.2" + "schemaVersion": "2.1.0", + "sdkVersion": "kfp-1.8.6" }, "runtimeConfig": { "gcsOutputDirectory": "dummy_root" diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_importer.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_importer.json index 94001ca7ff6..07622538962 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_importer.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_importer.json @@ -51,10 +51,10 @@ "inputDefinitions": { "parameters": { "pipelineparam--dataset2": { - "type": "STRING" + "parameterType": "STRING" }, "pipelineparam--train-scalar": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -124,14 +124,14 @@ "inputDefinitions": { "parameters": { "value": { - "type": "STRING" + "parameterType": "STRING" } } }, "outputDefinitions": { "parameters": { "Output": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -159,7 +159,7 @@ }, "parameters": { "scalar": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -187,7 +187,7 @@ }, "parameters": { "scalar": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -215,7 +215,7 @@ }, "parameters": { "scalar": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -286,7 +286,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.5' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -306,7 +306,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.5' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -326,7 +326,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.5' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -367,7 +367,7 @@ "name": "condition-1" }, "triggerPolicy": { - "condition": "inputs.parameters['pipelineparam--train-scalar'].string_value == '123'" + "condition": "inputs.parameter_values['pipelineparam--train-scalar'] == '123'" } }, "importer": { @@ -487,20 +487,18 @@ "inputDefinitions": { "parameters": { "dataset2": { - "type": "STRING" + "parameterType": "STRING" } } } }, - "schemaVersion": "2.0.0", - "sdkVersion": "kfp-1.8.5" + "schemaVersion": "2.1.0", + "sdkVersion": "kfp-1.8.6" }, "runtimeConfig": { "gcsOutputDirectory": "dummy_root", - "parameters": { - "dataset2": { - "stringValue": "gs://ml-pipeline-playground/shakespeare2.txt" - } + "parameterValues": { + "dataset2": "gs://ml-pipeline-playground/shakespeare2.txt" } } } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_loops.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_loops.json index e8b2082bcd5..4b1804eb688 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_loops.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_loops.json @@ -6,7 +6,7 @@ "outputDefinitions": { "parameters": { "Output": { - "type": "STRING" + "parameterType": "LIST" } } } @@ -37,10 +37,10 @@ "inputDefinitions": { "parameters": { "pipelineparam--loop_parameter": { - "type": "STRING" + "parameterType": "LIST" }, "pipelineparam--loop_parameter-loop-item": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -109,10 +109,10 @@ "inputDefinitions": { "parameters": { "pipelineparam--args-generator-op-Output": { - "type": "STRING" + "parameterType": "LIST" }, "pipelineparam--args-generator-op-Output-loop-item": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -181,7 +181,7 @@ "inputDefinitions": { "parameters": { "pipelineparam--loop-item-param-3": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -191,7 +191,7 @@ "inputDefinitions": { "parameters": { "msg": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -201,7 +201,7 @@ "inputDefinitions": { "parameters": { "msg": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -211,7 +211,7 @@ "inputDefinitions": { "parameters": { "msg": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -221,7 +221,7 @@ "inputDefinitions": { "parameters": { "msg": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -231,7 +231,7 @@ "inputDefinitions": { "parameters": { "msg": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -241,7 +241,7 @@ "inputDefinitions": { "parameters": { "msg": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -251,7 +251,7 @@ "inputDefinitions": { "parameters": { "msg": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -270,7 +270,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.5' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -290,7 +290,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.5' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -310,7 +310,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.5' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -330,7 +330,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.5' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -350,7 +350,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.5' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -370,7 +370,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.5' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -390,7 +390,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.5' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -410,7 +410,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.5' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -505,13 +505,13 @@ "inputDefinitions": { "parameters": { "loop_parameter": { - "type": "STRING" + "parameterType": "LIST" } } } }, - "schemaVersion": "2.0.0", - "sdkVersion": "kfp-1.8.5" + "schemaVersion": "2.1.0", + "sdkVersion": "kfp-1.8.6" }, "runtimeConfig": {} } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_loops_and_conditions.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_loops_and_conditions.json index b6d548523c6..cb23a9445cc 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_loops_and_conditions.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_loops_and_conditions.json @@ -6,7 +6,7 @@ "outputDefinitions": { "parameters": { "Output": { - "type": "STRING" + "parameterType": "LIST" } } } @@ -16,7 +16,7 @@ "outputDefinitions": { "parameters": { "Output": { - "type": "STRING" + "parameterType": "LIST" } } } @@ -79,16 +79,16 @@ "inputDefinitions": { "parameters": { "pipelineparam--args-generator-op-Output": { - "type": "STRING" + "parameterType": "LIST" }, "pipelineparam--flip-coin-op-Output": { - "type": "STRING" + "parameterType": "STRING" }, "pipelineparam--loop_parameter": { - "type": "STRING" + "parameterType": "LIST" }, "pipelineparam--msg": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -107,9 +107,7 @@ "parameters": { "msg": { "runtimeValue": { - "constantValue": { - "stringValue": "1" - } + "constant": "1" } } } @@ -123,10 +121,10 @@ "inputDefinitions": { "parameters": { "pipelineparam--flip-coin-op-Output": { - "type": "STRING" + "parameterType": "STRING" }, "pipelineparam--loop-item-param-11": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -164,10 +162,10 @@ "inputDefinitions": { "parameters": { "pipelineparam--loop_parameter-loop-item": { - "type": "STRING" + "parameterType": "STRING" }, "pipelineparam--loop_parameter-loop-item-subvar-A_a": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -199,13 +197,13 @@ "inputDefinitions": { "parameters": { "pipelineparam--args-generator-op-Output-loop-item": { - "type": "STRING" + "parameterType": "STRING" }, "pipelineparam--args-generator-op-Output-loop-item-subvar-A_a": { - "type": "STRING" + "parameterType": "STRING" }, "pipelineparam--flip-coin-op-Output": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -237,10 +235,10 @@ "inputDefinitions": { "parameters": { "pipelineparam--args-generator-op-Output-loop-item": { - "type": "STRING" + "parameterType": "STRING" }, "pipelineparam--flip-coin-op-Output": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -277,13 +275,13 @@ "inputDefinitions": { "parameters": { "pipelineparam--args-generator-op-Output-loop-item": { - "type": "STRING" + "parameterType": "STRING" }, "pipelineparam--args-generator-op-Output-loop-item-subvar-A_a": { - "type": "STRING" + "parameterType": "STRING" }, "pipelineparam--flip-coin-op-Output": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -293,7 +291,7 @@ "outputDefinitions": { "parameters": { "Output": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -328,16 +326,16 @@ "inputDefinitions": { "parameters": { "pipelineparam--args-generator-op-2-Output": { - "type": "STRING" + "parameterType": "LIST" }, "pipelineparam--args-generator-op-2-Output-loop-item": { - "type": "STRING" + "parameterType": "STRING" }, "pipelineparam--flip-coin-op-Output": { - "type": "STRING" + "parameterType": "STRING" }, "pipelineparam--loop_parameter-loop-item": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -363,7 +361,7 @@ "name": "condition-13" }, "triggerPolicy": { - "condition": "inputs.parameters['pipelineparam--loop-item-param-11'].string_value == '1'" + "condition": "inputs.parameter_values['pipelineparam--loop-item-param-11'] == '1'" } }, "print-op-8": { @@ -389,10 +387,10 @@ "inputDefinitions": { "parameters": { "pipelineparam--flip-coin-op-Output": { - "type": "STRING" + "parameterType": "STRING" }, "pipelineparam--loop-item-param-11": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -419,7 +417,7 @@ "name": "condition-15" }, "triggerPolicy": { - "condition": "inputs.parameters['pipelineparam--loop_parameter-loop-item-subvar-A_a'].string_value == 'heads'" + "condition": "inputs.parameter_values['pipelineparam--loop_parameter-loop-item-subvar-A_a'] == 'heads'" } } } @@ -427,10 +425,10 @@ "inputDefinitions": { "parameters": { "pipelineparam--loop_parameter": { - "type": "STRING" + "parameterType": "LIST" }, "pipelineparam--loop_parameter-loop-item": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -461,13 +459,13 @@ "inputDefinitions": { "parameters": { "pipelineparam--loop_parameter-loop-item": { - "type": "STRING" + "parameterType": "STRING" }, "pipelineparam--loop_parameter-loop-item-subvar-B_b": { - "type": "STRING" + "parameterType": "STRING" }, "pipelineparam--loop_parameter-loop-item-subvar-B_b-loop-item": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -497,7 +495,7 @@ "name": "condition-3" }, "triggerPolicy": { - "condition": "inputs.parameters['pipelineparam--args-generator-op-Output-loop-item-subvar-A_a'].string_value == 'heads'" + "condition": "inputs.parameter_values['pipelineparam--args-generator-op-Output-loop-item-subvar-A_a'] == 'heads'" } }, "condition-4": { @@ -518,7 +516,7 @@ "name": "condition-4" }, "triggerPolicy": { - "condition": "inputs.parameters['pipelineparam--flip-coin-op-Output'].string_value == 'heads'" + "condition": "inputs.parameter_values['pipelineparam--flip-coin-op-Output'] == 'heads'" } }, "condition-5": { @@ -543,7 +541,7 @@ "name": "condition-5" }, "triggerPolicy": { - "condition": "inputs.parameters['pipelineparam--args-generator-op-Output-loop-item-subvar-A_a'].string_value == 'tails'" + "condition": "inputs.parameter_values['pipelineparam--args-generator-op-Output-loop-item-subvar-A_a'] == 'tails'" } }, "for-loop-12": { @@ -645,22 +643,22 @@ "inputDefinitions": { "parameters": { "pipelineparam--args-generator-op-2-Output": { - "type": "STRING" + "parameterType": "LIST" }, "pipelineparam--args-generator-op-Output": { - "type": "STRING" + "parameterType": "LIST" }, "pipelineparam--args-generator-op-Output-loop-item": { - "type": "STRING" + "parameterType": "STRING" }, "pipelineparam--flip-coin-op-Output": { - "type": "STRING" + "parameterType": "STRING" }, "pipelineparam--loop_parameter": { - "type": "STRING" + "parameterType": "LIST" }, "pipelineparam--msg": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -691,13 +689,13 @@ "inputDefinitions": { "parameters": { "pipelineparam--args-generator-op-Output-loop-item": { - "type": "STRING" + "parameterType": "STRING" }, "pipelineparam--flip-coin-op-Output": { - "type": "STRING" + "parameterType": "STRING" }, "pipelineparam--loop-item-param-6": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -728,16 +726,16 @@ "inputDefinitions": { "parameters": { "pipelineparam--args-generator-op-Output-loop-item": { - "type": "STRING" + "parameterType": "STRING" }, "pipelineparam--args-generator-op-Output-loop-item-subvar-B_b": { - "type": "STRING" + "parameterType": "STRING" }, "pipelineparam--args-generator-op-Output-loop-item-subvar-B_b-loop-item": { - "type": "STRING" + "parameterType": "STRING" }, "pipelineparam--flip-coin-op-Output": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -795,16 +793,16 @@ "inputDefinitions": { "parameters": { "pipelineparam--args-generator-op-2-Output": { - "type": "STRING" + "parameterType": "LIST" }, "pipelineparam--flip-coin-op-Output": { - "type": "STRING" + "parameterType": "STRING" }, "pipelineparam--loop_parameter": { - "type": "STRING" + "parameterType": "LIST" }, "pipelineparam--loop_parameter-loop-item": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -814,7 +812,7 @@ "inputDefinitions": { "parameters": { "msg": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -824,7 +822,7 @@ "inputDefinitions": { "parameters": { "msg": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -834,7 +832,7 @@ "inputDefinitions": { "parameters": { "msg": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -844,7 +842,7 @@ "inputDefinitions": { "parameters": { "msg": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -854,7 +852,7 @@ "inputDefinitions": { "parameters": { "msg": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -864,7 +862,7 @@ "inputDefinitions": { "parameters": { "msg": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -874,7 +872,7 @@ "inputDefinitions": { "parameters": { "msg": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -884,10 +882,10 @@ "inputDefinitions": { "parameters": { "msg": { - "type": "STRING" + "parameterType": "STRING" }, "msg2": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -897,7 +895,7 @@ "inputDefinitions": { "parameters": { "msg": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -907,7 +905,7 @@ "inputDefinitions": { "parameters": { "msg": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -926,7 +924,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.5' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -946,7 +944,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.5' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -966,7 +964,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.5' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -986,7 +984,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.5' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -1006,7 +1004,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.5' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -1026,7 +1024,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.5' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -1046,7 +1044,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.5' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -1066,7 +1064,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.5' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -1086,7 +1084,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.5' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -1106,7 +1104,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.5' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -1126,7 +1124,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.5' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -1146,7 +1144,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.5' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -1166,7 +1164,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.5' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -1228,7 +1226,7 @@ "name": "condition-1" }, "triggerPolicy": { - "condition": "inputs.parameters['pipelineparam--flip-coin-op-Output'].string_value != 'no-such-result'" + "condition": "inputs.parameter_values['pipelineparam--flip-coin-op-Output'] != 'no-such-result'" } }, "flip-coin-op": { @@ -1268,25 +1266,37 @@ "inputDefinitions": { "parameters": { "loop_parameter": { - "type": "STRING" + "parameterType": "LIST" }, "msg": { - "type": "STRING" + "parameterType": "STRING" } } } }, - "schemaVersion": "2.0.0", - "sdkVersion": "kfp-1.8.5" + "schemaVersion": "2.1.0", + "sdkVersion": "kfp-1.8.6" }, "runtimeConfig": { - "parameters": { - "loop_parameter": { - "stringValue": "[{\"A_a\": \"heads\", \"B_b\": [\"A\", \"B\"]}, {\"A_a\": \"tails\", \"B_b\": [\"X\", \"Y\", \"Z\"]}]" - }, - "msg": { - "stringValue": "hello" - } + "parameterValues": { + "loop_parameter": [ + { + "A_a": "heads", + "B_b": [ + "A", + "B" + ] + }, + { + "A_a": "tails", + "B_b": [ + "X", + "Y", + "Z" + ] + } + ], + "msg": "hello" } } } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_metrics_outputs.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_metrics_outputs.json index 6c03ced8c8f..a7d052b95d2 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_metrics_outputs.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_metrics_outputs.json @@ -32,7 +32,7 @@ "inputDefinitions": { "parameters": { "pipelineparam--loop-item-param-1": { - "type": "STRING" + "parameterType": "STRING" } } }, @@ -87,7 +87,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.5' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -107,7 +107,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.5' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -188,8 +188,8 @@ } } }, - "schemaVersion": "2.0.0", - "sdkVersion": "kfp-1.8.5" + "schemaVersion": "2.1.0", + "sdkVersion": "kfp-1.8.6" }, "runtimeConfig": { "gcsOutputDirectory": "dummy_root" diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_nested_conditions.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_nested_conditions.json index ec28059eff1..0cc5c12e27a 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_nested_conditions.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_nested_conditions.json @@ -31,7 +31,7 @@ "name": "condition-2" }, "triggerPolicy": { - "condition": "inputs.parameters['pipelineparam--flip-coin-2-Output'].string_value == inputs.parameters['pipelineparam--flip-coin-3-Output'].string_value" + "condition": "inputs.parameter_values['pipelineparam--flip-coin-2-Output'] == inputs.parameter_values['pipelineparam--flip-coin-3-Output']" } }, "flip-coin-3": { @@ -74,10 +74,10 @@ "inputDefinitions": { "parameters": { "pipelineparam--flip-coin-2-Output": { - "type": "STRING" + "parameterType": "STRING" }, "pipelineparam--flip-coin-Output": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -125,13 +125,13 @@ "inputDefinitions": { "parameters": { "pipelineparam--flip-coin-2-Output": { - "type": "STRING" + "parameterType": "STRING" }, "pipelineparam--flip-coin-3-Output": { - "type": "STRING" + "parameterType": "STRING" }, "pipelineparam--flip-coin-Output": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -141,7 +141,7 @@ "outputDefinitions": { "parameters": { "Output": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -151,7 +151,7 @@ "outputDefinitions": { "parameters": { "Output": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -161,7 +161,7 @@ "outputDefinitions": { "parameters": { "Output": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -171,7 +171,7 @@ "outputDefinitions": { "parameters": { "Output": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -181,7 +181,7 @@ "inputDefinitions": { "parameters": { "msg": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -191,7 +191,7 @@ "inputDefinitions": { "parameters": { "msg": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -201,7 +201,7 @@ "inputDefinitions": { "parameters": { "msg": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -211,7 +211,7 @@ "inputDefinitions": { "parameters": { "msg": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -375,7 +375,7 @@ "name": "condition-1" }, "triggerPolicy": { - "condition": "inputs.parameters['pipelineparam--flip-coin-Output'].string_value != 'no-such-result'" + "condition": "inputs.parameter_values['pipelineparam--flip-coin-Output'] != 'no-such-result'" } }, "flip-coin": { @@ -451,8 +451,8 @@ } } }, - "schemaVersion": "2.0.0", - "sdkVersion": "kfp-1.7.2" + "schemaVersion": "2.1.0", + "sdkVersion": "kfp-1.8.6" }, "runtimeConfig": { "gcsOutputDirectory": "dummy_root" diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_nested_conditions_yaml.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_nested_conditions_yaml.json index 7cff1928080..6325680bca0 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_nested_conditions_yaml.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_nested_conditions_yaml.json @@ -28,7 +28,7 @@ "name": "condition-2" }, "triggerPolicy": { - "condition": "inputs.parameters['pipelineparam--generate-random-number-output'].int_value > 5" + "condition": "int(inputs.parameter_values['pipelineparam--generate-random-number-output']) > 5" } }, "condition-3": { @@ -55,7 +55,7 @@ "name": "condition-3" }, "triggerPolicy": { - "condition": "inputs.parameters['pipelineparam--generate-random-number-output'].int_value <= 5" + "condition": "int(inputs.parameter_values['pipelineparam--generate-random-number-output']) <= 5" } }, "generate-random-number": { @@ -74,7 +74,7 @@ "inputDefinitions": { "parameters": { "pipelineparam--flip-coin-output": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -93,9 +93,7 @@ "parameters": { "msg": { "runtimeValue": { - "constantValue": { - "stringValue": "heads and {{$.inputs.parameters['pipelineparam--generate-random-number-output']}} > 5!" - } + "constant": "heads and {{$.inputs.parameters['pipelineparam--generate-random-number-output']}} > 5!" } }, "pipelineparam--generate-random-number-output": { @@ -112,10 +110,10 @@ "inputDefinitions": { "parameters": { "pipelineparam--flip-coin-output": { - "type": "STRING" + "parameterType": "STRING" }, "pipelineparam--generate-random-number-output": { - "type": "INT" + "parameterType": "NUMBER_INTEGER" } } } @@ -134,9 +132,7 @@ "parameters": { "msg": { "runtimeValue": { - "constantValue": { - "stringValue": "heads and {{$.inputs.parameters['pipelineparam--generate-random-number-output']}} <= 5!" - } + "constant": "heads and {{$.inputs.parameters['pipelineparam--generate-random-number-output']}} <= 5!" } }, "pipelineparam--generate-random-number-output": { @@ -153,10 +149,10 @@ "inputDefinitions": { "parameters": { "pipelineparam--flip-coin-output": { - "type": "STRING" + "parameterType": "STRING" }, "pipelineparam--generate-random-number-output": { - "type": "INT" + "parameterType": "NUMBER_INTEGER" } } } @@ -188,7 +184,7 @@ "name": "condition-5" }, "triggerPolicy": { - "condition": "inputs.parameters['pipelineparam--generate-random-number-2-output'].int_value > 15" + "condition": "int(inputs.parameter_values['pipelineparam--generate-random-number-2-output']) > 15" } }, "condition-6": { @@ -215,7 +211,7 @@ "name": "condition-6" }, "triggerPolicy": { - "condition": "inputs.parameters['pipelineparam--generate-random-number-2-output'].int_value <= 15" + "condition": "int(inputs.parameter_values['pipelineparam--generate-random-number-2-output']) <= 15" } }, "generate-random-number-2": { @@ -234,7 +230,7 @@ "inputDefinitions": { "parameters": { "pipelineparam--flip-coin-output": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -253,9 +249,7 @@ "parameters": { "msg": { "runtimeValue": { - "constantValue": { - "stringValue": "tails and {{$.inputs.parameters['pipelineparam--generate-random-number-2-output']}} > 15!" - } + "constant": "tails and {{$.inputs.parameters['pipelineparam--generate-random-number-2-output']}} > 15!" } }, "pipelineparam--generate-random-number-2-output": { @@ -272,10 +266,10 @@ "inputDefinitions": { "parameters": { "pipelineparam--flip-coin-output": { - "type": "STRING" + "parameterType": "STRING" }, "pipelineparam--generate-random-number-2-output": { - "type": "INT" + "parameterType": "NUMBER_INTEGER" } } } @@ -294,9 +288,7 @@ "parameters": { "msg": { "runtimeValue": { - "constantValue": { - "stringValue": "tails and {{$.inputs.parameters['pipelineparam--generate-random-number-2-output']}} <= 15!" - } + "constant": "tails and {{$.inputs.parameters['pipelineparam--generate-random-number-2-output']}} <= 15!" } }, "pipelineparam--generate-random-number-2-output": { @@ -313,10 +305,10 @@ "inputDefinitions": { "parameters": { "pipelineparam--flip-coin-output": { - "type": "STRING" + "parameterType": "STRING" }, "pipelineparam--generate-random-number-2-output": { - "type": "INT" + "parameterType": "NUMBER_INTEGER" } } } @@ -326,7 +318,7 @@ "outputDefinitions": { "parameters": { "output": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -336,7 +328,7 @@ "outputDefinitions": { "parameters": { "output": { - "type": "INT" + "parameterType": "NUMBER_INTEGER" } } } @@ -346,7 +338,7 @@ "outputDefinitions": { "parameters": { "output": { - "type": "INT" + "parameterType": "NUMBER_INTEGER" } } } @@ -356,7 +348,7 @@ "inputDefinitions": { "parameters": { "msg": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -366,7 +358,7 @@ "inputDefinitions": { "parameters": { "msg": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -376,7 +368,7 @@ "inputDefinitions": { "parameters": { "msg": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -386,7 +378,7 @@ "inputDefinitions": { "parameters": { "msg": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -502,7 +494,7 @@ "name": "condition-1" }, "triggerPolicy": { - "condition": "inputs.parameters['pipelineparam--flip-coin-output'].string_value == 'heads'" + "condition": "inputs.parameter_values['pipelineparam--flip-coin-output'] == 'heads'" } }, "condition-4": { @@ -526,7 +518,7 @@ "name": "condition-4" }, "triggerPolicy": { - "condition": "inputs.parameters['pipelineparam--flip-coin-output'].string_value == 'tails'" + "condition": "inputs.parameter_values['pipelineparam--flip-coin-output'] == 'tails'" } }, "flip-coin": { @@ -543,8 +535,8 @@ } } }, - "schemaVersion": "2.0.0", - "sdkVersion": "kfp-1.8.4" + "schemaVersion": "2.1.0", + "sdkVersion": "kfp-1.8.6" }, "runtimeConfig": { "gcsOutputDirectory": "dummy_root" diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_nested_loops.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_nested_loops.json index 4a5c7b8eb4a..6b6b73cd03b 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_nested_loops.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_nested_loops.json @@ -34,10 +34,10 @@ "inputDefinitions": { "parameters": { "pipelineparam--loop_parameter": { - "type": "STRING" + "parameterType": "LIST" }, "pipelineparam--loop_parameter-loop-item": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -69,13 +69,13 @@ "inputDefinitions": { "parameters": { "pipelineparam--loop_parameter-loop-item": { - "type": "STRING" + "parameterType": "STRING" }, "pipelineparam--loop_parameter-loop-item-subvar-p_a": { - "type": "STRING" + "parameterType": "STRING" }, "pipelineparam--loop_parameter-loop-item-subvar-p_a-loop-item": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -127,7 +127,7 @@ "inputDefinitions": { "parameters": { "pipelineparam--loop-item-param-3": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -161,10 +161,10 @@ "inputDefinitions": { "parameters": { "pipelineparam--loop-item-param-3": { - "type": "STRING" + "parameterType": "STRING" }, "pipelineparam--loop-item-param-5": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -174,7 +174,7 @@ "inputDefinitions": { "parameters": { "msg": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -184,7 +184,7 @@ "inputDefinitions": { "parameters": { "msg": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -194,10 +194,10 @@ "inputDefinitions": { "parameters": { "msg": { - "type": "STRING" + "parameterType": "STRING" }, "msg2": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -216,7 +216,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.5' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -236,7 +236,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.5' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -256,7 +256,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.5' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -313,19 +313,40 @@ "inputDefinitions": { "parameters": { "loop_parameter": { - "type": "STRING" + "parameterType": "LIST" } } } }, - "schemaVersion": "2.0.0", - "sdkVersion": "kfp-1.8.5" + "schemaVersion": "2.1.0", + "sdkVersion": "kfp-1.8.6" }, "runtimeConfig": { - "parameters": { - "loop_parameter": { - "stringValue": "[{\"p_a\": [{\"q_a\": 1}, {\"q_a\": 2}], \"p_b\": \"hello\"}, {\"p_a\": [{\"q_a\": 11}, {\"q_a\": 22}], \"p_b\": \"halo\"}]" - } + "parameterValues": { + "loop_parameter": [ + { + "p_a": [ + { + "q_a": 1.0 + }, + { + "q_a": 2.0 + } + ], + "p_b": "hello" + }, + { + "p_a": [ + { + "q_a": 11.0 + }, + { + "q_a": 22.0 + } + ], + "p_b": "halo" + } + ] } } } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_ontology.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_ontology.json index 74bb82f1bbd..97cf271e27d 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_ontology.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_ontology.json @@ -6,7 +6,7 @@ "inputDefinitions": { "parameters": { "input_location": { - "type": "STRING" + "parameterType": "STRING" } } }, @@ -34,10 +34,10 @@ }, "parameters": { "n_epochs": { - "type": "INT" + "parameterType": "NUMBER_INTEGER" }, "optimizer": { - "type": "STRING" + "parameterType": "STRING" } } }, @@ -144,32 +144,26 @@ "inputDefinitions": { "parameters": { "input_location": { - "type": "STRING" + "parameterType": "STRING" }, "n_epochs": { - "type": "INT" + "parameterType": "NUMBER_INTEGER" }, "optimizer": { - "type": "STRING" + "parameterType": "STRING" } } } }, - "schemaVersion": "2.0.0", - "sdkVersion": "kfp-1.7.2" + "schemaVersion": "2.1.0", + "sdkVersion": "kfp-1.8.6" }, "runtimeConfig": { "gcsOutputDirectory": "dummy_root", - "parameters": { - "input_location": { - "stringValue": "gs://test-bucket/pipeline_root" - }, - "n_epochs": { - "intValue": "200" - }, - "optimizer": { - "stringValue": "sgd" - } + "parameterValues": { + "input_location": "gs://test-bucket/pipeline_root", + "n_epochs": 200.0, + "optimizer": "sgd" } } } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_params_containing_format.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_params_containing_format.json index d12508fe456..0b61181cd03 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_params_containing_format.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_params_containing_format.json @@ -21,9 +21,7 @@ }, "text2": { "runtimeValue": { - "constantValue": { - "stringValue": " and {{$.inputs.parameters['pipelineparam--name']}}." - } + "constant": " and {{$.inputs.parameters['pipelineparam--name']}}." } } } @@ -37,10 +35,10 @@ "inputDefinitions": { "parameters": { "pipelineparam--loop-item-param-1": { - "type": "STRING" + "parameterType": "STRING" }, "pipelineparam--name": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -50,14 +48,14 @@ "inputDefinitions": { "parameters": { "text": { - "type": "STRING" + "parameterType": "STRING" } } }, "outputDefinitions": { "parameters": { "Output": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -67,14 +65,14 @@ "inputDefinitions": { "parameters": { "text": { - "type": "STRING" + "parameterType": "STRING" } } }, "outputDefinitions": { "parameters": { "Output": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -84,17 +82,17 @@ "inputDefinitions": { "parameters": { "text1": { - "type": "STRING" + "parameterType": "STRING" }, "text2": { - "type": "STRING" + "parameterType": "STRING" } } }, "outputDefinitions": { "parameters": { "Output": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -198,9 +196,7 @@ }, "text": { "runtimeValue": { - "constantValue": { - "stringValue": "Hello {{$.inputs.parameters['pipelineparam--name']}}" - } + "constant": "Hello {{$.inputs.parameters['pipelineparam--name']}}" } } } @@ -229,9 +225,7 @@ }, "text": { "runtimeValue": { - "constantValue": { - "stringValue": "{{$.inputs.parameters['pipelineparam--print-op-Output']}}, again." - } + "constant": "{{$.inputs.parameters['pipelineparam--print-op-Output']}}, again." } } } @@ -245,20 +239,18 @@ "inputDefinitions": { "parameters": { "name": { - "type": "STRING" + "parameterType": "STRING" } } } }, - "schemaVersion": "2.0.0", - "sdkVersion": "kfp-1.7.2" + "schemaVersion": "2.1.0", + "sdkVersion": "kfp-1.8.6" }, "runtimeConfig": { "gcsOutputDirectory": "dummy_root", - "parameters": { - "name": { - "stringValue": "KFP" - } + "parameterValues": { + "name": "KFP" } } } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_resource_spec.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_resource_spec.json index e1d4044de0d..c9e9476baef 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_resource_spec.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_resource_spec.json @@ -6,7 +6,7 @@ "inputDefinitions": { "parameters": { "input_location": { - "type": "STRING" + "parameterType": "STRING" } } }, @@ -34,10 +34,10 @@ }, "parameters": { "n_epochs": { - "type": "INT" + "parameterType": "NUMBER_INTEGER" }, "optimizer": { - "type": "STRING" + "parameterType": "STRING" } } }, @@ -152,32 +152,26 @@ "inputDefinitions": { "parameters": { "input_location": { - "type": "STRING" + "parameterType": "STRING" }, "n_epochs": { - "type": "INT" + "parameterType": "NUMBER_INTEGER" }, "optimizer": { - "type": "STRING" + "parameterType": "STRING" } } } }, - "schemaVersion": "2.0.0", - "sdkVersion": "kfp-1.7.2" + "schemaVersion": "2.1.0", + "sdkVersion": "kfp-1.8.6" }, "runtimeConfig": { "gcsOutputDirectory": "dummy_root", - "parameters": { - "input_location": { - "stringValue": "gs://test-bucket/pipeline_root" - }, - "n_epochs": { - "intValue": "200" - }, - "optimizer": { - "stringValue": "sgd" - } + "parameterValues": { + "input_location": "gs://test-bucket/pipeline_root", + "n_epochs": 200.0, + "optimizer": "sgd" } } } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_reused_component.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_reused_component.json index 41f8a6f44d1..4891dd4133f 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_reused_component.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_reused_component.json @@ -6,17 +6,17 @@ "inputDefinitions": { "parameters": { "op1": { - "type": "INT" + "parameterType": "NUMBER_INTEGER" }, "op2": { - "type": "INT" + "parameterType": "NUMBER_INTEGER" } } }, "outputDefinitions": { "parameters": { "sum": { - "type": "INT" + "parameterType": "NUMBER_INTEGER" } } } @@ -26,17 +26,17 @@ "inputDefinitions": { "parameters": { "op1": { - "type": "INT" + "parameterType": "NUMBER_INTEGER" }, "op2": { - "type": "INT" + "parameterType": "NUMBER_INTEGER" } } }, "outputDefinitions": { "parameters": { "sum": { - "type": "INT" + "parameterType": "NUMBER_INTEGER" } } } @@ -46,17 +46,17 @@ "inputDefinitions": { "parameters": { "op1": { - "type": "INT" + "parameterType": "NUMBER_INTEGER" }, "op2": { - "type": "INT" + "parameterType": "NUMBER_INTEGER" } } }, "outputDefinitions": { "parameters": { "sum": { - "type": "INT" + "parameterType": "NUMBER_INTEGER" } } } @@ -125,9 +125,7 @@ }, "op2": { "runtimeValue": { - "constantValue": { - "intValue": "3" - } + "constant": 3.0 } } } @@ -183,9 +181,7 @@ }, "op2": { "runtimeValue": { - "constantValue": { - "intValue": "7" - } + "constant": 7.0 } } } @@ -199,26 +195,22 @@ "inputDefinitions": { "parameters": { "a": { - "type": "INT" + "parameterType": "NUMBER_INTEGER" }, "b": { - "type": "INT" + "parameterType": "NUMBER_INTEGER" } } } }, - "schemaVersion": "2.0.0", - "sdkVersion": "kfp-1.7.2" + "schemaVersion": "2.1.0", + "sdkVersion": "kfp-1.8.6" }, "runtimeConfig": { "gcsOutputDirectory": "dummy_root", - "parameters": { - "a": { - "intValue": "2" - }, - "b": { - "intValue": "5" - } + "parameterValues": { + "a": 2.0, + "b": 5.0 } } } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_various_io_types.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_various_io_types.json index cfda8706bed..4cf44688eef 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_various_io_types.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_various_io_types.json @@ -56,7 +56,7 @@ }, "parameters": { "input_a": { - "type": "INT" + "parameterType": "NUMBER_INTEGER" } } } @@ -66,16 +66,16 @@ "inputDefinitions": { "parameters": { "input_1": { - "type": "STRING" + "parameterType": "STRING" }, "input_2": { - "type": "DOUBLE" + "parameterType": "NUMBER_DOUBLE" }, "input_3": { - "type": "STRING" + "parameterType": "STRING" }, "input_4": { - "type": "STRING" + "parameterType": "STRING" } } }, @@ -132,7 +132,7 @@ }, "parameters": { "output_1": { - "type": "INT" + "parameterType": "NUMBER_INTEGER" } } } @@ -270,9 +270,7 @@ }, "input_2": { "runtimeValue": { - "constantValue": { - "doubleValue": 3.1415926 - } + "constant": 3.1415926 } }, "input_3": { @@ -292,26 +290,24 @@ "inputDefinitions": { "parameters": { "input1": { - "type": "STRING" + "parameterType": "STRING" }, "input3": { - "type": "STRING" + "parameterType": "STRING" }, "input4": { - "type": "STRING" + "parameterType": "STRING" } } } }, - "schemaVersion": "2.0.0", - "sdkVersion": "kfp-1.8.3" + "schemaVersion": "2.1.0", + "sdkVersion": "kfp-1.8.6" }, "runtimeConfig": { "gcsOutputDirectory": "dummy_root", - "parameters": { - "input4": { - "stringValue": "" - } + "parameterValues": { + "input4": "" } } } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/two_step_pipeline.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/two_step_pipeline.json index 6e32cad5603..5c979bf7e5e 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/two_step_pipeline.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/two_step_pipeline.json @@ -19,7 +19,7 @@ "inputDefinitions": { "parameters": { "text": { - "type": "STRING" + "parameterType": "STRING" } } }, @@ -115,20 +115,18 @@ "inputDefinitions": { "parameters": { "text": { - "type": "STRING" + "parameterType": "STRING" } } } }, - "schemaVersion": "2.0.0", - "sdkVersion": "kfp-1.7.2" + "schemaVersion": "2.1.0", + "sdkVersion": "kfp-1.8.6" }, "runtimeConfig": { "gcsOutputDirectory": "dummy_root", - "parameters": { - "text": { - "stringValue": "Hello KFP!" - } + "parameterValues": { + "text": "Hello KFP!" } } } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/v2_component_with_optional_inputs.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/v2_component_with_optional_inputs.json index 8a6bf06390a..8ce7a0a5942 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/v2_component_with_optional_inputs.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/v2_component_with_optional_inputs.json @@ -6,10 +6,10 @@ "inputDefinitions": { "parameters": { "input1": { - "type": "STRING" + "parameterType": "STRING" }, "input2": { - "type": "STRING" + "parameterType": "STRING" } } } @@ -28,7 +28,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.5' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -56,16 +56,12 @@ "parameters": { "input1": { "runtimeValue": { - "constantValue": { - "stringValue": "Hello" - } + "constant": "Hello" } }, "input2": { "runtimeValue": { - "constantValue": { - "stringValue": "World" - } + "constant": "World" } } } @@ -77,8 +73,8 @@ } } }, - "schemaVersion": "2.0.0", - "sdkVersion": "kfp-1.8.5" + "schemaVersion": "2.1.0", + "sdkVersion": "kfp-1.8.6" }, "runtimeConfig": { "gcsOutputDirectory": "dummy_root" diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/xgboost_sample_pipeline.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/xgboost_sample_pipeline.json index d6354d2785f..68aac70d7d4 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/xgboost_sample_pipeline.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/xgboost_sample_pipeline.json @@ -6,16 +6,16 @@ "inputDefinitions": { "parameters": { "Format": { - "type": "STRING" + "parameterType": "STRING" }, "Limit": { - "type": "INT" + "parameterType": "NUMBER_INTEGER" }, "Select": { - "type": "STRING" + "parameterType": "STRING" }, "Where": { - "type": "STRING" + "parameterType": "STRING" } } }, @@ -72,7 +72,7 @@ }, "parameters": { "label_column": { - "type": "INT" + "parameterType": "NUMBER_INTEGER" } } }, @@ -106,7 +106,7 @@ }, "parameters": { "label_column_name": { - "type": "STRING" + "parameterType": "STRING" } } }, @@ -140,7 +140,7 @@ }, "parameters": { "label_column_name": { - "type": "STRING" + "parameterType": "STRING" } } }, @@ -174,7 +174,7 @@ }, "parameters": { "label_column": { - "type": "INT" + "parameterType": "NUMBER_INTEGER" } } }, @@ -202,25 +202,25 @@ }, "parameters": { "booster": { - "type": "STRING" + "parameterType": "STRING" }, "label_column": { - "type": "INT" + "parameterType": "NUMBER_INTEGER" }, "learning_rate": { - "type": "DOUBLE" + "parameterType": "NUMBER_DOUBLE" }, "max_depth": { - "type": "INT" + "parameterType": "NUMBER_INTEGER" }, "min_split_loss": { - "type": "DOUBLE" + "parameterType": "NUMBER_DOUBLE" }, "num_iterations": { - "type": "INT" + "parameterType": "NUMBER_INTEGER" }, "objective": { - "type": "STRING" + "parameterType": "STRING" } } }, @@ -254,25 +254,25 @@ }, "parameters": { "booster": { - "type": "STRING" + "parameterType": "STRING" }, "label_column_name": { - "type": "STRING" + "parameterType": "STRING" }, "learning_rate": { - "type": "DOUBLE" + "parameterType": "NUMBER_DOUBLE" }, "max_depth": { - "type": "INT" + "parameterType": "NUMBER_INTEGER" }, "min_split_loss": { - "type": "DOUBLE" + "parameterType": "NUMBER_DOUBLE" }, "num_iterations": { - "type": "INT" + "parameterType": "NUMBER_INTEGER" }, "objective": { - "type": "STRING" + "parameterType": "STRING" } } }, @@ -518,30 +518,22 @@ "parameters": { "Format": { "runtimeValue": { - "constantValue": { - "stringValue": "csv" - } + "constant": "csv" } }, "Limit": { "runtimeValue": { - "constantValue": { - "intValue": "10000" - } + "constant": 10000.0 } }, "Select": { "runtimeValue": { - "constantValue": { - "stringValue": "tips,trip_seconds,trip_miles,pickup_community_area,dropoff_community_area,fare,tolls,extras,trip_total" - } + "constant": "tips,trip_seconds,trip_miles,pickup_community_area,dropoff_community_area,fare,tolls,extras,trip_total" } }, "Where": { "runtimeValue": { - "constantValue": { - "stringValue": "trip_start_timestamp >= \"2019-01-01\" AND trip_start_timestamp < \"2019-02-01\"" - } + "constant": "trip_start_timestamp >= \"2019-01-01\" AND trip_start_timestamp < \"2019-02-01\"" } } } @@ -603,9 +595,7 @@ "parameters": { "label_column": { "runtimeValue": { - "constantValue": { - "intValue": "0" - } + "constant": 0.0 } } } @@ -643,9 +633,7 @@ "parameters": { "label_column_name": { "runtimeValue": { - "constantValue": { - "stringValue": "tips" - } + "constant": "tips" } } } @@ -683,9 +671,7 @@ "parameters": { "label_column_name": { "runtimeValue": { - "constantValue": { - "stringValue": "tips" - } + "constant": "tips" } } } @@ -723,9 +709,7 @@ "parameters": { "label_column": { "runtimeValue": { - "constantValue": { - "intValue": "0" - } + "constant": 0.0 } } } @@ -756,51 +740,37 @@ "parameters": { "booster": { "runtimeValue": { - "constantValue": { - "stringValue": "gbtree" - } + "constant": "gbtree" } }, "label_column": { "runtimeValue": { - "constantValue": { - "intValue": "0" - } + "constant": 0.0 } }, "learning_rate": { "runtimeValue": { - "constantValue": { - "doubleValue": 0.3 - } + "constant": 0.3 } }, "max_depth": { "runtimeValue": { - "constantValue": { - "intValue": "6" - } + "constant": 6.0 } }, "min_split_loss": { "runtimeValue": { - "constantValue": { - "doubleValue": 0.0 - } + "constant": 0.0 } }, "num_iterations": { "runtimeValue": { - "constantValue": { - "intValue": "200" - } + "constant": 200.0 } }, "objective": { "runtimeValue": { - "constantValue": { - "stringValue": "reg:squarederror" - } + "constant": "reg:squarederror" } } } @@ -831,51 +801,37 @@ "parameters": { "booster": { "runtimeValue": { - "constantValue": { - "stringValue": "gbtree" - } + "constant": "gbtree" } }, "label_column_name": { "runtimeValue": { - "constantValue": { - "stringValue": "tips" - } + "constant": "tips" } }, "learning_rate": { "runtimeValue": { - "constantValue": { - "doubleValue": 0.3 - } + "constant": 0.3 } }, "max_depth": { "runtimeValue": { - "constantValue": { - "intValue": "6" - } + "constant": 6.0 } }, "min_split_loss": { "runtimeValue": { - "constantValue": { - "doubleValue": 0.0 - } + "constant": 0.0 } }, "num_iterations": { "runtimeValue": { - "constantValue": { - "intValue": "200" - } + "constant": 200.0 } }, "objective": { "runtimeValue": { - "constantValue": { - "stringValue": "reg:squarederror" - } + "constant": "reg:squarederror" } } } @@ -887,8 +843,8 @@ } } }, - "schemaVersion": "2.0.0", - "sdkVersion": "kfp-1.7.2" + "schemaVersion": "2.1.0", + "sdkVersion": "kfp-1.8.6" }, "runtimeConfig": { "gcsOutputDirectory": "dummy_root" diff --git a/sdk/python/kfp/v2/components/executor.py b/sdk/python/kfp/v2/components/executor.py index 2e0a57f06ec..12783ac796f 100644 --- a/sdk/python/kfp/v2/components/executor.py +++ b/sdk/python/kfp/v2/components/executor.py @@ -68,6 +68,13 @@ def _get_output_artifact(self, name: str): def _get_input_parameter_value(self, parameter_name: str, parameter_type: Any): + parameter_values = self._input.get('inputs', + {}).get('parameterValues', None) + + if parameter_values is not None: + value = parameter_values.get(parameter_name) + return value + parameter = self._input.get('inputs', {}).get('parameters', {}).get(parameter_name, None) @@ -120,20 +127,22 @@ def _get_input_artifact_path(self, artifact_name: str): def _write_output_parameter_value(self, name: str, value: Union[str, int, float, bool, dict, list, Dict, List]): - if type(value) == str: - output = {'stringValue': value} - elif type(value) == int: - output = {'intValue': value} - elif type(value) == float: - output = {'doubleValue': value} + if isinstance(value, (float, int)): + output = str(value) + elif isinstance(value, str): + # value is already a string. + output = value + elif isinstance(value, (bool, list, dict)): + output = json.dumps(value) else: - # For bool, list, dict, List, Dict, json serialize the value. - output = {'stringValue': json.dumps(value)} + raise ValueError( + 'Unable to serialize unknown type `{}` for parameter' + ' input with value `{}`'.format(value, type(value))) - if not self._executor_output.get('parameters'): - self._executor_output['parameters'] = {} + if not self._executor_output.get('parameterValues'): + self._executor_output['parameterValues'] = {} - self._executor_output['parameters'][name] = output + self._executor_output['parameterValues'][name] = value def _write_output_artifact_payload(self, name: str, value: Any): path = self._get_output_artifact_path(name) diff --git a/sdk/python/kfp/v2/components/executor_main.py b/sdk/python/kfp/v2/components/executor_main.py index 0b535110d96..612b7396a08 100644 --- a/sdk/python/kfp/v2/components/executor_main.py +++ b/sdk/python/kfp/v2/components/executor_main.py @@ -94,6 +94,9 @@ def executor_main(): executor_input = json.loads(args.executor_input) function_to_execute = getattr(module, func_name) + logging.info('Got executor_input:\n{}'.format( + json.dumps(executor_input, indent=4))) + executor = component_executor.Executor( executor_input=executor_input, function_to_execute=function_to_execute) diff --git a/sdk/python/kfp/v2/components/executor_test.py b/sdk/python/kfp/v2/components/executor_test.py index 607f9a7fa2e..a46e6a153dc 100644 --- a/sdk/python/kfp/v2/components/executor_test.py +++ b/sdk/python/kfp/v2/components/executor_test.py @@ -29,10 +29,8 @@ _EXECUTOR_INPUT = """\ { "inputs": { - "parameters": { - "input_parameter": { - "stringValue": "Hello, KFP" - } + "parameterValues": { + "input_parameter": "Hello, KFP" }, "artifacts": { "input_artifact_one_path": { @@ -217,16 +215,10 @@ def test_function_string_output(self): executor_input = """\ { "inputs": { - "parameters": { - "first_message": { - "stringValue": "Hello" - }, - "second_message": { - "stringValue": "" - }, - "third_message": { - "stringValue": "World" - } + "parameterValues": { + "first_message": "Hello", + "second_message": "", + "third_message": "World" } }, "outputs": { @@ -252,10 +244,8 @@ def test_func( 'r') as f: output_metadata = json.loads(f.read()) self.assertDictEqual(output_metadata, { - "parameters": { - "Output": { - "stringValue": "Hello, , World" - } + "parameterValues": { + "Output": "Hello, , World" }, }) @@ -263,13 +253,9 @@ def test_function_with_int_output(self): executor_input = """\ { "inputs": { - "parameters": { - "first": { - "intValue": 40 - }, - "second": { - "intValue": 2 - } + "parameterValues": { + "first": 40, + "second": 2 } }, "outputs": { @@ -291,10 +277,8 @@ def test_func(first: int, second: int) -> int: 'r') as f: output_metadata = json.loads(f.read()) self.assertDictEqual(output_metadata, { - "parameters": { - "Output": { - "intValue": 42 - } + "parameterValues": { + "Output": 42 }, }) @@ -302,13 +286,9 @@ def test_function_with_float_output(self): executor_input = """\ { "inputs": { - "parameters": { - "first": { - "doubleValue": 0.0 - }, - "second": { - "doubleValue": 1.2 - } + "parameterValues": { + "first": 0.0, + "second": 1.2 } }, "outputs": { @@ -330,10 +310,8 @@ def test_func(first: float, second: float) -> float: 'r') as f: output_metadata = json.loads(f.read()) self.assertDictEqual(output_metadata, { - "parameters": { - "Output": { - "doubleValue": 1.2 - } + "parameterValues": { + "Output": 1.2 }, }) @@ -341,13 +319,9 @@ def test_function_with_list_output(self): executor_input = """\ { "inputs": { - "parameters": { - "first": { - "intValue": 40 - }, - "second": { - "intValue": 2 - } + "parameterValues": { + "first": 40, + "second": 2 } }, "outputs": { @@ -369,10 +343,8 @@ def test_func(first: int, second: int) -> List: 'r') as f: output_metadata = json.loads(f.read()) self.assertDictEqual(output_metadata, { - "parameters": { - "Output": { - "stringValue": "[40, 2]" - } + "parameterValues": { + "Output": [40, 2] }, }) @@ -380,13 +352,9 @@ def test_function_with_dict_output(self): executor_input = """\ { "inputs": { - "parameters": { - "first": { - "intValue": 40 - }, - "second": { - "intValue": 2 - } + "parameterValues": { + "first": 40, + "second": 2 } }, "outputs": { @@ -408,9 +376,10 @@ def test_func(first: int, second: int) -> Dict: 'r') as f: output_metadata = json.loads(f.read()) self.assertDictEqual(output_metadata, { - "parameters": { - "Output": { - "stringValue": "{\"first\": 40, \"second\": 2}" + 'parameterValues': { + 'Output': { + 'first': 40, + 'second': 2 } }, }) @@ -419,13 +388,9 @@ def test_function_with_typed_list_output(self): executor_input = """\ { "inputs": { - "parameters": { - "first": { - "intValue": 40 - }, - "second": { - "intValue": 2 - } + "parameterValues": { + "first": 40, + "second": 2 } }, "outputs": { @@ -447,10 +412,8 @@ def test_func(first: int, second: int) -> List[int]: 'r') as f: output_metadata = json.loads(f.read()) self.assertDictEqual(output_metadata, { - "parameters": { - "Output": { - "stringValue": "[40, 2]" - } + "parameterValues": { + "Output": [40, 2] }, }) @@ -458,13 +421,9 @@ def test_function_with_typed_dict_output(self): executor_input = """\ { "inputs": { - "parameters": { - "first": { - "intValue": 40 - }, - "second": { - "intValue": 2 - } + "parameterValues": { + "first": 40, + "second": 2 } }, "outputs": { @@ -486,9 +445,10 @@ def test_func(first: int, second: int) -> Dict[str, int]: 'r') as f: output_metadata = json.loads(f.read()) self.assertDictEqual(output_metadata, { - "parameters": { + "parameterValues": { "Output": { - "stringValue": "{\"first\": 40, \"second\": 2}" + "first": 40, + "second": 2 } }, }) @@ -497,13 +457,9 @@ def test_artifact_output(self): executor_input = """\ { "inputs": { - "parameters": { - "first": { - "stringValue": "Hello" - }, - "second": { - "stringValue": "World" - } + "parameterValues": { + "first": "Hello", + "second": "World" } }, "outputs": { @@ -617,13 +573,9 @@ def func_returning_plain_tuple() -> NamedTuple('Outputs', [ }] } }, - "parameters": { - "output_string": { - "stringValue": "Some output string" - }, - "output_int": { - "intValue": 101 - } + "parameterValues": { + "output_int": 101, + "output_string": "Some output string" }, }) @@ -637,13 +589,9 @@ def test_function_with_optional_inputs(self): executor_input = """\ { "inputs": { - "parameters": { - "first_message": { - "stringValue": "Hello" - }, - "second_message": { - "stringValue": "World" - } + "parameterValues": { + "first_message": "Hello", + "second_message": "World" } }, "outputs": { @@ -672,12 +620,10 @@ def test_func( output_metadata = json.loads(f.read()) self.assertDictEqual( output_metadata, { - "parameters": { - "Output": { - "stringValue": "Hello (), " - "World (), " - "None ()." - } + "parameterValues": { + "Output": "Hello (), " + "World (), " + "None ()." }, }) diff --git a/sdk/python/kfp/v2/components/types/type_utils.py b/sdk/python/kfp/v2/components/types/type_utils.py index 911e00369a9..2959431d358 100644 --- a/sdk/python/kfp/v2/components/types/type_utils.py +++ b/sdk/python/kfp/v2/components/types/type_utils.py @@ -13,6 +13,7 @@ # limitations under the License. """Utilities for component I/O type mapping.""" import inspect +import json import re import warnings from typing import Dict, List, Optional, Type, Union @@ -41,27 +42,19 @@ # The keys are normalized (lowercased). These are types viewed as Parameters. # The values are the corresponding IR parameter primitive types. _PARAMETER_TYPES_MAPPING = { - 'integer': pipeline_spec_pb2.PrimitiveType.INT, - 'int': pipeline_spec_pb2.PrimitiveType.INT, - 'double': pipeline_spec_pb2.PrimitiveType.DOUBLE, - 'float': pipeline_spec_pb2.PrimitiveType.DOUBLE, - 'string': pipeline_spec_pb2.PrimitiveType.STRING, - 'str': pipeline_spec_pb2.PrimitiveType.STRING, - 'text': pipeline_spec_pb2.PrimitiveType.STRING, - 'bool': pipeline_spec_pb2.PrimitiveType.STRING, - 'boolean': pipeline_spec_pb2.PrimitiveType.STRING, - 'dict': pipeline_spec_pb2.PrimitiveType.STRING, - 'list': pipeline_spec_pb2.PrimitiveType.STRING, - 'jsonobject': pipeline_spec_pb2.PrimitiveType.STRING, - 'jsonarray': pipeline_spec_pb2.PrimitiveType.STRING, -} - -# Mapping primitive types to their IR message field names. -# This is used in constructing condition strings. -_PARAMETER_TYPES_VALUE_REFERENCE_MAPPING = { - pipeline_spec_pb2.PrimitiveType.INT: 'int_value', - pipeline_spec_pb2.PrimitiveType.DOUBLE: 'double_value', - pipeline_spec_pb2.PrimitiveType.STRING: 'string_value', + 'integer': pipeline_spec_pb2.ParameterType.NUMBER_INTEGER, + 'int': pipeline_spec_pb2.ParameterType.NUMBER_INTEGER, + 'double': pipeline_spec_pb2.ParameterType.NUMBER_DOUBLE, + 'float': pipeline_spec_pb2.ParameterType.NUMBER_DOUBLE, + 'string': pipeline_spec_pb2.ParameterType.STRING, + 'str': pipeline_spec_pb2.ParameterType.STRING, + 'text': pipeline_spec_pb2.ParameterType.STRING, + 'bool': pipeline_spec_pb2.ParameterType.BOOLEAN, + 'boolean': pipeline_spec_pb2.ParameterType.BOOLEAN, + 'dict': pipeline_spec_pb2.ParameterType.STRUCT, + 'list': pipeline_spec_pb2.ParameterType.LIST, + 'jsonobject': pipeline_spec_pb2.ParameterType.STRUCT, + 'jsonarray': pipeline_spec_pb2.ParameterType.LIST, } @@ -110,7 +103,7 @@ def get_artifact_type_schema( def get_parameter_type( param_type: Optional[Union[Type, str, dict]] -) -> pipeline_spec_pb2.PrimitiveType: +) -> pipeline_spec_pb2.ParameterType: """Get the IR I/O parameter type for the given ComponentSpec I/O type. Args: @@ -132,22 +125,44 @@ def get_parameter_type( return _PARAMETER_TYPES_MAPPING.get(type_name.lower()) -def get_parameter_type_field_name(type_name: Optional[str]) -> str: - """Get the IR field name for the given primitive type. - - For example: 'str' -> 'string_value', 'double' -> 'double_value', etc. - - Args: - type_name: type name of the ComponentSpec I/O primitive type. - - Returns: - The IR value reference field name. +def deserialize_parameter_value( + value: str, parameter_type: pipeline_spec_pb2.ParameterType +) -> Union[str, float, int, bool, list, dict]: + if parameter_type == pipeline_spec_pb2.ParameterType.NUMBER_DOUBLE: + result = float(value) + elif parameter_type == pipeline_spec_pb2.ParameterType.NUMBER_INTEGER: + result = int(value) + elif parameter_type == pipeline_spec_pb2.ParameterType.STRING: + # value is already a string. + result = value + elif parameter_type == pipeline_spec_pb2.ParameterType.BOOLEAN: + result = (value == 'True' or value == 'true') + elif parameter_type == pipeline_spec_pb2.ParameterType.LIST: + result = json.loads(value) + elif parameter_type == pipeline_spec_pb2.ParameterType.STRUCT: + result = json.loads(value) + else: + raise ValueError( + 'Unknown parameter type `{}` for input with value `{}`'.format( + parameter_type, value)) + + return result + + +def serialize_parameter_value( + value: Union[str, float, int, bool, list, dict]) -> str: + if isinstance(value, (float, int)): + result = str(value) + elif isinstance(value, str): + # value is already a string. + result = value + elif isinstance(value, (bool, list, dict)): + result = json.dumps(value) + else: + raise ValueError('Unable to serialize unknown type `{}` for parameter' + ' input with value `{}`'.format(value, type(value))) - Raises: - AttributeError: if type_name is not a string type. - """ - return _PARAMETER_TYPES_VALUE_REFERENCE_MAPPING.get( - get_parameter_type(type_name)) + return result def get_input_artifact_type_schema( diff --git a/sdk/python/kfp/v2/components/types/type_utils_test.py b/sdk/python/kfp/v2/components/types/type_utils_test.py index 2fa1fbc7b6b..9a315867c55 100644 --- a/sdk/python/kfp/v2/components/types/type_utils_test.py +++ b/sdk/python/kfp/v2/components/types/type_utils_test.py @@ -199,71 +199,71 @@ def test_get_artifact_type_schema(self, artifact_class_or_type_name, @parameterized.parameters( { 'given_type': 'Int', - 'expected_type': pb.PrimitiveType.INT, + 'expected_type': pb.ParameterType.NUMBER_INTEGER, }, { 'given_type': 'Integer', - 'expected_type': pb.PrimitiveType.INT, + 'expected_type': pb.ParameterType.NUMBER_INTEGER, }, { 'given_type': int, - 'expected_type': pb.PrimitiveType.INT, + 'expected_type': pb.ParameterType.NUMBER_INTEGER, }, { 'given_type': 'Double', - 'expected_type': pb.PrimitiveType.DOUBLE, + 'expected_type': pb.ParameterType.NUMBER_DOUBLE, }, { 'given_type': 'Float', - 'expected_type': pb.PrimitiveType.DOUBLE, + 'expected_type': pb.ParameterType.NUMBER_DOUBLE, }, { 'given_type': float, - 'expected_type': pb.PrimitiveType.DOUBLE, + 'expected_type': pb.ParameterType.NUMBER_DOUBLE, }, { 'given_type': 'String', - 'expected_type': pb.PrimitiveType.STRING, + 'expected_type': pb.ParameterType.STRING, }, { 'given_type': 'Text', - 'expected_type': pb.PrimitiveType.STRING, + 'expected_type': pb.ParameterType.STRING, }, { 'given_type': str, - 'expected_type': pb.PrimitiveType.STRING, + 'expected_type': pb.ParameterType.STRING, }, { 'given_type': 'Boolean', - 'expected_type': pb.PrimitiveType.STRING, + 'expected_type': pb.ParameterType.BOOLEAN, }, { 'given_type': bool, - 'expected_type': pb.PrimitiveType.STRING, + 'expected_type': pb.ParameterType.BOOLEAN, }, { 'given_type': 'Dict', - 'expected_type': pb.PrimitiveType.STRING, + 'expected_type': pb.ParameterType.STRUCT, }, { 'given_type': dict, - 'expected_type': pb.PrimitiveType.STRING, + 'expected_type': pb.ParameterType.STRUCT, }, { 'given_type': 'List', - 'expected_type': pb.PrimitiveType.STRING, + 'expected_type': pb.ParameterType.LIST, }, { 'given_type': list, - 'expected_type': pb.PrimitiveType.STRING, + 'expected_type': pb.ParameterType.LIST, }, { 'given_type': Dict[str, int], - 'expected_type': pb.PrimitiveType.STRING, + 'expected_type': pb.ParameterType.STRUCT, }, { 'given_type': List[Any], - 'expected_type': pb.PrimitiveType.STRING, + 'expected_type': pb.ParameterType.LIST, }, { 'given_type': { @@ -271,7 +271,7 @@ def test_get_artifact_type_schema(self, artifact_class_or_type_name, 'data_type': 'proto:tfx.components.trainer.TrainArgs' } }, - 'expected_type': pb.PrimitiveType.STRING, + 'expected_type': pb.ParameterType.STRUCT, }, ) def test_get_parameter_type(self, given_type, expected_type): @@ -279,7 +279,7 @@ def test_get_parameter_type(self, given_type, expected_type): type_utils.get_parameter_type(given_type)) # Test get parameter by Python type. - self.assertEqual(pb.PrimitiveType.INT, + self.assertEqual(pb.ParameterType.NUMBER_INTEGER, type_utils.get_parameter_type(int)) def test_get_parameter_type_invalid(self): @@ -314,14 +314,6 @@ def test_get_input_artifact_type_schema(self): type_utils.get_input_artifact_type_schema('input3', input_specs).schema_title) - def test_get_parameter_type_field_name(self): - self.assertEqual('string_value', - type_utils.get_parameter_type_field_name('String')) - self.assertEqual('int_value', - type_utils.get_parameter_type_field_name('Integer')) - self.assertEqual('double_value', - type_utils.get_parameter_type_field_name('Float')) - @parameterized.parameters( { 'given_type': 'String', diff --git a/v2/cacheutils/cache.go b/v2/cacheutils/cache.go index d3336c19531..048c7ffdd82 100644 --- a/v2/cacheutils/cache.go +++ b/v2/cacheutils/cache.go @@ -7,17 +7,15 @@ import ( "encoding/json" "fmt" "os" - "strconv" - "strings" "google.golang.org/grpc" "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/types/known/structpb" "github.com/golang/glog" "github.com/kubeflow/pipelines/api/v2alpha1/go/cachekey" "github.com/kubeflow/pipelines/api/v2alpha1/go/pipelinespec" api "github.com/kubeflow/pipelines/v2/kfp-api" - "github.com/kubeflow/pipelines/v2/third_party/ml_metadata" ) const ( @@ -61,6 +59,7 @@ func GenerateCacheKey( InputParameters: make(map[string]*pipelinespec.Value), OutputArtifactsSpec: make(map[string]*pipelinespec.RuntimeArtifact), OutputParametersSpec: make(map[string]string), + InputParameterValues: make(map[string]*structpb.Value), } for inputArtifactName, inputArtifactList := range inputs.GetArtifacts() { @@ -77,6 +76,10 @@ func GenerateCacheKey( } } + for inputParameterName, inputParameterValue := range inputs.GetParameterValues() { + cacheKey.InputParameterValues[inputParameterName] = inputParameterValue + } + for outputArtifactName, outputArtifactList := range outputs.GetArtifacts() { if len(outputArtifactList.Artifacts) == 0 { continue @@ -189,29 +192,3 @@ func (c *Client) CreateExecutionCache(ctx context.Context, task *api.Task) error } return nil } - -func GetMLMDOutputParams(cachedExecution *ml_metadata.Execution) (map[string]string, error) { - mlmdOutputParameters := make(map[string]string) - for customPropName, customPropValue := range cachedExecution.CustomProperties { - if strings.HasPrefix(customPropName, "output:") { - slice := strings.Split(customPropName, ":") - if len(slice) != 2 { - return nil, fmt.Errorf("failed to parse output parameter from MLMD execution custom property %v", customPropName) - } - outputParamName := slice[1] - var outputParamValue string - switch t := customPropValue.Value.(type) { - case *ml_metadata.Value_StringValue: - outputParamValue = customPropValue.GetStringValue() - case *ml_metadata.Value_DoubleValue: - outputParamValue = strconv.FormatFloat(customPropValue.GetDoubleValue(), 'f', -1, 64) - case *ml_metadata.Value_IntValue: - outputParamValue = strconv.FormatInt(customPropValue.GetIntValue(), 10) - default: - return nil, fmt.Errorf("unknown PipelineSpec Value type %T", t) - } - mlmdOutputParameters[outputParamName] = outputParamValue - } - } - return mlmdOutputParameters, nil -} diff --git a/v2/component/launcher.go b/v2/component/launcher.go index a6959f12793..35d9448a415 100644 --- a/v2/component/launcher.go +++ b/v2/component/launcher.go @@ -18,6 +18,7 @@ package component import ( "context" + "encoding/json" "errors" "flag" "fmt" @@ -42,9 +43,11 @@ import ( "gocloud.dev/blob" _ "gocloud.dev/blob/gcsblob" "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/types/known/structpb" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" ) + const OutputMetadataFilepath = "/tmp/kfp_outputs/output_metadata.json" // Launcher is used to launch KFP components. It handles the recording of the @@ -276,19 +279,12 @@ func (l *Launcher) executeWithCacheHit(ctx context.Context, executorInput *pipel if err != nil { return fmt.Errorf("failure while transfering cachedMLMDExecutionID %s from string to int64: %w", cachedMLMDExecutionID, err) } - executions, err := l.metadataClient.GetExecutions(ctx, []int64{cachedMLMDExecutionIDInt64}) + execution, err := l.metadataClient.GetExecution(ctx, cachedMLMDExecutionIDInt64) if err != nil { return fmt.Errorf("failure while getting execution of cachedMLMDExecutionID %v: %w", cachedMLMDExecutionIDInt64, err) } - if len(executions) == 0 { - return fmt.Errorf("the execution with id %s does not exist in MLMD", cachedMLMDExecutionID) - } - if len(executions) > 1 { - return fmt.Errorf("got multiple executions with id %s in MLMD", cachedMLMDExecutionID) - } - cachedExecution := executions[0] - outputParameters, err := l.storeOutputParameterValueFromCache(cachedExecution) + outputParameters, err := l.storeOutputParameterValueFromCache(execution) if err != nil { return fmt.Errorf("failed to store output parameter value from cache: %w", err) } @@ -304,47 +300,26 @@ func (l *Launcher) executeWithCacheHit(ctx context.Context, executorInput *pipel return nil } -func (l *Launcher) storeOutputParameterValueFromCache(cachedExecution *pb.Execution) (*metadata.Parameters, error) { - mlmdOutputParameters, err := cacheutils.GetMLMDOutputParams(cachedExecution) +func (l *Launcher) storeOutputParameterValueFromCache(execution *metadata.Execution) (map[string]*structpb.Value, error) { + _, outputs, err := execution.GetParameters() if err != nil { return nil, err } - // Read output parameters. - outputParameters := &metadata.Parameters{ - IntParameters: make(map[string]int64), - StringParameters: make(map[string]string), - DoubleParameters: make(map[string]float64), - } - for name, param := range l.runtimeInfo.OutputParameters { filename := param.Path - outputParamValue, ok := mlmdOutputParameters[name] + value, ok := outputs[name] if !ok { return nil, fmt.Errorf("can't find parameter %v in mlmdOutputParameters", name) } - if err := ioutil.WriteFile(filename, []byte(outputParamValue), 0644); err != nil { - return nil, fmt.Errorf("failed to write output parameter %q to file %q: %w", name, filename, err) + text, err := metadata.PbValueToText(value) + if err != nil { + return nil, err } - switch param.Type { - case "STRING": - outputParameters.StringParameters[name] = outputParamValue - case "INT": - i, err := strconv.ParseInt(strings.TrimSpace(outputParamValue), 10, 0) - if err != nil { - return nil, fmt.Errorf("failed to parse parameter name=%q value =%v to int: %w", name, outputParamValue, err) - } - outputParameters.IntParameters[name] = i - case "DOUBLE": - f, err := strconv.ParseFloat(strings.TrimSpace(outputParamValue), 0) - if err != nil { - return nil, fmt.Errorf("failed to parse parameter name=%q value =%v to double: %w", name, outputParamValue, err) - } - outputParameters.DoubleParameters[name] = f - default: - return nil, fmt.Errorf("unknown type. Expected STRING, INT or DOUBLE") + if err := ioutil.WriteFile(filename, []byte(text), 0644); err != nil { + return nil, fmt.Errorf("failed to write output parameter %q to file %q: %w", name, filename, err) } } - return outputParameters, nil + return outputs, nil } func (l *Launcher) storeOutputArtifactMetadataFromCache(ctx context.Context, executorInputOutputs *pipelinespec.ExecutorInput_Outputs, cachedMLMDExecutionID int64) ([]*metadata.OutputArtifact, error) { @@ -496,22 +471,14 @@ func (l *Launcher) publish(ctx context.Context, executorInput *pipelinespec.Exec } func (l *Launcher) dumpOutputParameters(executorOutput *pipelinespec.ExecutorOutput) error { - for name, parameter := range executorOutput.Parameters { + for name, parameter := range executorOutput.ParameterValues { wrap := func(err error) error { return fmt.Errorf("failed to dump output parameter %q in executor output to disk: %w", name, err) } - var value string - switch t := parameter.Value.(type) { - case *pipelinespec.Value_StringValue: - value = parameter.GetStringValue() - case *pipelinespec.Value_DoubleValue: - value = strconv.FormatFloat(parameter.GetDoubleValue(), 'f', -1, 64) - case *pipelinespec.Value_IntValue: - value = strconv.FormatInt(parameter.GetIntValue(), 10) - default: - return wrap(fmt.Errorf("unknown PipelineSpec Value type %T", t)) + value, err := metadata.PbValueToText(parameter) + if err != nil { + return wrap(err) } - outputParam, ok := l.runtimeInfo.OutputParameters[name] if !ok { return wrap(fmt.Errorf("parameter is not defined in component")) @@ -608,37 +575,49 @@ func (l *Launcher) dumpOutputArtifactsMetadata(outputArtifacts []*metadata.Outpu return nil } -func (l *Launcher) readOutputParameters() (*metadata.Parameters, error) { - outputParameters := &metadata.Parameters{ - IntParameters: make(map[string]int64), - StringParameters: make(map[string]string), - DoubleParameters: make(map[string]float64), - } +func (l *Launcher) readOutputParameters() (map[string]*structpb.Value, error) { + outputParameters := make(map[string]*structpb.Value) + for n, op := range l.runtimeInfo.OutputParameters { - msg := func(err error) error { + wrap := func(err error) error { return fmt.Errorf("Failed to read output parameter name=%q type=%q path=%q: %w", n, op.Type, op.Path, err) } + b, err := ioutil.ReadFile(op.Path) if err != nil { - return nil, msg(err) + return nil, wrap(err) } switch op.Type { case "STRING": - outputParameters.StringParameters[n] = string(b) - case "INT": - i, err := strconv.ParseInt(strings.TrimSpace(string(b)), 10, 0) + outputParameters[n] = structpb.NewStringValue(string(b)) + case "NUMBER_INTEGER", "NUMBER_DOUBLE": + f, err := strconv.ParseFloat(strings.TrimSpace(string(b)), 0) if err != nil { - return nil, msg(err) + return nil, wrap(fmt.Errorf("failed to parse number parameter: %w", err)) } - outputParameters.IntParameters[n] = i - case "DOUBLE": - f, err := strconv.ParseFloat(strings.TrimSpace(string(b)), 0) + outputParameters[n] = structpb.NewNumberValue(f) + case "BOOLEAN": + b, err := strconv.ParseBool(strings.TrimSpace(string(b))) if err != nil { - return nil, msg(err) + return nil, wrap(fmt.Errorf("failed to parse boolean parameter: %w", err)) } - outputParameters.DoubleParameters[n] = f + outputParameters[n] = structpb.NewBoolValue(b) + case "LIST": + value := &structpb.Value{} + if err := value.UnmarshalJSON(b); err != nil { + return nil, wrap(fmt.Errorf("failed to parse list parameter: %w", err)) + + } + outputParameters[n] = value + case "STRUCT": + value := &structpb.Value{} + if err := value.UnmarshalJSON(b); err != nil { + return nil, wrap(fmt.Errorf("failed to parse dict parameter: %w", err)) + + } + outputParameters[n] = value default: - return nil, msg(fmt.Errorf("unknown type. Expected STRING, INT or DOUBLE")) + return nil, wrap(fmt.Errorf("unknown ParameterType %q", op.Type)) } } return outputParameters, nil @@ -783,15 +762,27 @@ func getPlaceholders(executorInput *pipelinespec.ExecutorInput) (placeholders ma } // Prepare input parameter placeholders. - for name, parameter := range executorInput.Inputs.Parameters { + for name, parameter := range executorInput.Inputs.ParameterValues { key := fmt.Sprintf(`{{$.inputs.parameters['%s']}}`, name) - switch t := parameter.Value.(type) { - case *pipelinespec.Value_StringValue: + switch t := parameter.Kind.(type) { + case *structpb.Value_StringValue: placeholders[key] = parameter.GetStringValue() - case *pipelinespec.Value_DoubleValue: - placeholders[key] = strconv.FormatFloat(parameter.GetDoubleValue(), 'f', -1, 64) - case *pipelinespec.Value_IntValue: - placeholders[key] = strconv.FormatInt(parameter.GetIntValue(), 10) + case *structpb.Value_NumberValue: + placeholders[key] = strconv.FormatFloat(parameter.GetNumberValue(), 'f', -1, 64) + case *structpb.Value_BoolValue: + placeholders[key] = strconv.FormatBool(parameter.GetBoolValue()) + case *structpb.Value_ListValue: + b, err := json.Marshal(parameter.GetListValue()) + if err != nil { + return nil, fmt.Errorf("failed to JSON-marshal list input parameter %q: %w", name, err) + } + placeholders[key] = string(b) + case *structpb.Value_StructValue: + b, err := json.Marshal(parameter.GetStructValue()) + if err != nil { + return nil, fmt.Errorf("failed to JSON-marshal dict input parameter %q: %w", name, err) + } + placeholders[key] = string(b) default: return nil, fmt.Errorf("unknown PipelineSpec Value type %T", t) } @@ -838,8 +829,8 @@ func mergeRuntimeArtifacts(src, dst *pipelinespec.RuntimeArtifact) { func getExecutorOutputFile(path string) (*pipelinespec.ExecutorOutput, error) { // collect user executor output file executorOutput := &pipelinespec.ExecutorOutput{ - Parameters: map[string]*pipelinespec.Value{}, - Artifacts: map[string]*pipelinespec.ArtifactList{}, + ParameterValues: map[string]*structpb.Value{}, + Artifacts: map[string]*pipelinespec.ArtifactList{}, } _, err := os.Stat(path) diff --git a/v2/component/launcher_v2.go b/v2/component/launcher_v2.go index f7f6e442f3b..89cbe38aca1 100644 --- a/v2/component/launcher_v2.go +++ b/v2/component/launcher_v2.go @@ -5,20 +5,22 @@ import ( "context" "encoding/json" "fmt" - "github.com/golang/protobuf/ptypes/timestamp" - "github.com/kubeflow/pipelines/v2/cacheutils" - api "github.com/kubeflow/pipelines/v2/kfp-api" "io/ioutil" "strconv" "strings" "time" + "github.com/golang/protobuf/ptypes/timestamp" + "github.com/kubeflow/pipelines/v2/cacheutils" + api "github.com/kubeflow/pipelines/v2/kfp-api" + "github.com/kubeflow/pipelines/api/v2alpha1/go/pipelinespec" "github.com/kubeflow/pipelines/v2/metadata" "github.com/kubeflow/pipelines/v2/objectstore" pb "github.com/kubeflow/pipelines/v2/third_party/ml_metadata" "gocloud.dev/blob" "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/types/known/structpb" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" ) @@ -28,7 +30,7 @@ type LauncherV2Options struct { PodName, PodUID, MLMDServerAddress, - MLMDServerPort , + MLMDServerPort, PipelineName, RunID string } @@ -98,7 +100,7 @@ func NewLauncherV2(ctx context.Context, executionID int64, executorInputJSON, co options: *opts, metadataClient: metadataClient, k8sClient: k8sClient, - cacheClient: cacheClient, + cacheClient: cacheClient, }, nil } @@ -145,7 +147,7 @@ func (l *LauncherV2) Execute(ctx context.Context) (err error) { RunId: l.options.RunID, MlmdExecutionID: strconv.FormatInt(id, 10), CreatedAt: ×tamp.Timestamp{Seconds: executedStartedTime}, - FinishedAt: ×tamp.Timestamp{Seconds: time.Now().Unix()}, + FinishedAt: ×tamp.Timestamp{Seconds: time.Now().Unix()}, Fingerprint: fingerPrint, } return l.cacheClient.CreateExecutionCache(ctx, task) @@ -212,17 +214,14 @@ func (l *LauncherV2) publish(ctx context.Context, execution *metadata.Execution, err = fmt.Errorf("failed to publish results to ML Metadata: %w", err) } }() - outputParameters, err := metadata.NewParameters(executorOutput.GetParameters()) - if err != nil { - return err - } + outputParameters := executorOutput.GetParameterValues() // TODO(Bobgy): upload output artifacts. // TODO(Bobgy): when adding artifacts, we will need execution.pipeline to be non-nil, because we need // to publish output artifacts to the context too. return l.metadataClient.PublishExecution(ctx, execution, outputParameters, outputArtifacts, pb.Execution_COMPLETE) } -func executeV2(ctx context.Context, executorInput *pipelinespec.ExecutorInput, component *pipelinespec.ComponentSpec, cmd string, args []string, bucket *blob.Bucket, bucketConfig *objectstore.Config, metadataClient *metadata.Client, namespace string, k8sClient *kubernetes.Clientset ) (*pipelinespec.ExecutorOutput, []*metadata.OutputArtifact, error) { +func executeV2(ctx context.Context, executorInput *pipelinespec.ExecutorInput, component *pipelinespec.ComponentSpec, cmd string, args []string, bucket *blob.Bucket, bucketConfig *objectstore.Config, metadataClient *metadata.Client, namespace string, k8sClient *kubernetes.Clientset) (*pipelinespec.ExecutorOutput, []*metadata.OutputArtifact, error) { executorOutput, err := execute(ctx, executorInput, cmd, args, bucket, bucketConfig, namespace, k8sClient) if err != nil { return nil, nil, err @@ -251,10 +250,10 @@ func executeV2(ctx context.Context, executorInput *pipelinespec.ExecutorInput, c // collectOutputParameters collect output parameters from local disk and add them // to executor output. func collectOutputParameters(executorInput *pipelinespec.ExecutorInput, executorOutput *pipelinespec.ExecutorOutput, component *pipelinespec.ComponentSpec) error { - if executorOutput.Parameters == nil { - executorOutput.Parameters = make(map[string]*pipelinespec.Value) + if executorOutput.ParameterValues == nil { + executorOutput.ParameterValues = make(map[string]*structpb.Value) } - outputParameters := executorOutput.GetParameters() + outputParameters := executorOutput.GetParameterValues() for name, param := range executorInput.GetOutputs().GetParameters() { _, ok := outputParameters[name] if ok { @@ -274,24 +273,11 @@ func collectOutputParameters(executorInput *pipelinespec.ExecutorInput, executor if err != nil { return msg(err) } - switch paramSpec.GetType() { - case pipelinespec.PrimitiveType_STRING: - outputParameters[name] = metadata.StringValue(string(b)) - case pipelinespec.PrimitiveType_INT: - i, err := strconv.ParseInt(strings.TrimSpace(string(b)), 10, 0) - if err != nil { - return msg(err) - } - outputParameters[name] = metadata.IntValue(i) - case pipelinespec.PrimitiveType_DOUBLE: - f, err := strconv.ParseFloat(strings.TrimSpace(string(b)), 0) - if err != nil { - return msg(err) - } - outputParameters[name] = metadata.DoubleValue(f) - default: - return msg(fmt.Errorf("unknown type. Expected STRING, INT or DOUBLE")) + value, err := metadata.TextToPbValue(string(b), paramSpec.GetParameterType()) + if err != nil { + return msg(err) } + outputParameters[name] = value } return nil } diff --git a/v2/component/runtime_info.go b/v2/component/runtime_info.go index 81826c88263..f5cca36fdfe 100644 --- a/v2/component/runtime_info.go +++ b/v2/component/runtime_info.go @@ -29,7 +29,13 @@ import ( ) type inputParameter struct { - // Type should be one of "INT", "STRING" or "DOUBLE". + // Type should be one of: + // - STRING + // - NUMBER_INTEGER + // - NUMBER_DOUBLE + // - BOOLEAN + // - LIST + // - STRUCT Type string // File used to read input parameters. Value string @@ -46,7 +52,13 @@ type inputArtifact struct { } type outputParameter struct { - // Type should be one of "INT", "STRING" or "DOUBLE". + // Type should be one of: + // - STRING + // - NUMBER_INTEGER + // - NUMBER_DOUBLE + // - BOOLEAN + // - LIST + // - STRUCT Type string // File used to write output parameters to. Path string @@ -153,8 +165,8 @@ type generateOutputURI func(outputName string) string func (r *runtimeInfo) generateExecutorInput(genOutputURI generateOutputURI, outputMetadataFilepath string) (*pipelinespec.ExecutorInput, error) { inputs := &pipelinespec.ExecutorInput_Inputs{ - Parameters: make(map[string]*pipelinespec.Value), - Artifacts: make(map[string]*pipelinespec.ArtifactList), + ParameterValues: make(map[string]*structpb.Value), + Artifacts: make(map[string]*pipelinespec.ArtifactList), } outputs := &pipelinespec.ExecutorInput_Outputs{ @@ -164,26 +176,38 @@ func (r *runtimeInfo) generateExecutorInput(genOutputURI generateOutputURI, outp } for name, ip := range r.InputParameters { - value := &pipelinespec.Value{} + var value *structpb.Value switch ip.Type { case "STRING": - value.Value = &pipelinespec.Value_StringValue{StringValue: ip.Value} - case "INT": - i, err := strconv.ParseInt(ip.Value, 10, 0) + value = structpb.NewStringValue(ip.Value) + case "NUMBER_INTEGER", "NUMBER_DOUBLE": + f, err := strconv.ParseFloat(ip.Value, 0) if err != nil { - return nil, fmt.Errorf("failed to parse int parameter %q from '%v': %w", name, i, err) + return nil, fmt.Errorf("failed to parse number parameter %q from '%v': %w", name, ip.Value, err) } - value.Value = &pipelinespec.Value_IntValue{IntValue: i} - case "DOUBLE": - f, err := strconv.ParseFloat(ip.Value, 0) + value = structpb.NewNumberValue(f) + case "BOOLEAN": + b, err := strconv.ParseBool(ip.Value) if err != nil { - return nil, fmt.Errorf("failed to parse double parameter %q from '%v': %w", name, f, err) + return nil, fmt.Errorf("failed to parse boolean parameter %q from '%v': %w", name, ip.Value, err) + } + value = structpb.NewBoolValue(b) + case "LIST": + value = &structpb.Value{} + if err := value.UnmarshalJSON([]byte(ip.Value)); err != nil { + return nil, fmt.Errorf("failed to parse list parameter %q from '%v': %w", name, ip.Value, err) + + } + case "STRUCT": + value = &structpb.Value{} + if err := value.UnmarshalJSON([]byte(ip.Value)); err != nil { + return nil, fmt.Errorf("failed to parse struct parameter %q from '%v': %w", name, ip.Value, err) + } - value.Value = &pipelinespec.Value_DoubleValue{DoubleValue: f} default: return nil, fmt.Errorf("unknown ParameterType for parameter %q: %q", name, ip.Type) } - inputs.Parameters[name] = value + inputs.ParameterValues[name] = value } for name, ia := range r.InputArtifacts { diff --git a/v2/component/runtime_info_test.go b/v2/component/runtime_info_test.go index 25a268fb24e..ac0d11c084d 100644 --- a/v2/component/runtime_info_test.go +++ b/v2/component/runtime_info_test.go @@ -194,7 +194,15 @@ func TestExecutorInputGeneration(t *testing.T) { "type": "STRING" }, "num_steps": { - "type": "INT" + "type": "NUMBER_INTEGER" + }, + "list_parameter": { + "type": "LIST", + "value": "[1, 2, 3]" + }, + "dict_parameter": { + "type": "STRUCT", + "value": "{\"key_1\": \"value_1\", \"key_2\": 2}" } }, "inputArtifacts": { @@ -215,7 +223,7 @@ func TestExecutorInputGeneration(t *testing.T) { "path": "/tmp/outputs/output_parameter_one/data" }, "output_parameter_two": { - "type": "INT", + "type": "NUMBER_INTEGER", "path": "/tmp/outputs/output_parameter_two/data" } }, @@ -234,9 +242,22 @@ func TestExecutorInputGeneration(t *testing.T) { }`, dataset_one_path, dataset_two_path), want: &pipelinespec.ExecutorInput{ Inputs: &pipelinespec.ExecutorInput_Inputs{ - Parameters: map[string]*pipelinespec.Value{ - "message": {Value: &pipelinespec.Value_StringValue{StringValue: "Some string value with { \"special\": \"chars\" }"}}, - "num_steps": {Value: &pipelinespec.Value_IntValue{IntValue: 5}}, + ParameterValues: map[string]*structpb.Value{ + "message": structpb.NewStringValue("Some string value with { \"special\": \"chars\" }"), + "num_steps": structpb.NewNumberValue(5), + "list_parameter": structpb.NewListValue(&structpb.ListValue{ + Values: []*structpb.Value{ + structpb.NewNumberValue(1), + structpb.NewNumberValue(2), + structpb.NewNumberValue(3), + }, + }), + "dict_parameter": structpb.NewStructValue(&structpb.Struct{ + Fields: map[string]*structpb.Value{ + "key_1": structpb.NewStringValue("value_1"), + "key_2": structpb.NewNumberValue(2), + }, + }), }, Artifacts: map[string]*pipelinespec.ArtifactList{ "dataset_one": { @@ -312,7 +333,7 @@ func TestExecutorInputGeneration(t *testing.T) { } if diff := cmp.Diff(test.want, got, cmpopts.EquateEmpty(), protocmp.Transform()); diff != "" { - t.Errorf("generateExecutorInput() = %+v, want %+v\nDiff (-want, +got)\n%s", got, test.want, diff) + t.Errorf("generateExecutorInput() =\n%+v\nWant:\n%+v\nDiff (-want, +got)\n%s", got, test.want, diff) s, _ := json.MarshalIndent(test.want, "", " ") fmt.Printf("Want\n%s", s) } diff --git a/v2/driver/driver.go b/v2/driver/driver.go index 085683aab9a..1290ccddd89 100644 --- a/v2/driver/driver.go +++ b/v2/driver/driver.go @@ -3,7 +3,12 @@ package driver import ( "context" "fmt" + "path" + "strconv" + "strings" + "github.com/golang/glog" + structpb "github.com/golang/protobuf/ptypes/struct" "github.com/kubeflow/pipelines/api/v2alpha1/go/pipelinespec" "github.com/kubeflow/pipelines/v2/cacheutils" "github.com/kubeflow/pipelines/v2/component" @@ -12,9 +17,6 @@ import ( pb "github.com/kubeflow/pipelines/v2/third_party/ml_metadata" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" - "path" - "strconv" - "strings" ) // TODO Move driver to component package @@ -33,7 +35,7 @@ type Options struct { // required only by container driver DAGExecutionID int64 DAGContextID int64 - Container *pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec + Container *pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec // required only by root DAG driver Namespace string } @@ -106,7 +108,7 @@ func RootDAG(ctx context.Context, opts Options, mlmd *metadata.Client) (executio } executorInput := &pipelinespec.ExecutorInput{ Inputs: &pipelinespec.ExecutorInput_Inputs{ - Parameters: opts.RuntimeConfig.Parameters, + ParameterValues: opts.RuntimeConfig.GetParameterValues(), }, } // TODO(Bobgy): validate executorInput matches component spec types @@ -205,7 +207,7 @@ func Container(ctx context.Context, opts Options, mlmd *metadata.Client, cacheCl if err != nil { return nil, fmt.Errorf("failure while getting fingerPrint: %w", err) } - cachedMLMDExecutionID, err := cacheClient.GetExecutionCache(fingerPrint, "pipeline/" + opts.PipelineName, opts.Namespace) + cachedMLMDExecutionID, err := cacheClient.GetExecutionCache(fingerPrint, "pipeline/"+opts.PipelineName, opts.Namespace) if err != nil { return nil, fmt.Errorf("failure while getting executionCache: %w", err) } @@ -224,21 +226,17 @@ func Container(ctx context.Context, opts Options, mlmd *metadata.Client, cacheCl if err != nil { return nil, err } - outputParameters, err := metadata.NewParameters(executorOutput.GetParameters()) - if err != nil { - return nil, err - } // TODO(Bobgy): upload output artifacts. // TODO(Bobgy): when adding artifacts, we will need execution.pipeline to be non-nil, because we need // to publish output artifacts to the context too. - if err := mlmd.PublishExecution(ctx, createdExecution, outputParameters, outputArtifacts, pb.Execution_CACHED); err != nil { + if err := mlmd.PublishExecution(ctx, createdExecution, executorOutput.GetParameterValues(), outputArtifacts, pb.Execution_CACHED); err != nil { return nil, fmt.Errorf("failed to publish cached execution: %w", err) } glog.Infof("Cached") - return &Execution{ + return &Execution{ ID: createdExecution.GetID(), ExecutorInput: executorInput, - Cached: true, + Cached: true, }, nil } @@ -257,57 +255,19 @@ func reuseCachedOutputs(ctx context.Context, executorInput *pipelinespec.Executo if err != nil { return nil, nil, fmt.Errorf("failure while getting execution of cachedMLMDExecutionID %v: %w", cachedMLMDExecutionIDInt64, err) } - cachedExecution := execution.GetExecution() executorOutput := &pipelinespec.ExecutorOutput{ - Parameters: map[string]*pipelinespec.Value{}, - Artifacts: map[string]*pipelinespec.ArtifactList{}, + Artifacts: map[string]*pipelinespec.ArtifactList{}, } - if err := collectOutPutParametersFromCache(executorOutput, outputDefinitions, executorInput, cachedExecution); err != nil { + _, outputs, err := execution.GetParameters() + if err != nil { return nil, nil, fmt.Errorf("failed to collect output parameters from cache: %w", err) } + executorOutput.ParameterValues = outputs outputArtifacts, err := collectOutputArtifactMetadataFromCache(ctx, executorInput, cachedMLMDExecutionIDInt64, mlmd) if err != nil { return nil, nil, fmt.Errorf("failed collect output artifact metadata from cache: %w", err) } return executorOutput, outputArtifacts, nil - -} - -func collectOutPutParametersFromCache(executorOutput *pipelinespec.ExecutorOutput, outputDefinitions *pipelinespec.ComponentOutputsSpec, executorInput *pipelinespec.ExecutorInput, cachedExecution *pb.Execution, ) error { - mlmdOutputParameters, err := cacheutils.GetMLMDOutputParams(cachedExecution) - if err != nil { - return err - } - outputParameters := executorOutput.GetParameters() - for name, _ := range executorInput.GetOutputs().GetParameters() { - paramSpec, ok := outputDefinitions.GetParameters()[name] - if !ok { - return fmt.Errorf("can't find parameter %v in outputDefinitions", name) - } - outputParamValue, ok := mlmdOutputParameters[name] - if !ok { - return fmt.Errorf("can't find parameter %v in mlmdOutputParameters", name) - } - switch paramSpec.GetType() { - case pipelinespec.PrimitiveType_STRING: - outputParameters[name] = metadata.StringValue(outputParamValue) - case pipelinespec.PrimitiveType_INT: - i, err := strconv.ParseInt(strings.TrimSpace(outputParamValue), 10, 0) - if err != nil { - return fmt.Errorf("failed to parse parameter name=%q value =%v to int: %w", name, outputParamValue, err) - } - outputParameters[name] = metadata.IntValue(i) - case pipelinespec.PrimitiveType_DOUBLE: - f, err := strconv.ParseFloat(strings.TrimSpace(outputParamValue), 0) - if err != nil { - return fmt.Errorf("failed to parse parameter name=%q value =%v to double: %w", name, outputParamValue, err) - } - outputParameters[name] = metadata.DoubleValue(f) - default: - return fmt.Errorf("unknown type. Expected STRING, INT or DOUBLE") - } - } - return nil } func collectOutputArtifactMetadataFromCache(ctx context.Context, executorInput *pipelinespec.ExecutorInput, cachedMLMDExecutionID int64, mlmd *metadata.Client) ([]*metadata.OutputArtifact, error) { @@ -391,8 +351,8 @@ func resolveInputs(ctx context.Context, dag *metadata.DAG, task *pipelinespec.Pi } glog.Infof("parent DAG input parameters %+v", inputParams) inputs := &pipelinespec.ExecutorInput_Inputs{ - Parameters: make(map[string]*pipelinespec.Value), - Artifacts: make(map[string]*pipelinespec.ArtifactList), + ParameterValues: make(map[string]*structpb.Value), + Artifacts: make(map[string]*pipelinespec.ArtifactList), } // get executions in context on demand var tasksCache map[string]*metadata.Execution @@ -424,7 +384,7 @@ func resolveInputs(ctx context.Context, dag *metadata.DAG, task *pipelinespec.Pi if !ok { return nil, paramError(fmt.Errorf("parent DAG does not have input parameter %s", componentInput)) } - inputs.Parameters[name] = v + inputs.ParameterValues[name] = v case *pipelinespec.TaskInputsSpec_InputParameterSpec_TaskOutputParameter: taskOutput := paramSpec.GetTaskOutputParameter() @@ -450,10 +410,13 @@ func resolveInputs(ctx context.Context, dag *metadata.DAG, task *pipelinespec.Pi if !ok { return nil, paramError(fmt.Errorf("cannot find output parameter key %q in producer task %q", taskOutput.GetOutputParameterKey(), taskOutput.GetProducerTask())) } - inputs.Parameters[name] = param + inputs.ParameterValues[name] = param case *pipelinespec.TaskInputsSpec_InputParameterSpec_RuntimeValue: runtimeValue := paramSpec.GetRuntimeValue() switch t := runtimeValue.Value.(type) { + case *pipelinespec.ValueOrRuntimeParameter_Constant: + inputs.ParameterValues[name] = runtimeValue.GetConstant() + // TODO(v2): clean up pipelinespec.Value usages case *pipelinespec.ValueOrRuntimeParameter_ConstantValue: inputs.Parameters[name] = runtimeValue.GetConstantValue() default: @@ -516,7 +479,7 @@ func resolveInputs(ctx context.Context, dag *metadata.DAG, task *pipelinespec.Pi func provisionOutputs(pipelineRoot, taskName string, outputsSpec *pipelinespec.ComponentOutputsSpec) *pipelinespec.ExecutorInput_Outputs { outputs := &pipelinespec.ExecutorInput_Outputs{ - Artifacts: make(map[string]*pipelinespec.ArtifactList), + Artifacts: make(map[string]*pipelinespec.ArtifactList), Parameters: make(map[string]*pipelinespec.ExecutorInput_OutputParameter), OutputFile: component.OutputMetadataFilepath, } diff --git a/v2/go.mod b/v2/go.mod index 882479c4a48..02f95fa4e3b 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -7,18 +7,20 @@ require ( github.com/aws/aws-sdk-go v1.36.1 github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b - github.com/golang/protobuf v1.5.0 + github.com/golang/protobuf v1.5.2 github.com/google/go-cmp v0.5.5 github.com/google/uuid v1.1.2 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 github.com/kubeflow/pipelines/api v0.0.0-20211020193552-20f28631517d github.com/stretchr/testify v1.7.0 gocloud.dev v0.22.0 - google.golang.org/genproto v0.0.0-20201203001206-6486ece9c497 - google.golang.org/grpc v1.36.0 + google.golang.org/genproto v0.0.0-20211026145609-4688e4c4e024 + google.golang.org/grpc v1.40.0 google.golang.org/protobuf v1.27.1 gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.20.4 k8s.io/apimachinery v0.21.2 k8s.io/client-go v0.20.4 ) + +replace github.com/kubeflow/pipelines/api => ../api diff --git a/v2/go.sum b/v2/go.sum index 6383d392a69..623eb83b0bc 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -198,6 +198,7 @@ github.com/cloudfoundry/jibber_jabber v0.0.0-20151120183258-bcc4c8345a21/go.mod github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/colinmarc/hdfs v1.1.4-0.20180802165501-48eb8d6c34a9/go.mod h1:0DumPviB681UcSuJErAbDIOx6SIaJWj463TymfZG02I= github.com/colinmarc/hdfs v1.1.4-0.20180805212432-9746310a4d31/go.mod h1:vSBumefK4HA5uiRSwNP+3ofgrEoScpCS2MMWcWXEuQ4= @@ -258,6 +259,7 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= @@ -451,8 +453,9 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0 h1:LUVKkCeviFUMKqHa4tXIIij/lbhnMbP7Fn5wKdKkRh4= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -642,8 +645,6 @@ github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kubeflow/pipelines/api v0.0.0-20211020193552-20f28631517d h1:Dga88zE1XJZUWDejk7qjZSXjKPYQvMRo8Y1CghdTwQ4= -github.com/kubeflow/pipelines/api v0.0.0-20211020193552-20f28631517d/go.mod h1:ItI8RjFTt0RY6X0g6B3VocSaphuE+DNuNTzAY9NF8EY= github.com/labstack/echo v3.2.1+incompatible/go.mod h1:0INS7j/VjnFxD4E2wkz67b8cVwCLbBmJyDaka6Cmk1s= github.com/labstack/gommon v0.2.7/go.mod h1:/tj9csK2iPSBvn+3NLM9e52usepMtrd5ilFYA+wQNJ4= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= @@ -938,6 +939,7 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -1073,8 +1075,9 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226101413-39120d07d75e/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1164,8 +1167,10 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201202213521-69691e467435/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210426230700-d19ff857e887 h1:dXfMednGJh/SUUFjTLsWJz3P+TQt9qnR11GgeI3vWKs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -1347,8 +1352,9 @@ google.golang.org/genproto v0.0.0-20200914193844-75d14daec038/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200921151605-7abf4a1a14d5/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201203001206-6486ece9c497 h1:jDYzwXmX9tLnuG4sL85HPmE1ruErXOopALp2i/0AHnI= google.golang.org/genproto v0.0.0-20201203001206-6486ece9c497/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20211026145609-4688e4c4e024 h1:aePO4E0x+Urj9V5NQHjqOpaNG4oMeHQq0l2ob05z5tI= +google.golang.org/genproto v0.0.0-20211026145609-4688e4c4e024/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1369,8 +1375,9 @@ google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc v1.36.0 h1:o1bcQ6imQMIOpdrO3SWf2z5RV72WbDwdXuK0MDlc8As= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.40.0 h1:AGJ0Ih4mHjSeibYkFGh1dD9KJ/eOtZ93I6hoHhukQ5Q= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc/examples v0.0.0-20201226181154-53788aa5dcb4/go.mod h1:Ly7ZA/ARzg8fnPU9TyZIxoz33sEUuWX7txiqs8lPTgE= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -1383,6 +1390,7 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= diff --git a/v2/metadata/client.go b/v2/metadata/client.go index d63fcc235a9..52e31005ddc 100644 --- a/v2/metadata/client.go +++ b/v2/metadata/client.go @@ -36,6 +36,7 @@ import ( "google.golang.org/grpc/status" "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/structpb" "gopkg.in/yaml.v2" ) @@ -98,41 +99,12 @@ func NewClient(serverAddress, serverPort string) (*Client, error) { }, nil } -// Parameters is used to represent input or output parameters (which are scalar -// values) from pipeline components. -type Parameters struct { - IntParameters map[string]int64 - StringParameters map[string]string - DoubleParameters map[string]float64 -} - -func NewParameters(params map[string]*pipelinespec.Value) (*Parameters, error) { - result := &Parameters{ - IntParameters: make(map[string]int64), - StringParameters: make(map[string]string), - DoubleParameters: make(map[string]float64), - } - for name, parameter := range params { - switch t := parameter.Value.(type) { - case *pipelinespec.Value_StringValue: - result.StringParameters[name] = parameter.GetStringValue() - case *pipelinespec.Value_IntValue: - result.IntParameters[name] = parameter.GetIntValue() - case *pipelinespec.Value_DoubleValue: - result.DoubleParameters[name] = parameter.GetDoubleValue() - default: - return nil, fmt.Errorf("failed to convert from map[string]*pipelinespec.Value to metadata.Parameters: unknown parameter type for parameter name=%q: %T", name, t) - } - } - return result, nil -} - // ExecutionConfig represents the input parameters and artifacts to an Execution. type ExecutionConfig struct { - InputParameters *Parameters + InputParameters map[string]*structpb.Value InputArtifactIDs map[string][]int64 TaskName, PodName, PodUID, Namespace, - Image, CachedMLMDExecutionID, ExecutionType , FingerPrint string + Image, CachedMLMDExecutionID, ExecutionType, FingerPrint string // a temporary flag to special case some logic for root DAG IsRootDAG bool } @@ -400,21 +372,21 @@ func getArtifactName(eventPath *pb.Event_Path) (string, error) { // PublishExecution publishes the specified execution with the given output // parameters, artifacts and state. -func (c *Client) PublishExecution(ctx context.Context, execution *Execution, outputParameters *Parameters, outputArtifacts []*OutputArtifact, state pb.Execution_State) error { +func (c *Client) PublishExecution(ctx context.Context, execution *Execution, outputParameters map[string]*structpb.Value, outputArtifacts []*OutputArtifact, state pb.Execution_State) error { e := execution.execution e.LastKnownState = state.Enum() if outputParameters != nil { // Record output parameters. - for n, p := range outputParameters.IntParameters { - e.CustomProperties["output:"+n] = intValue(p) - } - for n, p := range outputParameters.DoubleParameters { - e.CustomProperties["output:"+n] = doubleValue(p) + outputs := &pb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: make(map[string]*structpb.Value), + }, } - for n, p := range outputParameters.StringParameters { - e.CustomProperties["output:"+n] = stringValue(p) + for n, p := range outputParameters { + outputs.StructValue.Fields[n] = p } + e.CustomProperties[keyOutputs] = &pb.Value{Value: outputs} } contexts := []*pb.Context{} @@ -427,8 +399,7 @@ func (c *Client) PublishExecution(ctx context.Context, execution *Execution, out } for _, oa := range outputArtifacts { - aePair := &pb.PutExecutionRequest_ArtifactAndEvent{ - } + aePair := &pb.PutExecutionRequest_ArtifactAndEvent{} if oa.Artifact.GetId() == 0 { glog.Infof("the id of output artifact is not set, will create new artifact when publishing execution") aePair = &pb.PutExecutionRequest_ArtifactAndEvent{ @@ -456,16 +427,18 @@ func (c *Client) PublishExecution(ctx context.Context, execution *Execution, out // metadata keys const ( - keyDisplayName = "display_name" - keyTaskName = "task_name" - keyImage = "image" - keyPodName = "pod_name" - keyPodUID = "pod_uid" - keyNamespace = "namespace" - keyResourceName = "resource_name" - keyPipelineRoot = "pipeline_root" - keyCacheFingerPrint = "cache_fingerprint" + keyDisplayName = "display_name" + keyTaskName = "task_name" + keyImage = "image" + keyPodName = "pod_name" + keyPodUID = "pod_uid" + keyNamespace = "namespace" + keyResourceName = "resource_name" + keyPipelineRoot = "pipeline_root" + keyCacheFingerPrint = "cache_fingerprint" keyCachedExecutionID = "cached_execution_id" + keyInputs = "inputs" + keyOutputs = "outputs" ) // CreateExecution creates a new MLMD execution under the specified Pipeline. @@ -498,15 +471,15 @@ func (c *Client) CreateExecution(ctx context.Context, pipeline *Pipeline, config } if config.InputParameters != nil { - for k, v := range config.InputParameters.StringParameters { - e.CustomProperties["input:"+k] = stringValue(v) - } - for k, v := range config.InputParameters.IntParameters { - e.CustomProperties["input:"+k] = intValue(v) + inputs := &pb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: make(map[string]*structpb.Value), + }, } - for k, v := range config.InputParameters.DoubleParameters { - e.CustomProperties["input:"+k] = doubleValue(v) + for n, p := range config.InputParameters { + inputs.StructValue.Fields[n] = p } + e.CustomProperties[keyInputs] = &pb.Value{Value: inputs} } req := &pb.PutExecutionRequest{ @@ -984,11 +957,7 @@ func GenerateExecutionConfig(executorInput *pipelinespec.ExecutorInput) (*Execut } } - parameters, err := NewParameters(executorInput.Inputs.Parameters) - if err != nil { - return nil, err - } - ecfg.InputParameters = parameters + ecfg.InputParameters = executorInput.Inputs.ParameterValues return ecfg, nil } diff --git a/v2/metadata/converter.go b/v2/metadata/converter.go index f2a537f05ad..f939e798210 100644 --- a/v2/metadata/converter.go +++ b/v2/metadata/converter.go @@ -1,8 +1,10 @@ package metadata import ( + "encoding/json" "fmt" "strconv" + "strings" "github.com/kubeflow/pipelines/api/v2alpha1/go/pipelinespec" pb "github.com/kubeflow/pipelines/v2/third_party/ml_metadata" @@ -41,6 +43,87 @@ func IntValue(v int64) *pipelinespec.Value { } } +func PbValueToText(v *structpb.Value) (string, error) { + wrap := func(err error) error { + return fmt.Errorf("failed to convert structpb.Value to text: %w", err) + } + if v == nil { + return "", nil + } + var text string + switch t := v.Kind.(type) { + case *structpb.Value_StringValue: + text = v.GetStringValue() + case *structpb.Value_NumberValue: + text = strconv.FormatFloat(v.GetNumberValue(), 'f', -1, 64) + case *structpb.Value_BoolValue: + text = strconv.FormatBool(v.GetBoolValue()) + case *structpb.Value_ListValue: + b, err := json.Marshal(v.GetListValue()) + if err != nil { + return "", wrap(fmt.Errorf("failed to JSON-marshal a list: %w", err)) + } + text = string(b) + case *structpb.Value_StructValue: + b, err := json.Marshal(v.GetStructValue()) + if err != nil { + return "", wrap(fmt.Errorf("failed to JSON-marshal a struct: %w", err)) + } + text = string(b) + default: + return "", wrap(fmt.Errorf("unknown type %T", t)) + } + return text, nil +} + +func TextToPbValue(text string, t pipelinespec.ParameterType_ParameterTypeEnum) (*structpb.Value, error) { + msg := func(err error) error { + return fmt.Errorf("textToPbValue(text=%q, t=%q) failed: %w", text, t, err) + } + switch t { + case pipelinespec.ParameterType_STRING: + return structpb.NewStringValue(text), nil + case pipelinespec.ParameterType_NUMBER_INTEGER: + i, err := strconv.ParseInt(strings.TrimSpace(text), 10, 0) + if err != nil { + return nil, msg(err) + } + return structpb.NewNumberValue(float64(i)), nil + case pipelinespec.ParameterType_NUMBER_DOUBLE: + f, err := strconv.ParseFloat(strings.TrimSpace(text), 0) + if err != nil { + return nil, msg(err) + } + return structpb.NewNumberValue(f), nil + case pipelinespec.ParameterType_BOOLEAN: + v, err := strconv.ParseBool(strings.TrimSpace(text)) + if err != nil { + return nil, msg(err) + } + return structpb.NewBoolValue(v), nil + case pipelinespec.ParameterType_LIST: + v := &structpb.Value{} + if err := v.UnmarshalJSON([]byte(text)); err != nil { + return nil, msg(err) + } + if _, ok := v.GetKind().(*structpb.Value_ListValue); !ok { + return nil, msg(fmt.Errorf("unexpected type")) + } + return v, nil + case pipelinespec.ParameterType_STRUCT: + v := &structpb.Value{} + if err := v.UnmarshalJSON([]byte(text)); err != nil { + return nil, msg(err) + } + if _, ok := v.GetKind().(*structpb.Value_StructValue); !ok { + return nil, msg(fmt.Errorf("unexpected type")) + } + return v, nil + default: + return nil, msg(fmt.Errorf("unknown type. Expected STRING, NUMBER_INTEGER, NUMBER_DOUBLE, BOOLEAN, LIST or STRUCT")) + } +} + func pipelineSpecValueToMLMDValue(v *pipelinespec.Value) (*pb.Value, error) { switch t := v.Value.(type) { case *pipelinespec.Value_StringValue: @@ -173,5 +256,3 @@ func toRuntimeArtifact(artifact *pb.Artifact) (*pipelinespec.RuntimeArtifact, er return rta, nil } - - diff --git a/v2/metadata/model.go b/v2/metadata/model.go index 62fdd2ab854..cf4989ce0ee 100644 --- a/v2/metadata/model.go +++ b/v2/metadata/model.go @@ -18,19 +18,20 @@ package metadata import ( "fmt" - "strings" - "github.com/kubeflow/pipelines/api/v2alpha1/go/pipelinespec" + pb "github.com/kubeflow/pipelines/v2/third_party/ml_metadata" + "google.golang.org/protobuf/types/known/structpb" ) -const ( - prefixInput = "input:" - prefixOutput = "output:" -) +// A hacky way to get Execution from pb.Execution, usually you should get +// an Execution from this metadata package directly without using ml_metadata.Execution +func NewExecution(e *pb.Execution) *Execution { + return &Execution{execution: e} +} -func (e *Execution) GetParameters() (inputs, outputs map[string]*pipelinespec.Value, err error) { - inputs = make(map[string]*pipelinespec.Value) - outputs = make(map[string]*pipelinespec.Value) +func (e *Execution) GetParameters() (inputs, outputs map[string]*structpb.Value, err error) { + inputs = make(map[string]*structpb.Value) + outputs = make(map[string]*structpb.Value) defer func() { if err != nil { err = fmt.Errorf("execution(ID=%v).GetParameters failed: %w", e.GetID(), err) @@ -39,21 +40,14 @@ func (e *Execution) GetParameters() (inputs, outputs map[string]*pipelinespec.Va if e == nil || e.execution == nil { return nil, nil, nil } - for key, value := range e.execution.CustomProperties { - if strings.HasPrefix(key, prefixInput) { - name := strings.TrimPrefix(key, prefixInput) - kfpValue, err := mlmdValueToPipelineSpecValue(value) - if err != nil { - return nil, nil, err - } - inputs[name] = kfpValue - } else if strings.HasPrefix(key, prefixOutput) { - name := strings.TrimPrefix(key, prefixOutput) - kfpValue, err := mlmdValueToPipelineSpecValue(value) - if err != nil { - return nil, nil, err - } - outputs[name] = kfpValue + if stored_inputs, ok := e.execution.CustomProperties[keyInputs]; ok { + for name, value := range stored_inputs.GetStructValue().GetFields() { + inputs[name] = value + } + } + if stored_outputs, ok := e.execution.CustomProperties[keyOutputs]; ok { + for name, value := range stored_outputs.GetStructValue().GetFields() { + outputs[name] = value } } return inputs, outputs, nil diff --git a/v2/test/Makefile b/v2/test/Makefile index fb739dc6cc4..1aac77773e6 100644 --- a/v2/test/Makefile +++ b/v2/test/Makefile @@ -21,7 +21,7 @@ sample-test: context --host $(HOST) \ --gcs_root $(GCS_ROOT)/data \ --gcr_root $(GCR_ROOT) \ - --kfp_package_path $(KFP_PACKAGE_PATH) + --kfp_package_path "$(KFP_PACKAGE_PATH)" .PHONY: context context: