From 609944253457b4c60b6f010ec355a6c189fbd858 Mon Sep 17 00:00:00 2001 From: Nicolau Leal Werneck Date: Sun, 24 Jul 2022 11:01:08 +0200 Subject: [PATCH 1/7] NoCredentials --- src/GoogleCloud.jl | 2 +- src/api/api.jl | 12 ++++++++---- src/credentials.jl | 6 +++++- src/session.jl | 2 ++ 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/GoogleCloud.jl b/src/GoogleCloud.jl index 0b77083..7a6c1f7 100644 --- a/src/GoogleCloud.jl +++ b/src/GoogleCloud.jl @@ -4,7 +4,7 @@ Google Cloud APIs module GoogleCloud export - JSONCredentials, MetadataCredentials, GoogleSession, authorize, + JSONCredentials, MetadataCredentials, NoCredentials, GoogleSession, authorize, set_session!, get_session export iam, storage, compute, container, pubsub, logging, datastore diff --git a/src/api/api.jl b/src/api/api.jl index 837c936..3a9e563 100644 --- a/src/api/api.jl +++ b/src/api/api.jl @@ -239,9 +239,13 @@ function execute(session::GoogleSession, resource::APIResource, method::APIMetho # obtain and use access token auth = authorize(session) - headers = Dict{String, String}( - "Authorization" => "$(auth[:token_type]) $(auth[:access_token])" - ) + headers = if isnothing(auth) + Dict{String, String}() + else + Dict{String, String}( + "Authorization" => "$(auth[:token_type]) $(auth[:access_token])" + ) + end params = Dict(params) # check if data provided when not expected @@ -273,7 +277,7 @@ function execute(session::GoogleSession, resource::APIResource, method::APIMetho # merge in default parameters and evaluate any expressions params = merge!(copy(method.default_params), Dict(params)) - extra = Dict(:project_id => session.credentials.project_id) + extra = hasproperty(session.credentials, :project_id) ? Dict(:project_id => session.credentials.project_id) : Dict() for (key, val) in params if isa(val, Symbol) params[key] = extra[val] diff --git a/src/credentials.jl b/src/credentials.jl index 5ddebe8..f90ca3f 100644 --- a/src/credentials.jl +++ b/src/credentials.jl @@ -3,7 +3,7 @@ Google Cloud Platform service-account API credentials. """ module credentials -export Credentials, JSONCredentials, MetadataCredentials +export Credentials, JSONCredentials, MetadataCredentials, NoCredentials import Base: show, print import JSON @@ -126,4 +126,8 @@ function print(io::IO, x::Credentials) end show(io::IO, x::JSONCredentials) = print(io, x) +struct NoCredentials <: Credentials end + +end + diff --git a/src/session.jl b/src/session.jl index 669ea7b..ae2e5ef 100644 --- a/src/session.jl +++ b/src/session.jl @@ -195,4 +195,6 @@ function authorize(session::GoogleSession; cache::Bool=true) authorization end +authorize(session::GoogleSession{T}) where {T <: NoCredentials} = nothing + end From d6f99ab99811e61e59961ec8e64e18af667c0849 Mon Sep 17 00:00:00 2001 From: Nicolau Leal Werneck Date: Sun, 24 Jul 2022 11:55:53 +0200 Subject: [PATCH 2/7] test for nocredentials --- test/nocredentials.jl | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 test/nocredentials.jl diff --git a/test/nocredentials.jl b/test/nocredentials.jl new file mode 100644 index 0000000..e359d54 --- /dev/null +++ b/test/nocredentials.jl @@ -0,0 +1,16 @@ +using Test + +using GoogleCloud +using JSON + +creds = NoCredentials() + +session = GoogleSession(creds, ["devstorage.full_control"]) +bucketName = "atari-replay-datasets" +fileList = GoogleCloud.storage(:Object, :list, bucketName; prefix="dqn/Pong/1/replay_logs/\$store\$_action", session=session) |> IOBuffer |> JSON.parse + +parities = map(fileList["items"]) do item + reduce(⊻, GoogleCloud.storage(:Object, :get, bucketName, item["name"], session=session)) +end + +@test reduce(⊻, fetch.(parities)) == 0xf1 From 5fd90dd5e992e61772a0d83d594fcf072f5c6f42 Mon Sep 17 00:00:00 2001 From: Nicolau Leal Werneck Date: Tue, 6 Jun 2023 22:52:53 +0200 Subject: [PATCH 3/7] rebasing on master and adding a testset statement --- test/nocredentials.jl | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/test/nocredentials.jl b/test/nocredentials.jl index e359d54..737ea6c 100644 --- a/test/nocredentials.jl +++ b/test/nocredentials.jl @@ -3,14 +3,16 @@ using Test using GoogleCloud using JSON -creds = NoCredentials() +@testset "NoCredentials" begin + creds = NoCredentials() -session = GoogleSession(creds, ["devstorage.full_control"]) -bucketName = "atari-replay-datasets" -fileList = GoogleCloud.storage(:Object, :list, bucketName; prefix="dqn/Pong/1/replay_logs/\$store\$_action", session=session) |> IOBuffer |> JSON.parse + session = GoogleSession(creds, ["devstorage.full_control"]) + bucketName = "atari-replay-datasets" + fileList = GoogleCloud.storage(:Object, :list, bucketName; prefix="dqn/Pong/1/replay_logs/\$store\$_action", session=session) |> IOBuffer |> JSON.parse -parities = map(fileList["items"]) do item - reduce(⊻, GoogleCloud.storage(:Object, :get, bucketName, item["name"], session=session)) -end + parities = map(fileList["items"]) do item + reduce(⊻, GoogleCloud.storage(:Object, :get, bucketName, item["name"], session=session)) + end -@test reduce(⊻, fetch.(parities)) == 0xf1 + @test reduce(⊻, fetch.(parities)) == 0xf1 +end From f9ec24945bde2482cc1b0673391fe7fbee7bda30 Mon Sep 17 00:00:00 2001 From: Nicolau Leal Werneck Date: Tue, 6 Jun 2023 22:56:34 +0200 Subject: [PATCH 4/7] adding nocredentials to runtests.jl --- test/runtests.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/test/runtests.jl b/test/runtests.jl index e2f88af..3f824dd 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2,4 +2,5 @@ using GoogleCloud using Test include("api.jl") +include("nocredentials.jl") #include("storage.jl") From bdbbd502f6f99313b82215e1a269cf9dec013e62 Mon Sep 17 00:00:00 2001 From: Nicolau Leal Werneck Date: Tue, 6 Jun 2023 23:05:54 +0200 Subject: [PATCH 5/7] testing intriguing error --- test/nocredentials.jl | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/test/nocredentials.jl b/test/nocredentials.jl index 737ea6c..b74c99a 100644 --- a/test/nocredentials.jl +++ b/test/nocredentials.jl @@ -3,16 +3,17 @@ using Test using GoogleCloud using JSON -@testset "NoCredentials" begin - creds = NoCredentials() +creds = NoCredentials() - session = GoogleSession(creds, ["devstorage.full_control"]) - bucketName = "atari-replay-datasets" - fileList = GoogleCloud.storage(:Object, :list, bucketName; prefix="dqn/Pong/1/replay_logs/\$store\$_action", session=session) |> IOBuffer |> JSON.parse +session = GoogleSession(creds, ["devstorage.full_control"]) +bucketName = "atari-replay-datasets" +@show mydata = GoogleCloud.storage(:Object, :list, bucketName; prefix="dqn/Pong/1/replay_logs/\$store\$_action", session=session) +fileList = JSON.parse(IOBuffer(mydata)) - parities = map(fileList["items"]) do item - reduce(⊻, GoogleCloud.storage(:Object, :get, bucketName, item["name"], session=session)) - end +parities = map(fileList["items"]) do item + reduce(⊻, GoogleCloud.storage(:Object, :get, bucketName, item["name"], session=session)) +end +@testset "NoCredentials" begin @test reduce(⊻, fetch.(parities)) == 0xf1 end From ea007068fc6e0378cf10ce5495e304da314fea43 Mon Sep 17 00:00:00 2001 From: Nicolau Leal Werneck Date: Tue, 6 Jun 2023 23:33:49 +0200 Subject: [PATCH 6/7] fixing test with new working json decoding --- test/nocredentials.jl | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/test/nocredentials.jl b/test/nocredentials.jl index b74c99a..07c29c6 100644 --- a/test/nocredentials.jl +++ b/test/nocredentials.jl @@ -7,13 +7,16 @@ creds = NoCredentials() session = GoogleSession(creds, ["devstorage.full_control"]) bucketName = "atari-replay-datasets" -@show mydata = GoogleCloud.storage(:Object, :list, bucketName; prefix="dqn/Pong/1/replay_logs/\$store\$_action", session=session) -fileList = JSON.parse(IOBuffer(mydata)) +prefix="dqn/Pong/1/replay_logs/\$store\$_action" -parities = map(fileList["items"]) do item - reduce(⊻, GoogleCloud.storage(:Object, :get, bucketName, item["name"], session=session)) +fileList = GoogleCloud.storage(:Object, :list, bucketName; prefix, session) + +parities = map(fileList) do item + reduce(⊻, GoogleCloud.storage(:Object, :get, bucketName, item[:name]; session)) end +EXPECTED_PARITY = 0xf1 + @testset "NoCredentials" begin - @test reduce(⊻, fetch.(parities)) == 0xf1 + @test reduce(⊻, fetch.(parities)) == EXPECTED_PARITY end From 72985515cf62641ce1fd508e252fc109c08c3e56 Mon Sep 17 00:00:00 2001 From: Nicolau Leal Werneck Date: Tue, 6 Jun 2023 23:57:49 +0200 Subject: [PATCH 7/7] added binary file download to test case of download without serialization --- test/nocredentials.jl | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/test/nocredentials.jl b/test/nocredentials.jl index 07c29c6..61f281e 100644 --- a/test/nocredentials.jl +++ b/test/nocredentials.jl @@ -6,6 +6,8 @@ using JSON creds = NoCredentials() session = GoogleSession(creds, ["devstorage.full_control"]) + +## Test download of JSON data from Pong dataset bucketName = "atari-replay-datasets" prefix="dqn/Pong/1/replay_logs/\$store\$_action" @@ -17,6 +19,13 @@ end EXPECTED_PARITY = 0xf1 -@testset "NoCredentials" begin +## Test download of binary data from sentinel dataset +sentinelbucket = "gcp-public-data-sentinel-2" +sentinelpath = "L2/tiles/32/T/NS/S2A_MSIL2A_20210506T102021_N0300_R065_T32TNS_20210506T132458.SAFE/GRANULE/L2A_T32TNS_A030664_20210506T102022/IMG_DATA/R20m/T32TNS_20210506T102021_B07_20m.jp2" +sentinelimage = GoogleCloud.storage(:Object, :get, sentinelbucket, sentinelpath; session) +EXPECTED_PARITY_SENTINEL = 0x6e + +@testset "storage access and NoCredentials" begin @test reduce(⊻, fetch.(parities)) == EXPECTED_PARITY + @test reduce(⊻, sentinelimage) == EXPECTED_PARITY_SENTINEL end