[Tour Of Beam] persistence_key for Pg::SaveSnippet

.github/workflows/tour_of_beam_examples_ci.yml

@@ -16,10 +16,10 @@
 name: Tour Of Beam Examples CI
 
 on:
-  push:
+  pull_request:
     paths:
-     - ./.github/workflows/playground_examples_ci_reusable.yml
-     - ./.github/workflows/tour_of_beam_examples_ci.yml
+     - .github/workflows/playground_examples_ci_reusable.yml
+     - .github/workflows/tour_of_beam_examples_ci.yml
      - playground/backend/**
      - playground/infrastructure/**
      - learning/tour-of-beam/learning-content/**

learning/tour-of-beam/backend/README.md

@@ -86,10 +86,12 @@ Prerequisites:
     * Billing API
     * Cloud Functions API
     * Firebase Admin API
+    * Secret Manager API
  - set environment variables:
    * PROJECT_ID: GCP id
    * REGION: the region, "us-central1" fe
  - existing setup of Playground backend in a project
+ - create a secret `PERSISTENCE_KEY_SALT` in Secret Manager, random string >100symbols long
 
 1. Deploy Datastore indexes (but don't delete existing Playground indexes!)
 ```
@@ -101,7 +103,9 @@ gcloud datastore indexes create ./internal/storage/index.yaml
 for endpoint in getSdkList getContentTree getUnitComplete getUserProgress postUnitComplete postUserCode; do
 gcloud functions deploy $endpoint --entry-point $endpoint \
   --region $REGION --runtime go116 --allow-unauthenticated \
-  --trigger-http --set-env-vars="DATASTORE_PROJECT_ID=$PROJECT_ID,GOOGLE_PROJECT_ID=$PROJECT_ID"
+  --trigger-http \
+  --set-env-vars="DATASTORE_PROJECT_ID=$PROJECT_ID,GOOGLE_PROJECT_ID=$PROJECT_ID" \
+  --set-secrets "PERSISTENCE_KEY_SALT=PERSISTENCE_KEY_SALT:latest"
 done
 
 ```

learning/tour-of-beam/backend/function.go

@@ -86,7 +86,7 @@ func MakePlaygroundClient(ctx context.Context) pb.PlaygroundServiceClient {
 	// * PLAYGROUND_ROUTER_HOST: playground API host/port
 	if os.Getenv("TOB_MOCK") > "" {
 		fmt.Println("Using mock playground client")
-		return pb.GetMockClient()
+		return service.GetMockClient()
 	} else {
 		host := os.Getenv("PLAYGROUND_ROUTER_HOST")
 		cc, err := grpc.Dial(host, grpc.WithTransportCredentials(insecure.NewCredentials()))

learning/tour-of-beam/backend/integration_tests/auth_emulator.go

@@ -113,10 +113,10 @@ func (e *EmulatorClient) do(method, endpoint string, jsonBody map[string]string)
 // Simulate Frontend client authorization logic
 // Here, we use the simplest possible authorization: email/password
 // Firebase Admin SDK lacks methods to create a user and get ID token
-func (e *EmulatorClient) getIDToken() string {
+func (e *EmulatorClient) getIDToken(email string) string {
 	// create a user (sign-up with dummy email/password)
 	endpoint := "identitytoolkit.googleapis.com/v1/accounts:signUp?key=anything_goes"
-	body := map[string]string{"email": "[email protected]", "password": "1q2w3e"}
+	body := map[string]string{"email": email, "password": "1q2w3e"}
 	resp, err := e.do(http.MethodPost, endpoint, body)
 	if err != nil {
 		log.Fatalf("emulator request error: %+v", err)

learning/tour-of-beam/backend/integration_tests/auth_test.go

@@ -16,11 +16,13 @@
 package main
 
 import (
+	"context"
 	"flag"
 	"net/http"
 	"os"
 	"path/filepath"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 )
@@ -59,7 +61,7 @@ func checkBadHttpCode(t *testing.T, err error, code int) {
 }
 
 func TestSaveGetProgress(t *testing.T) {
-	idToken := emulator.getIDToken()
+	idToken := emulator.getIDToken("[email protected]")
 
 	// postUnitCompleteURL
 	port := os.Getenv(PORT_POST_UNIT_COMPLETE)
@@ -134,6 +136,101 @@ func TestSaveGetProgress(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
+		assert.Equal(t, len(exp.Units), len(resp.Units))
+		assert.Equal(t, exp.Units[1].Id, resp.Units[1].Id)
+		// snippet_id is derived from random uid
+		exp.Units[1].UserSnippetId = resp.Units[1].UserSnippetId
 		assert.Equal(t, exp, resp)
 	})
 }
+
+func TestUserCode(t *testing.T) {
+	var snippetId1, snippetId2, snippetId3 string
+	idToken1 := emulator.getIDToken("[email protected]")
+	idToken2 := emulator.getIDToken("[email protected]")
+	req := makeUserCodeRequest()
+	originalCode := req.Files[0].Content
+
+	// postUserCodeURL
+	port := os.Getenv(PORT_POST_USER_CODE)
+	if port == "" {
+		t.Fatal(PORT_POST_USER_CODE, "env not set")
+	}
+	postUserCodeURL := "http://localhost:" + port
+
+	// getUserProgressURL
+	port = os.Getenv(PORT_GET_USER_PROGRESS)
+	if port == "" {
+		t.Fatal(PORT_GET_USER_PROGRESS, "env not set")
+	}
+	getUserProgressURL := "http://localhost:" + port
+
+	t.Run("save_code_user1_example1", func(t *testing.T) {
+		_, err := PostUserCode(postUserCodeURL, "python", "example1", idToken1, req)
+		if err != nil {
+			t.Fatal(err)
+		}
+	})
+	t.Run("save_code_user2_example1", func(t *testing.T) {
+		_, err := PostUserCode(postUserCodeURL, "python", "example1", idToken2, req)
+		if err != nil {
+			t.Fatal(err)
+		}
+	})
+	t.Run("check1", func(t *testing.T) {
+		resp, err := GetUserProgress(getUserProgressURL, "python", idToken1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		assert.Equal(t, "example1", resp.Units[0].Id)
+		snippetId1 = resp.Units[0].UserSnippetId
+	})
+	t.Run("check2", func(t *testing.T) {
+		resp, err := GetUserProgress(getUserProgressURL, "python", idToken2)
+		if err != nil {
+			t.Fatal(err)
+		}
+		assert.Equal(t, "example1", resp.Units[0].Id)
+		snippetId2 = resp.Units[0].UserSnippetId
+		assert.NotEqual(t, snippetId1, snippetId2, "different users, same snippet ids")
+	})
+	t.Run("save_code_user1_updated", func(t *testing.T) {
+		// modify snippet code
+		req.Files[0].Content += "; sys.exit(1)"
+
+		_, err := PostUserCode(postUserCodeURL, "python", "example1", idToken1, req)
+		if err != nil {
+			t.Fatal(err)
+		}
+	})
+	t.Run("check3", func(t *testing.T) {
+		resp, err := GetUserProgress(getUserProgressURL, "python", idToken1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		assert.Equal(t, "example1", resp.Units[0].Id)
+		snippetId3 = resp.Units[0].UserSnippetId
+		assert.NotEqual(t, snippetId1, snippetId3, "updated code, same snippet ids")
+	})
+	t.Run("check_snippet1", func(t *testing.T) {
+		ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)
+		_, err := GetSnippet(ctx, snippetId1)
+		assert.NotNil(t, err, "previous snippet available")
+
+		resp, err := GetSnippet(ctx, snippetId3)
+		assert.Nil(t, err)
+		assert.Equal(t, req.Files[0].Content, resp.Files[0].Content)
+		assert.Equal(t, req.Files[0].IsMain, resp.Files[0].IsMain)
+		assert.Equal(t, req.Files[0].Name, resp.Files[0].Name)
+		assert.Equal(t, req.PipelineOptions, resp.PipelineOptions)
+	})
+	t.Run("check_snippet2", func(t *testing.T) {
+		ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)
+		resp, err := GetSnippet(ctx, snippetId2)
+		assert.Nil(t, err)
+		assert.Equal(t, originalCode, resp.Files[0].Content)
+		assert.Equal(t, req.Files[0].IsMain, resp.Files[0].IsMain)
+		assert.Equal(t, req.Files[0].Name, resp.Files[0].Name)
+		assert.Equal(t, req.PipelineOptions, resp.PipelineOptions)
+	})
+}

learning/tour-of-beam/backend/integration_tests/client_pg.go

@@ -0,0 +1,39 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	pb "beam.apache.org/learning/tour-of-beam/backend/playground_api"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+func GetSnippet(ctx context.Context, snippetId string) (*pb.GetSnippetResponse, error) {
+	routerHost := os.Getenv("PLAYGROUND_ROUTER_HOST")
+	conn, err := grpc.DialContext(ctx, routerHost, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		return nil, fmt.Errorf("dial grpc: %w", err)
+	}
+	client := pb.NewPlaygroundServiceClient(conn)
+
+	req := &pb.GetSnippetRequest{Id: snippetId}
+	resp, err := client.GetSnippet(ctx, req)
+	if err != nil {
+		return nil, fmt.Errorf("get snippet: %w", err)
+	}
+	return resp, nil
+}

learning/tour-of-beam/backend/internal/service/content.go

@@ -82,7 +82,8 @@ func (s *Svc) SaveUserCode(ctx context.Context, sdk tob.Sdk, unitId, uid string,
 		return err
 	}
 
-	req := MakePgSaveRequest(userRequest, sdk)
+	persistence_key := makePersistenceKey(sdk, unitId, uid)
+	req := MakePgSaveRequest(userRequest, sdk, persistence_key)
 	resp, err := s.PgClient.SaveSnippet(ctx, &req)
 	if err != nil {
 		return err

learning/tour-of-beam/backend/playground_api/helper.go → learning/tour-of-beam/backend/internal/service/mock_helper.go

@@ -13,26 +13,28 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package playground
+package service
 
 import (
 	context "context"
 
+	pb "beam.apache.org/learning/tour-of-beam/backend/playground_api"
 	grpc "google.golang.org/grpc"
 )
 
-func GetMockClient() PlaygroundServiceClient {
+// Mock GRPC client which is enabled by TOB_MOCK env flag
+func GetMockClient() pb.PlaygroundServiceClient {
 
-	return &PlaygroundServiceClientMock{
-		SaveSnippetFunc: func(ctx context.Context, in *SaveSnippetRequest, opts ...grpc.CallOption) (*SaveSnippetResponse, error) {
-			return &SaveSnippetResponse{Id: "snippet_id_1"}, nil
+	return &pb.PlaygroundServiceClientMock{
+		SaveSnippetFunc: func(ctx context.Context, in *pb.SaveSnippetRequest, opts ...grpc.CallOption) (*pb.SaveSnippetResponse, error) {
+			return &pb.SaveSnippetResponse{Id: "snippet_id_1"}, nil
 		},
-		GetSnippetFunc: func(ctx context.Context, in *GetSnippetRequest, opts ...grpc.CallOption) (*GetSnippetResponse, error) {
-			return &GetSnippetResponse{
-				Files: []*SnippetFile{
+		GetSnippetFunc: func(ctx context.Context, in *pb.GetSnippetRequest, opts ...grpc.CallOption) (*pb.GetSnippetResponse, error) {
+			return &pb.GetSnippetResponse{
+				Files: []*pb.SnippetFile{
 					{Name: "main.py", Content: "import sys; sys.exit(0)", IsMain: true},
 				},
-				Sdk:             Sdk_SDK_PYTHON,
+				Sdk:             pb.Sdk_SDK_PYTHON,
 				PipelineOptions: "some opts",
 			}, nil
 		},

learning/tour-of-beam/backend/internal/service/persistence_key.go

@@ -0,0 +1,43 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package service
+
+import (
+	"crypto/sha256"
+	"encoding/base64"
+	"os"
+	"strings"
+
+	tob "beam.apache.org/learning/tour-of-beam/backend/internal"
+)
+
+func makePersistenceKey(sdk tob.Sdk, unitId, uid string) string {
+	h := sha256.New()
+	// never share!
+	plainKey := strings.Join(
+		[]string{os.Getenv("PERSISTENCE_KEY_SALT"), sdk.String(), unitId, uid},
+		"|")
+	_, err := h.Write([]byte(plainKey))
+	if err != nil {
+		panic(err)
+	}
+	raw := h.Sum(nil)
+
+	// base64 encode to pass as protobuf string
+	encoded := base64.URLEncoding.EncodeToString(raw)
+
+	return encoded
+}

learning/tour-of-beam/backend/internal/service/pg_adapter.go

@@ -22,7 +22,7 @@ import (
 	pb "beam.apache.org/learning/tour-of-beam/backend/playground_api"
 )
 
-func MakePgSaveRequest(userRequest tob.UserCodeRequest, sdk tob.Sdk) pb.SaveSnippetRequest {
+func MakePgSaveRequest(userRequest tob.UserCodeRequest, sdk tob.Sdk, persistence_key string) pb.SaveSnippetRequest {
 	filesProto := make([]*pb.SnippetFile, 0)
 	for _, file := range userRequest.Files {
 		filesProto = append(filesProto,
@@ -40,5 +40,6 @@ func MakePgSaveRequest(userRequest tob.UserCodeRequest, sdk tob.Sdk) pb.SaveSnip
 		Sdk:             pb.Sdk(sdkIdx),
 		Files:           filesProto,
 		PipelineOptions: userRequest.PipelineOptions,
+		PersistenceKey:  persistence_key,
 	}
 }

learning/tour-of-beam/backend/internal/storage/index.yaml

@@ -11,6 +11,10 @@ indexes:
   properties:
   - name: "id"
   - name: "type"
+- kind: "pg_snippets"
+  properties:
+  - name: "persistenceKey"
+  - name: "numberOfFiles"
 - kind: "tb_learning_module"
   ancestor: yes
   properties: