[Tour Of Beam] API adjustments (#23349)

* sdk * use sample/api * sdk_list.json * nits * title * fix integration_tests * unit/module id * sdks * unitId->id in param * id/title fix * empty * CORS * optimize * Update sdk.go * fixing format error Co-authored-by: oborysevych <[email protected]>
apache · Sep 28, 2022 · a393efa · a393efa
1 parent 91d79d9
commit a393efa
Show file tree

Hide file tree

Showing 24 changed files with 293 additions and 173 deletions.
diff --git a/.github/workflows/tour_of_beam_backend.yml b/.github/workflows/tour_of_beam_backend.yml
@@ -23,6 +23,7 @@ on:
   push:
     branches: ['master', 'release-*']
     tags: 'v*'
+    paths: ['learning/tour-of-beam/backend/**']
   pull_request:
     branches: ['master', 'release-*']
     tags: 'v*'

diff --git a/.github/workflows/tour_of_beam_backend_integration.yml b/.github/workflows/tour_of_beam_backend_integration.yml
@@ -23,6 +23,7 @@ on:
   push:
     branches: ['master', 'release-*']
     tags: 'v*'
+    paths: ['learning/tour-of-beam/backend/**']
   pull_request:
     branches: ['master', 'release-*']
     tags: 'v*'
@@ -73,8 +74,8 @@ jobs:
       # 2. start function-framework processes in BG
       - name: Compile CF
         run: go build -o ./tob_function cmd/main.go
-      - name: Run sdkList in background
-        run: PORT=${{ env.PORT_SDK_LIST }} FUNCTION_TARGET=sdkList ./tob_function &
+      - name: Run getSdkList in background
+        run: PORT=${{ env.PORT_SDK_LIST }} FUNCTION_TARGET=getSdkList ./tob_function &
       - name: Run getContentTree in background
         run: PORT=${{ env.PORT_GET_CONTENT_TREE }} FUNCTION_TARGET=getContentTree ./tob_function &
       - name: Run getUnitContent in background

diff --git a/learning/tour-of-beam/backend/README.md b/learning/tour-of-beam/backend/README.md
@@ -19,8 +19,8 @@ and currently logged-in user's snippets and progress.
 Currently it supports Java, Python, and Go Beam SDK.
 
 It is comprised of several Cloud Functions, with Firerstore in Datastore mode as a storage.
-* list-sdks
-* get-content-tree?sdk=(Java|Go|Python)
+* get-sdk-list
+* get-content-tree?sdk=(java|go|python)
 * get-unit-content?unitId=<id>
 TODO: add response schemas
 TODO: add save functions info

diff --git a/learning/tour-of-beam/backend/docker-compose.yml b/learning/tour-of-beam/backend/docker-compose.yml
@@ -25,4 +25,4 @@ services:
       - DATASTORE_LISTEN_ADDRESS=0.0.0.0:8081
     ports:
       - "8081:8081"
-    command: --consistency=1.0
+    command: --consistency=1.0 --store-on-disk
diff --git a/learning/tour-of-beam/backend/function.go b/learning/tour-of-beam/backend/function.go
@@ -20,7 +20,6 @@ package tob
 import (
 	"context"
 	"encoding/json"
-	"fmt"
 	"log"
 	"net/http"
 	"os"
@@ -38,54 +37,6 @@ const (
 	NOT_FOUND      = "NOT_FOUND"
 )
 
-// Middleware-maker for setting a header
-// We also make this less generic: it works with HandlerFunc's
-// so that to be convertible to func(w http ResponseWriter, r *http.Request)
-// and be accepted by functions.HTTP.
-func AddHeader(header, value string) func(http.HandlerFunc) http.HandlerFunc {
-	return func(next http.HandlerFunc) http.HandlerFunc {
-		return func(w http.ResponseWriter, r *http.Request) {
-			w.Header().Add(header, value)
-			next(w, r)
-		}
-	}
-}
-
-// Middleware to check http method.
-func EnsureMethod(method string) func(http.HandlerFunc) http.HandlerFunc {
-	return func(next http.HandlerFunc) http.HandlerFunc {
-		return func(w http.ResponseWriter, r *http.Request) {
-			if r.Method == method {
-				next(w, r)
-			} else {
-				w.WriteHeader(http.StatusMethodNotAllowed)
-			}
-		}
-	}
-}
-
-// HandleFunc enriched with sdk.
-type HandlerFuncWithSdk func(w http.ResponseWriter, r *http.Request, sdk tob.Sdk)
-
-// middleware to parse sdk query param and pass it as additional handler param.
-func ParseSdkParam(next HandlerFuncWithSdk) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		sdkStr := r.URL.Query().Get("sdk")
-		sdk := tob.ParseSdk(sdkStr)
-
-		if sdk == tob.SDK_UNDEFINED {
-			log.Printf("Bad sdk: %v", sdkStr)
-
-			message := fmt.Sprintf("Sdk not in: %v", tob.SdksList())
-			finalizeErrResponse(w, http.StatusBadRequest, BAD_FORMAT, message)
-
-			return
-		}
-
-		next(w, r, sdk)
-	}
-}
-
 // Helper to format http error messages.
 func finalizeErrResponse(w http.ResponseWriter, status int, code, message string) {
 	resp := tob.CodeMessage{Code: code, Message: message}
@@ -115,19 +66,23 @@ func init() {
 		svc = &service.Svc{Repo: &storage.DatastoreDb{Client: client}}
 	}
 
-	addHeader := AddHeader("Content-Type", "application/json")
-	ensureGet := EnsureMethod(http.MethodGet)
-
 	// functions framework
-	functions.HTTP("sdkList", ensureGet(addHeader(sdkList)))
-	functions.HTTP("getContentTree", ensureGet(addHeader(ParseSdkParam(getContentTree))))
-	functions.HTTP("getUnitContent", ensureGet(addHeader(ParseSdkParam(getUnitContent))))
+	functions.HTTP("getSdkList", Common(getSdkList))
+	functions.HTTP("getContentTree", Common(ParseSdkParam(getContentTree)))
+	functions.HTTP("getUnitContent", Common(ParseSdkParam(getUnitContent)))
 }
 
 // Get list of SDK names
 // Used in both representation and accessing content.
-func sdkList(w http.ResponseWriter, r *http.Request) {
-	fmt.Fprint(w, `{"names": ["Java", "Python", "Go"]}`)
+func getSdkList(w http.ResponseWriter, r *http.Request) {
+	sdks := tob.MakeSdkList()
+
+	err := json.NewEncoder(w).Encode(sdks)
+	if err != nil {
+		log.Println("Format sdk list error:", err)
+		finalizeErrResponse(w, http.StatusInternalServerError, INTERNAL_ERROR, "format sdk list")
+		return
+	}
 }
 
 // Get the content tree for a given SDK and user
@@ -155,7 +110,7 @@ func getContentTree(w http.ResponseWriter, r *http.Request, sdk tob.Sdk) {
 // description, hints, code snippets
 // Required to be wrapped into ParseSdkParam middleware.
 func getUnitContent(w http.ResponseWriter, r *http.Request, sdk tob.Sdk) {
-	unitId := r.URL.Query().Get("unitId")
+	unitId := r.URL.Query().Get("id")
 
 	unit, err := svc.GetUnitContent(r.Context(), sdk, unitId, nil /*TODO userId*/)
 	if err == service.ErrNoUnit {

diff --git a/learning/tour-of-beam/backend/integration_tests/api.go b/learning/tour-of-beam/backend/integration_tests/api.go
@@ -16,13 +16,18 @@ package main
 // * No hidden fields
 // * Internal enumerations: sdk, node.type to string params
 
-type sdkListResponse struct {
-	Names []string
+type SdkItem struct {
+	Id    string `json:"id"`
+	Title string `json:"title"`
+}
+
+type SdkList struct {
+	Sdks []SdkItem `json:"sdks"`
 }
 
 type Unit struct {
-	Id   string `json:"unitId"`
-	Name string `json:"name"`
+	Id    string `json:"id"`
+	Title string `json:"title"`
 
 	// optional
 	Description       string   `json:"description,omitempty"`
@@ -36,7 +41,7 @@ type Unit struct {
 }
 
 type Group struct {
-	Name  string `json:"name"`
+	Title string `json:"title"`
 	Nodes []Node `json:"nodes"`
 }
 
@@ -47,14 +52,14 @@ type Node struct {
 }
 
 type Module struct {
-	Id         string `json:"moduleId"`
-	Name       string `json:"name"`
+	Id         string `json:"id"`
+	Title      string `json:"title"`
 	Complexity string `json:"complexity"`
 	Nodes      []Node `json:"nodes"`
 }
 
 type ContentTree struct {
-	Sdk     string   `json:"sdk"`
+	Sdk     string   `json:"sdkId"`
 	Modules []Module `json:"modules"`
 }
 

diff --git a/learning/tour-of-beam/backend/integration_tests/client.go b/learning/tour-of-beam/backend/integration_tests/client.go
@@ -14,13 +14,31 @@ package main
 
 import (
 	"encoding/json"
+	"fmt"
 	"io"
 	"net/http"
 	"os"
 )
 
-func SdkList(url string) (sdkListResponse, error) {
-	var result sdkListResponse
+var (
+	ExpectedHeaders = map[string]string{
+		"Access-Control-Allow-Origin": "*",
+		"Content-Type":                "application/json",
+	}
+)
+
+func verifyHeaders(header http.Header) error {
+	for k, v := range ExpectedHeaders {
+		if actual := header.Get(k); actual != v {
+			return fmt.Errorf("header %s mismatch: %s (expected %s)", k, actual, v)
+		}
+	}
+
+	return nil
+}
+
+func GetSdkList(url string) (SdkList, error) {
+	var result SdkList
 	err := Get(&result, url, nil)
 	return result, err
 }
@@ -33,7 +51,7 @@ func GetContentTree(url, sdk string) (ContentTree, error) {
 
 func GetUnitContent(url, sdk, unitId string) (Unit, error) {
 	var result Unit
-	err := Get(&result, url, map[string]string{"sdk": sdk, "unitId": unitId})
+	err := Get(&result, url, map[string]string{"sdk": sdk, "id": unitId})
 	return result, err
 }
 
@@ -62,6 +80,10 @@ func Get(dst interface{}, url string, queryParams map[string]string) error {
 
 	defer resp.Body.Close()
 
+	if err := verifyHeaders(resp.Header); err != nil {
+		return err
+	}
+
 	tee := io.TeeReader(resp.Body, os.Stdout)
 	return json.NewDecoder(tee).Decode(dst)
 }
diff --git a/learning/tour-of-beam/backend/integration_tests/function_test.go b/learning/tour-of-beam/backend/integration_tests/function_test.go
@@ -57,11 +57,14 @@ func TestSdkList(t *testing.T) {
 		t.Fatal(PORT_SDK_LIST, "env not set")
 	}
 	url := "http://localhost:" + port
-	exp := sdkListResponse{
-		Names: []string{"Java", "Python", "Go"},
+
+	mock_path := filepath.Join("..", "samples", "api", "get_sdk_list.json")
+	var exp SdkList
+	if err := loadJson(mock_path, &exp); err != nil {
+		t.Fatal(err)
 	}
 
-	resp, err := SdkList(url)
+	resp, err := GetSdkList(url)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -81,7 +84,7 @@ func TestGetContentTree(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	resp, err := GetContentTree(url, "Python")
+	resp, err := GetContentTree(url, "python")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -101,7 +104,7 @@ func TestGetUnitContent(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	resp, err := GetUnitContent(url, "Python", "challenge1")
+	resp, err := GetUnitContent(url, "python", "challenge1")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -117,14 +120,14 @@ func TestNegative(t *testing.T) {
 		{PORT_GET_CONTENT_TREE, nil,
 			ErrorResponse{
 				Code:    "BAD_FORMAT",
-				Message: "Sdk not in: [Java Python Go SCIO]",
+				Message: "unknown sdk",
 			},
 		},
-		{PORT_GET_CONTENT_TREE, map[string]string{"sdk": "SCIO"},
+		{PORT_GET_CONTENT_TREE, map[string]string{"sdk": "scio"},
 			// TODO: actually here should be a NOT_FOUND error
 			ErrorResponse{Code: "INTERNAL_ERROR", Message: "storage error"},
 		},
-		{PORT_GET_UNIT_CONTENT, map[string]string{"sdk": "Python", "unitId": "unknown_unitId"},
+		{PORT_GET_UNIT_CONTENT, map[string]string{"sdk": "python", "unitId": "unknown_unitId"},
 			ErrorResponse{
 				Code:    "NOT_FOUND",
 				Message: "unit not found",

diff --git a/learning/tour-of-beam/backend/integration_tests/local.sh b/learning/tour-of-beam/backend/integration_tests/local.sh
@@ -27,7 +27,7 @@ docker-compose up -d
 
 go build -o tob_function cmd/main.go
 
-PORT=$PORT_SDK_LIST FUNCTION_TARGET=sdkList         ./tob_function &
+PORT=$PORT_SDK_LIST FUNCTION_TARGET=getSdkList         ./tob_function &
 PORT=$PORT_GET_CONTENT_TREE FUNCTION_TARGET=getContentTree  ./tob_function &
 PORT=$PORT_GET_UNIT_CONTENT FUNCTION_TARGET=getUnitContent  ./tob_function &
 
@@ -49,7 +49,7 @@ docker-compose down
 ls "$DATASTORE_EMULATOR_DATADIR"
 cat "$DATASTORE_EMULATOR_DATADIR/WEB-INF/index.yaml"
 
-diff -q "$DATASTORE_EMULATOR_DATADIR/WEB-INF/index.yaml" internal/storage/index.yaml || ( echo "index.yaml mismatch"; exit 1)
+diff "$DATASTORE_EMULATOR_DATADIR/WEB-INF/index.yaml" internal/storage/index.yaml || ( echo "index.yaml mismatch"; exit 1)
 
 
 rm -rf "$DATASTORE_EMULATOR_DATADIR"
diff --git a/learning/tour-of-beam/backend/internal/entity.go b/learning/tour-of-beam/backend/internal/entity.go
@@ -15,9 +15,18 @@
 
 package internal
 
+type SdkItem struct {
+	Id    string `json:"id"`
+	Title string `json:"title"`
+}
+
+type SdkList struct {
+	Sdks []SdkItem `json:"sdks"`
+}
+
 type Unit struct {
-	Id   string `json:"unitId"`
-	Name string `json:"name"`
+	Id    string `json:"id"`
+	Title string `json:"title"`
 
 	// optional
 	Description       string   `json:"description,omitempty"`
@@ -41,7 +50,7 @@ const (
 )
 
 type Group struct {
-	Name  string `json:"name"`
+	Title string `json:"title"`
 	Nodes []Node `json:"nodes"`
 }
 
@@ -52,14 +61,14 @@ type Node struct {
 }
 
 type Module struct {
-	Id         string `json:"moduleId"`
-	Name       string `json:"name"`
+	Id         string `json:"id"`
+	Title      string `json:"title"`
 	Complexity string `json:"complexity"`
 	Nodes      []Node `json:"nodes"`
 }
 
 type ContentTree struct {
-	Sdk     Sdk      `json:"sdk"`
+	Sdk     Sdk      `json:"sdkId"`
 	Modules []Module `json:"modules"`
 }
 

diff --git a/learning/tour-of-beam/backend/internal/fs_content/builders.go b/learning/tour-of-beam/backend/internal/fs_content/builders.go
@@ -26,7 +26,7 @@ type UnitBuilder struct {
 func NewUnitBuilder(info learningUnitInfo) UnitBuilder {
 	return UnitBuilder{tob.Unit{
 		Id:           info.Id,
-		Name:         info.Name,
+		Title:        info.Name,
 		TaskName:     info.TaskName,
 		SolutionName: info.SolutionName,
 	}}

diff --git a/learning/tour-of-beam/backend/internal/fs_content/load.go b/learning/tour-of-beam/backend/internal/fs_content/load.go
@@ -115,7 +115,7 @@ func collectUnit(infopath string, ids_watcher *idsWatcher) (unit *tob.Unit, err
 func collectGroup(infopath string, ids_watcher *idsWatcher) (*tob.Group, error) {
 	info := loadLearningGroupInfo(infopath)
 	log.Printf("Found Group %v metadata at %v\n", info.Name, infopath)
-	group := tob.Group{Name: info.Name}
+	group := tob.Group{Title: info.Name}
 	for _, item := range info.Content {
 		node, err := collectNode(filepath.Join(infopath, "..", item), ids_watcher)
 		if err != nil {
@@ -153,7 +153,7 @@ func collectModule(infopath string, ids_watcher *idsWatcher) (tob.Module, error)
 	info := loadLearningModuleInfo(infopath)
 	log.Printf("Found Module %v metadata at %v\n", info.Id, infopath)
 	ids_watcher.CheckId(info.Id)
-	module := tob.Module{Id: info.Id, Name: info.Name, Complexity: info.Complexity}
+	module := tob.Module{Id: info.Id, Title: info.Name, Complexity: info.Complexity}
 	for _, item := range info.Content {
 		node, err := collectNode(filepath.Join(infopath, "..", item), ids_watcher)
 		if err != nil {