From af2a817bd39bfa8d32afbc77159ae5c0341b1350 Mon Sep 17 00:00:00 2001 From: liooooo <91940150+halalala222@users.noreply.github.com> Date: Wed, 14 Aug 2024 12:33:13 +0800 Subject: [PATCH] [ISSUE#653] Add Pagination Parameters to DAG List API to limit the response (#664) --- internal/client/client.go | 45 +- internal/client/client_test.go | 512 +++++++++++++++++- internal/client/interface.go | 8 + internal/frontend/dag/handler.go | 32 +- .../frontend/gen/models/list_dags_response.go | 17 + .../frontend/gen/models/list_tag_response.go | 88 +++ .../frontend/gen/restapi/embedded_spec.go | 154 +++++- .../operations/dags/delete_dag_parameters.go | 2 +- .../dags/get_dag_details_parameters.go | 2 +- .../operations/dags/list_dags_parameters.go | 132 +++++ .../operations/dags/list_dags_urlbuilder.go | 39 ++ .../gen/restapi/operations/dags/list_tags.go | 56 ++ .../operations/dags/list_tags_parameters.go | 46 ++ .../operations/dags/list_tags_responses.go | 118 ++++ .../operations/dags/list_tags_urlbuilder.go | 87 +++ .../dags/post_dag_action_parameters.go | 2 +- .../gen/restapi/operations/dagu_api.go | 12 + internal/frontend/gen/restapi/server.go | 8 +- internal/persistence/interface.go | 9 + internal/persistence/local/dag_store.go | 124 ++++- swagger.yaml | 52 ++ ui/package.json | 1 + .../components/molecules/CreateDAGButton.tsx | 8 +- ui/src/components/molecules/DAGPagination.tsx | 34 ++ ui/src/components/molecules/DAGTable.tsx | 80 +-- ui/src/models/api.ts | 6 + ui/src/pages/dags/dag/index.tsx | 4 +- ui/src/pages/dags/index.tsx | 58 +- ui/src/pages/search/index.tsx | 36 +- ui/yarn.lock | 5 + 30 files changed, 1681 insertions(+), 96 deletions(-) create mode 100644 internal/frontend/gen/models/list_tag_response.go create mode 100644 internal/frontend/gen/restapi/operations/dags/list_tags.go create mode 100644 internal/frontend/gen/restapi/operations/dags/list_tags_parameters.go create mode 100644 internal/frontend/gen/restapi/operations/dags/list_tags_responses.go create mode 100644 internal/frontend/gen/restapi/operations/dags/list_tags_urlbuilder.go create mode 100644 ui/src/components/molecules/DAGPagination.tsx diff --git a/internal/client/client.go b/internal/client/client.go index 150828ba6..fa2176a01 100644 --- a/internal/client/client.go +++ b/internal/client/client.go @@ -18,6 +18,7 @@ package client import ( "errors" "fmt" + "math" "os" "os/exec" "path/filepath" @@ -26,6 +27,7 @@ import ( "github.com/daguflow/dagu/internal/dag" "github.com/daguflow/dagu/internal/dag/scheduler" + "github.com/daguflow/dagu/internal/frontend/gen/restapi/operations/dags" "github.com/daguflow/dagu/internal/logger" "github.com/daguflow/dagu/internal/persistence" "github.com/daguflow/dagu/internal/persistence/model" @@ -268,10 +270,10 @@ func (e *client) GetAllStatus() ( statuses []*DAGStatus, errs []string, err error, ) { dagStore := e.dataStore.DAGStore() - dags, errs, err := dagStore.List() + dagList, errs, err := dagStore.List() var ret []*DAGStatus - for _, d := range dags { + for _, d := range dagList { status, err := e.readStatus(d) if err != nil { errs = append(errs, err.Error()) @@ -282,6 +284,41 @@ func (e *client) GetAllStatus() ( return ret, errs, err } +func (e *client) getPageCount(total int64, limit int64) int { + pageCount := int(math.Ceil(float64(total) / float64(limit))) + if pageCount == 0 { + pageCount = 1 + } + + return pageCount +} + +func (e *client) GetAllStatusPagination(params dags.ListDagsParams) ([]*DAGStatus, *DagListPaginationSummaryResult, error) { + var ( + dagListPaginationResult *persistence.DagListPaginationResult + err error + dagStore = e.dataStore.DAGStore() + dagStatusList = make([]*DAGStatus, 0) + currentStatus *DAGStatus + ) + + if dagListPaginationResult, err = dagStore.ListPagination(params); err != nil { + return dagStatusList, &DagListPaginationSummaryResult{PageCount: 1}, err + } + + for _, currentDag := range dagListPaginationResult.DagList { + if currentStatus, err = e.readStatus(currentDag); err != nil { + dagListPaginationResult.ErrorList = append(dagListPaginationResult.ErrorList, err.Error()) + } + dagStatusList = append(dagStatusList, currentStatus) + } + + return dagStatusList, &DagListPaginationSummaryResult{ + PageCount: e.getPageCount(dagListPaginationResult.Count, params.Limit), + ErrorList: dagListPaginationResult.ErrorList, + }, nil +} + func (e *client) getDAG(name string) (*dag.DAG, error) { dagStore := e.dataStore.DAGStore() dagDetail, err := dagStore.GetDetails(name) @@ -348,3 +385,7 @@ func escapeArg(input string) string { return escaped.String() } + +func (e *client) GetTagList() ([]string, []string, error) { + return e.dataStore.DAGStore().TagList() +} diff --git a/internal/client/client_test.go b/internal/client/client_test.go index b6d1a1e43..1ce70385e 100644 --- a/internal/client/client_test.go +++ b/internal/client/client_test.go @@ -16,19 +16,23 @@ package client_test import ( + "fmt" "net/http" "path/filepath" "testing" "time" + "github.com/go-openapi/swag" + "github.com/stretchr/testify/require" + "github.com/daguflow/dagu/internal/client" "github.com/daguflow/dagu/internal/dag" "github.com/daguflow/dagu/internal/dag/scheduler" + "github.com/daguflow/dagu/internal/frontend/gen/restapi/operations/dags" "github.com/daguflow/dagu/internal/persistence/model" "github.com/daguflow/dagu/internal/sock" "github.com/daguflow/dagu/internal/test" "github.com/daguflow/dagu/internal/util" - "github.com/stretchr/testify/require" ) var testdataDir = filepath.Join(util.MustGetwd(), "./testdata") @@ -372,6 +376,475 @@ func TestClient_ReadHistory(t *testing.T) { }) } +func TestClient_GetAllStatusPagination(t *testing.T) { + t.Run("TestClient_Empty", func(t *testing.T) { + setup := test.SetupTest(t) + defer setup.Cleanup() + + cli := setup.Client() + + _, result, err := cli.GetAllStatusPagination(dags.ListDagsParams{ + Limit: 10, + Page: 1, + }) + require.Equal(t, 1, result.PageCount) + require.NoError(t, err) + }) + + t.Run("TestClient_All", func(t *testing.T) { + setup := test.SetupTest(t) + defer setup.Cleanup() + + cli := setup.Client() + + // Create DAG List + + for i := 0; i < 20; i++ { + _, err := cli.CreateDAG("test-dag-pagination" + fmt.Sprintf("%d", i)) + require.NoError(t, err) + } + + // Get all statuses. + allDagStatus, result, err := cli.GetAllStatusPagination(dags.ListDagsParams{ + Limit: 10, + Page: 1, + }) + require.NoError(t, err) + require.Equal(t, 10, len(allDagStatus)) + require.Equal(t, 2, result.PageCount) + + allDagStatus, result, err = cli.GetAllStatusPagination(dags.ListDagsParams{ + Limit: 10, + Page: 2, + }) + require.NoError(t, err) + require.Equal(t, 10, len(allDagStatus)) + require.Equal(t, 2, result.PageCount) + + allDagStatus, result, err = cli.GetAllStatusPagination(dags.ListDagsParams{ + Limit: 10, + Page: 3, + }) + require.NoError(t, err) + require.Equal(t, 0, len(allDagStatus)) + require.Equal(t, 2, result.PageCount) + }) + + t.Run("TestClient_WithTags", func(t *testing.T) { + setup := test.SetupTest(t) + defer setup.Cleanup() + + cli := setup.Client() + + // Create DAG List + + for i := 0; i < 40; i++ { + spec := "" + id, err := cli.CreateDAG("test-dag-pagination" + fmt.Sprintf("%d", i)) + require.NoError(t, err) + if i%2 == 0 { + spec = "tags: tag1,tag2\nsteps:\n - name: step1\n command: echo hello\n" + } else { + spec = "tags: tag2,tag3\nsteps:\n - name: step1\n command: echo hello\n" + } + if err = cli.UpdateDAG(id, spec); err != nil { + t.Fatal(err) + } + + } + + // Get all statuses. + allDagStatus, result, err := cli.GetAllStatusPagination(dags.ListDagsParams{ + Limit: 10, + Page: 1, + SearchTag: swag.String("tag1"), + }) + require.NoError(t, err) + require.Equal(t, 10, len(allDagStatus)) + require.Equal(t, 2, result.PageCount) + + allDagStatus, result, err = cli.GetAllStatusPagination(dags.ListDagsParams{ + Limit: 10, + Page: 2, + SearchTag: swag.String("tag1"), + }) + require.NoError(t, err) + require.Equal(t, 10, len(allDagStatus)) + require.Equal(t, 2, result.PageCount) + + allDagStatus, result, err = cli.GetAllStatusPagination(dags.ListDagsParams{ + Limit: 10, + Page: 3, + SearchTag: swag.String("tag1"), + }) + require.NoError(t, err) + require.Equal(t, 0, len(allDagStatus)) + require.Equal(t, 2, result.PageCount) + + allDagStatus, result, err = cli.GetAllStatusPagination(dags.ListDagsParams{ + Limit: 10, + Page: 1, + SearchTag: swag.String("tag2"), + }) + require.NoError(t, err) + require.Equal(t, 10, len(allDagStatus)) + require.Equal(t, 4, result.PageCount) + + allDagStatus, result, err = cli.GetAllStatusPagination(dags.ListDagsParams{ + Limit: 10, + Page: 2, + SearchTag: swag.String("tag2"), + }) + require.NoError(t, err) + require.Equal(t, 10, len(allDagStatus)) + require.Equal(t, 4, result.PageCount) + + allDagStatus, result, err = cli.GetAllStatusPagination(dags.ListDagsParams{ + Limit: 10, + Page: 3, + SearchTag: swag.String("tag2"), + }) + require.NoError(t, err) + require.Equal(t, 10, len(allDagStatus)) + require.Equal(t, 4, result.PageCount) + + allDagStatus, result, err = cli.GetAllStatusPagination(dags.ListDagsParams{ + Limit: 10, + Page: 4, + SearchTag: swag.String("tag2"), + }) + require.NoError(t, err) + require.Equal(t, 10, len(allDagStatus)) + require.Equal(t, 4, result.PageCount) + + allDagStatus, result, err = cli.GetAllStatusPagination(dags.ListDagsParams{ + Limit: 10, + Page: 5, + SearchTag: swag.String("tag2"), + }) + require.NoError(t, err) + require.Equal(t, 0, len(allDagStatus)) + require.Equal(t, 4, result.PageCount) + + allDagStatus, result, err = cli.GetAllStatusPagination(dags.ListDagsParams{ + Limit: 10, + Page: 1, + SearchTag: swag.String("tag3"), + }) + require.NoError(t, err) + require.Equal(t, 10, len(allDagStatus)) + require.Equal(t, 2, result.PageCount) + + allDagStatus, result, err = cli.GetAllStatusPagination(dags.ListDagsParams{ + Limit: 10, + Page: 2, + SearchTag: swag.String("tag3"), + }) + require.NoError(t, err) + require.Equal(t, 10, len(allDagStatus)) + require.Equal(t, 2, result.PageCount) + + allDagStatus, result, err = cli.GetAllStatusPagination(dags.ListDagsParams{ + Limit: 10, + Page: 3, + SearchTag: swag.String("tag3"), + }) + require.NoError(t, err) + require.Equal(t, 0, len(allDagStatus)) + require.Equal(t, 2, result.PageCount) + + allDagStatus, result, err = cli.GetAllStatusPagination(dags.ListDagsParams{ + Limit: 10, + Page: 1, + SearchTag: swag.String("tag4"), + }) + require.NoError(t, err) + require.Equal(t, 0, len(allDagStatus)) + require.Equal(t, 1, result.PageCount) + }) + + t.Run("TestClient_WithName", func(t *testing.T) { + setup := test.SetupTest(t) + defer setup.Cleanup() + + cli := setup.Client() + + // Create DAG List + for i := 0; i < 40; i++ { + if i%2 == 0 { + _, err := cli.CreateDAG("1test-dag-pagination" + fmt.Sprintf("%d", i)) + require.NoError(t, err) + } else { + _, err := cli.CreateDAG("2test-dag-pagination" + fmt.Sprintf("%d", i)) + require.NoError(t, err) + } + } + + // Get all statuses. + allDagStatus, result, err := cli.GetAllStatusPagination(dags.ListDagsParams{ + Limit: 10, + Page: 1, + SearchName: swag.String("1test-dag-pagination"), + }) + require.NoError(t, err) + require.Equal(t, 10, len(allDagStatus)) + require.Equal(t, 2, result.PageCount) + + allDagStatus, result, err = cli.GetAllStatusPagination(dags.ListDagsParams{ + Limit: 10, + Page: 2, + SearchName: swag.String("1test-dag-pagination"), + }) + require.NoError(t, err) + require.Equal(t, 10, len(allDagStatus)) + require.Equal(t, 2, result.PageCount) + + allDagStatus, result, err = cli.GetAllStatusPagination(dags.ListDagsParams{ + Limit: 10, + Page: 3, + SearchName: swag.String("1test-dag-pagination"), + }) + require.NoError(t, err) + require.Equal(t, 0, len(allDagStatus)) + require.Equal(t, 2, result.PageCount) + + allDagStatus, result, err = cli.GetAllStatusPagination(dags.ListDagsParams{ + Limit: 10, + Page: 1, + SearchName: swag.String("2test-dag-pagination"), + }) + require.NoError(t, err) + require.Equal(t, 10, len(allDagStatus)) + require.Equal(t, 2, result.PageCount) + + allDagStatus, result, err = cli.GetAllStatusPagination(dags.ListDagsParams{ + Limit: 10, + Page: 2, + SearchName: swag.String("2test-dag-pagination"), + }) + require.NoError(t, err) + require.Equal(t, 10, len(allDagStatus)) + require.Equal(t, 2, result.PageCount) + + allDagStatus, result, err = cli.GetAllStatusPagination(dags.ListDagsParams{ + Limit: 10, + Page: 3, + SearchName: swag.String("2test-dag-pagination"), + }) + require.NoError(t, err) + require.Equal(t, 0, len(allDagStatus)) + require.Equal(t, 2, result.PageCount) + + allDagStatus, result, err = cli.GetAllStatusPagination(dags.ListDagsParams{ + Limit: 10, + Page: 1, + SearchName: swag.String("test-dag-pagination"), + }) + require.NoError(t, err) + require.Equal(t, 10, len(allDagStatus)) + require.Equal(t, 4, result.PageCount) + + allDagStatus, result, err = cli.GetAllStatusPagination(dags.ListDagsParams{ + Limit: 10, + Page: 2, + SearchName: swag.String("test-dag-pagination"), + }) + require.NoError(t, err) + require.Equal(t, 10, len(allDagStatus)) + require.Equal(t, 4, result.PageCount) + + allDagStatus, result, err = cli.GetAllStatusPagination(dags.ListDagsParams{ + Limit: 10, + Page: 3, + SearchName: swag.String("test-dag-pagination"), + }) + require.NoError(t, err) + require.Equal(t, 10, len(allDagStatus)) + require.Equal(t, 4, result.PageCount) + + allDagStatus, result, err = cli.GetAllStatusPagination(dags.ListDagsParams{ + Limit: 10, + Page: 4, + SearchName: swag.String("test-dag-pagination"), + }) + require.NoError(t, err) + require.Equal(t, 10, len(allDagStatus)) + require.Equal(t, 4, result.PageCount) + + allDagStatus, result, err = cli.GetAllStatusPagination(dags.ListDagsParams{ + Limit: 10, + Page: 5, + SearchName: swag.String("test-dag-pagination"), + }) + require.NoError(t, err) + require.Equal(t, 0, len(allDagStatus)) + require.Equal(t, 4, result.PageCount) + + allDagStatus, result, err = cli.GetAllStatusPagination(dags.ListDagsParams{ + Limit: 10, + Page: 1, + SearchName: swag.String("not-exist"), + }) + require.NoError(t, err) + require.Equal(t, 0, len(allDagStatus)) + require.Equal(t, 1, result.PageCount) + }) + + t.Run("TestClient_WithTagsAndName", func(t *testing.T) { + setup := test.SetupTest(t) + defer setup.Cleanup() + + cli := setup.Client() + + // Create DAG List + for i := 0; i < 40; i++ { + spec := "" + id, err := cli.CreateDAG("1test-dag-pagination" + fmt.Sprintf("%d", i)) + require.NoError(t, err) + if i%2 == 0 { + spec = "tags: tag1,tag2\nsteps:\n - name: step1\n command: echo hello\n" + } else { + spec = "tags: tag2,tag3\nsteps:\n - name: step1\n command: echo hello\n" + } + if err = cli.UpdateDAG(id, spec); err != nil { + t.Fatal(err) + } + + } + + // Get all statuses. + allDagStatus, result, err := cli.GetAllStatusPagination(dags.ListDagsParams{ + Limit: 10, + Page: 1, + SearchName: swag.String("1test-dag-pagination"), + SearchTag: swag.String("tag1"), + }) + require.NoError(t, err) + require.Equal(t, 10, len(allDagStatus)) + require.Equal(t, 2, result.PageCount) + + allDagStatus, result, err = cli.GetAllStatusPagination(dags.ListDagsParams{ + Limit: 10, + Page: 2, + SearchName: swag.String("1test-dag-pagination"), + SearchTag: swag.String("tag1"), + }) + require.NoError(t, err) + require.Equal(t, 10, len(allDagStatus)) + require.Equal(t, 2, result.PageCount) + + allDagStatus, result, err = cli.GetAllStatusPagination(dags.ListDagsParams{ + Limit: 10, + Page: 3, + SearchName: swag.String("1test-dag-pagination"), + SearchTag: swag.String("tag1"), + }) + require.NoError(t, err) + require.Equal(t, 0, len(allDagStatus)) + require.Equal(t, 2, result.PageCount) + + allDagStatus, result, err = cli.GetAllStatusPagination(dags.ListDagsParams{ + Limit: 10, + Page: 1, + SearchName: swag.String("1test-dag-pagination"), + SearchTag: swag.String("tag2"), + }) + require.NoError(t, err) + require.Equal(t, 10, len(allDagStatus)) + require.Equal(t, 4, result.PageCount) + + allDagStatus, result, err = cli.GetAllStatusPagination(dags.ListDagsParams{ + Limit: 10, + Page: 2, + SearchName: swag.String("1test-dag-pagination"), + SearchTag: swag.String("tag2"), + }) + require.NoError(t, err) + require.Equal(t, 10, len(allDagStatus)) + require.Equal(t, 4, result.PageCount) + + allDagStatus, result, err = cli.GetAllStatusPagination(dags.ListDagsParams{ + Limit: 10, + Page: 3, + SearchName: swag.String("1test-dag-pagination"), + SearchTag: swag.String("tag2"), + }) + require.NoError(t, err) + require.Equal(t, 10, len(allDagStatus)) + require.Equal(t, 4, result.PageCount) + + allDagStatus, result, err = cli.GetAllStatusPagination(dags.ListDagsParams{ + Limit: 10, + Page: 4, + SearchName: swag.String("1test-dag-pagination"), + SearchTag: swag.String("tag2"), + }) + require.NoError(t, err) + require.Equal(t, 10, len(allDagStatus)) + require.Equal(t, 4, result.PageCount) + + allDagStatus, result, err = cli.GetAllStatusPagination(dags.ListDagsParams{ + Limit: 10, + Page: 5, + SearchName: swag.String("1test-dag-pagination"), + SearchTag: swag.String("tag2"), + }) + require.NoError(t, err) + require.Equal(t, 0, len(allDagStatus)) + require.Equal(t, 4, result.PageCount) + + allDagStatus, result, err = cli.GetAllStatusPagination(dags.ListDagsParams{ + Limit: 10, + Page: 1, + SearchName: swag.String("1test-dag-pagination"), + SearchTag: swag.String("tag3"), + }) + require.NoError(t, err) + require.Equal(t, 10, len(allDagStatus)) + require.Equal(t, 2, result.PageCount) + + allDagStatus, result, err = cli.GetAllStatusPagination(dags.ListDagsParams{ + Limit: 10, + Page: 2, + SearchName: swag.String("1test-dag-pagination"), + SearchTag: swag.String("tag3"), + }) + require.NoError(t, err) + require.Equal(t, 10, len(allDagStatus)) + require.Equal(t, 2, result.PageCount) + + allDagStatus, result, err = cli.GetAllStatusPagination(dags.ListDagsParams{ + Limit: 10, + Page: 3, + SearchName: swag.String("1test-dag-pagination"), + SearchTag: swag.String("tag3"), + }) + require.NoError(t, err) + require.Equal(t, 0, len(allDagStatus)) + require.Equal(t, 2, result.PageCount) + + allDagStatus, result, err = cli.GetAllStatusPagination(dags.ListDagsParams{ + Limit: 10, + Page: 1, + SearchName: swag.String("not-exist"), + SearchTag: swag.String("tag1"), + }) + require.NoError(t, err) + require.Equal(t, 0, len(allDagStatus)) + require.Equal(t, 1, result.PageCount) + + allDagStatus, result, err = cli.GetAllStatusPagination(dags.ListDagsParams{ + Limit: 10, + Page: 1, + SearchName: swag.String("1test-dag-pagination"), + SearchTag: swag.String("not-exist"), + }) + require.NoError(t, err) + require.Equal(t, 0, len(allDagStatus)) + require.Equal(t, 1, result.PageCount) + }) +} + func testDAG(name string) string { return filepath.Join(testdataDir, name) } @@ -393,3 +866,40 @@ func testNewStatus(workflow *dag.DAG, requestID string, status scheduler.Status, ret.RequestID = requestID return ret } + +func TestClient_GetTagList(t *testing.T) { + setup := test.SetupTest(t) + defer setup.Cleanup() + + cli := setup.Client() + + // Create DAG List + for i := 0; i < 40; i++ { + spec := "" + id, err := cli.CreateDAG("1test-dag-pagination" + fmt.Sprintf("%d", i)) + require.NoError(t, err) + if i%2 == 0 { + spec = "tags: tag1,tag2\nsteps:\n - name: step1\n command: echo hello\n" + } else { + spec = "tags: tag2,tag3\nsteps:\n - name: step1\n command: echo hello\n" + } + if err = cli.UpdateDAG(id, spec); err != nil { + t.Fatal(err) + } + + } + + tags, errs, err := cli.GetTagList() + require.NoError(t, err) + require.Equal(t, 0, len(errs)) + require.Equal(t, 3, len(tags)) + + mapTags := make(map[string]bool) + for _, tag := range tags { + mapTags[tag] = true + } + + require.True(t, mapTags["tag1"]) + require.True(t, mapTags["tag2"]) + require.True(t, mapTags["tag3"]) +} diff --git a/internal/client/interface.go b/internal/client/interface.go index 9aaa95e3e..e35a308b6 100644 --- a/internal/client/interface.go +++ b/internal/client/interface.go @@ -19,6 +19,7 @@ import ( "path/filepath" "github.com/daguflow/dagu/internal/dag" + "github.com/daguflow/dagu/internal/frontend/gen/restapi/operations/dags" "github.com/daguflow/dagu/internal/persistence" "github.com/daguflow/dagu/internal/persistence/model" ) @@ -41,9 +42,11 @@ type Client interface { UpdateDAG(id string, spec string) error DeleteDAG(id, loc string) error GetAllStatus() (statuses []*DAGStatus, errs []string, err error) + GetAllStatusPagination(params dags.ListDagsParams) ([]*DAGStatus, *DagListPaginationSummaryResult, error) GetStatus(dagLocation string) (*DAGStatus, error) IsSuspended(id string) bool ToggleSuspend(id string, suspend bool) error + GetTagList() ([]string, []string, error) } type StartOptions struct { @@ -65,6 +68,11 @@ type DAGStatus struct { ErrorT *string } +type DagListPaginationSummaryResult struct { + PageCount int + ErrorList []string +} + func newDAGStatus( workflow *dag.DAG, s *model.Status, suspended bool, err error, ) *DAGStatus { diff --git a/internal/frontend/dag/handler.go b/internal/frontend/dag/handler.go index 05ff0910a..260b46446 100644 --- a/internal/frontend/dag/handler.go +++ b/internal/frontend/dag/handler.go @@ -133,6 +133,16 @@ func (h *Handler) Configure(api *operations.DaguAPI) { } return dags.NewSearchDagsOK().WithPayload(resp) }) + + api.DagsListTagsHandler = dags.ListTagsHandlerFunc( + func(params dags.ListTagsParams) middleware.Responder { + tags, err := h.getTagList(params) + if err != nil { + return dags.NewListTagsDefault(err.Code). + WithPayload(err.APIError) + } + return dags.NewListTagsOK().WithPayload(tags) + }) } func (h *Handler) createDAG( @@ -167,13 +177,13 @@ func (h *Handler) deleteDAG(params dags.DeleteDagParams) *codedError { return nil } -func (h *Handler) getList(_ dags.ListDagsParams) (*models.ListDagsResponse, *codedError) { - dgs, errs, err := h.client.GetAllStatus() +func (h *Handler) getList(params dags.ListDagsParams) (*models.ListDagsResponse, *codedError) { + dgs, result, err := h.client.GetAllStatusPagination(params) if err != nil { return nil, newInternalError(err) } - hasErr := len(errs) > 0 + hasErr := len(result.ErrorList) > 0 if !hasErr { // Check if any DAG has an error for _, d := range dgs { @@ -185,8 +195,9 @@ func (h *Handler) getList(_ dags.ListDagsParams) (*models.ListDagsResponse, *cod } resp := &models.ListDagsResponse{ - Errors: errs, - HasError: swag.Bool(hasErr), + Errors: result.ErrorList, + PageCount: swag.Int64(int64(result.PageCount)), + HasError: swag.Bool(hasErr), } for _, dagStatus := range dgs { @@ -766,3 +777,14 @@ func readFileContent(f string, decoder *encoding.Decoder) ([]byte, error) { ret, err := io.ReadAll(tr) return ret, err } + +func (h *Handler) getTagList(_ dags.ListTagsParams) (*models.ListTagResponse, *codedError) { + tags, errs, err := h.client.GetTagList() + if err != nil { + return nil, newInternalError(err) + } + return &models.ListTagResponse{ + Errors: errs, + Tags: tags, + }, nil +} diff --git a/internal/frontend/gen/models/list_dags_response.go b/internal/frontend/gen/models/list_dags_response.go index db0b52a17..38b0c28e6 100644 --- a/internal/frontend/gen/models/list_dags_response.go +++ b/internal/frontend/gen/models/list_dags_response.go @@ -31,6 +31,10 @@ type ListDagsResponse struct { // has error // Required: true HasError *bool `json:"HasError"` + + // page count + // Required: true + PageCount *int64 `json:"PageCount"` } // Validate validates this list dags response @@ -49,6 +53,10 @@ func (m *ListDagsResponse) Validate(formats strfmt.Registry) error { res = append(res, err) } + if err := m.validatePageCount(formats); err != nil { + res = append(res, err) + } + if len(res) > 0 { return errors.CompositeValidationError(res...) } @@ -100,6 +108,15 @@ func (m *ListDagsResponse) validateHasError(formats strfmt.Registry) error { return nil } +func (m *ListDagsResponse) validatePageCount(formats strfmt.Registry) error { + + if err := validate.Required("PageCount", "body", m.PageCount); err != nil { + return err + } + + return nil +} + // ContextValidate validate this list dags response based on the context it is used func (m *ListDagsResponse) ContextValidate(ctx context.Context, formats strfmt.Registry) error { var res []error diff --git a/internal/frontend/gen/models/list_tag_response.go b/internal/frontend/gen/models/list_tag_response.go new file mode 100644 index 000000000..a0f074ea3 --- /dev/null +++ b/internal/frontend/gen/models/list_tag_response.go @@ -0,0 +1,88 @@ +// Code generated by go-swagger; DO NOT EDIT. + +package models + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "context" + + "github.com/go-openapi/errors" + "github.com/go-openapi/strfmt" + "github.com/go-openapi/swag" + "github.com/go-openapi/validate" +) + +// ListTagResponse list tag response +// +// swagger:model listTagResponse +type ListTagResponse struct { + + // errors + // Required: true + Errors []string `json:"Errors"` + + // tags + // Required: true + Tags []string `json:"Tags"` +} + +// Validate validates this list tag response +func (m *ListTagResponse) Validate(formats strfmt.Registry) error { + var res []error + + if err := m.validateErrors(formats); err != nil { + res = append(res, err) + } + + if err := m.validateTags(formats); err != nil { + res = append(res, err) + } + + if len(res) > 0 { + return errors.CompositeValidationError(res...) + } + return nil +} + +func (m *ListTagResponse) validateErrors(formats strfmt.Registry) error { + + if err := validate.Required("Errors", "body", m.Errors); err != nil { + return err + } + + return nil +} + +func (m *ListTagResponse) validateTags(formats strfmt.Registry) error { + + if err := validate.Required("Tags", "body", m.Tags); err != nil { + return err + } + + return nil +} + +// ContextValidate validates this list tag response based on context it is used +func (m *ListTagResponse) ContextValidate(ctx context.Context, formats strfmt.Registry) error { + return nil +} + +// MarshalBinary interface implementation +func (m *ListTagResponse) MarshalBinary() ([]byte, error) { + if m == nil { + return nil, nil + } + return swag.WriteJSON(m) +} + +// UnmarshalBinary interface implementation +func (m *ListTagResponse) UnmarshalBinary(b []byte) error { + var res ListTagResponse + if err := swag.ReadJSON(b, &res); err != nil { + return err + } + *m = res + return nil +} diff --git a/internal/frontend/gen/restapi/embedded_spec.go b/internal/frontend/gen/restapi/embedded_spec.go index c5d2dda37..64195d47f 100644 --- a/internal/frontend/gen/restapi/embedded_spec.go +++ b/internal/frontend/gen/restapi/embedded_spec.go @@ -49,6 +49,30 @@ func init() { "dags" ], "operationId": "listDags", + "parameters": [ + { + "type": "integer", + "name": "page", + "in": "query", + "required": true + }, + { + "type": "integer", + "name": "limit", + "in": "query", + "required": true + }, + { + "type": "string", + "name": "searchName", + "in": "query" + }, + { + "type": "string", + "name": "searchTag", + "in": "query" + } + ], "responses": { "200": { "description": "A successful response.", @@ -290,6 +314,32 @@ func init() { } } } + }, + "/tags": { + "get": { + "description": "Returns a list of tags.", + "produces": [ + "application/json" + ], + "tags": [ + "dags" + ], + "operationId": "listTags", + "responses": { + "200": { + "description": "A successful response.", + "schema": { + "$ref": "#/definitions/listTagResponse" + } + }, + "default": { + "description": "Generic error response.", + "schema": { + "$ref": "#/definitions/ApiError" + } + } + } + } } }, "definitions": { @@ -801,7 +851,8 @@ func init() { "required": [ "DAGs", "Errors", - "HasError" + "HasError", + "PageCount" ], "properties": { "DAGs": { @@ -818,6 +869,30 @@ func init() { }, "HasError": { "type": "boolean" + }, + "PageCount": { + "type": "integer" + } + } + }, + "listTagResponse": { + "type": "object", + "required": [ + "Tags", + "Errors" + ], + "properties": { + "Errors": { + "type": "array", + "items": { + "type": "string" + } + }, + "Tags": { + "type": "array", + "items": { + "type": "string" + } } } }, @@ -1071,6 +1146,30 @@ func init() { "dags" ], "operationId": "listDags", + "parameters": [ + { + "type": "integer", + "name": "page", + "in": "query", + "required": true + }, + { + "type": "integer", + "name": "limit", + "in": "query", + "required": true + }, + { + "type": "string", + "name": "searchName", + "in": "query" + }, + { + "type": "string", + "name": "searchTag", + "in": "query" + } + ], "responses": { "200": { "description": "A successful response.", @@ -1312,6 +1411,32 @@ func init() { } } } + }, + "/tags": { + "get": { + "description": "Returns a list of tags.", + "produces": [ + "application/json" + ], + "tags": [ + "dags" + ], + "operationId": "listTags", + "responses": { + "200": { + "description": "A successful response.", + "schema": { + "$ref": "#/definitions/listTagResponse" + } + }, + "default": { + "description": "Generic error response.", + "schema": { + "$ref": "#/definitions/ApiError" + } + } + } + } } }, "definitions": { @@ -1823,7 +1948,8 @@ func init() { "required": [ "DAGs", "Errors", - "HasError" + "HasError", + "PageCount" ], "properties": { "DAGs": { @@ -1840,6 +1966,30 @@ func init() { }, "HasError": { "type": "boolean" + }, + "PageCount": { + "type": "integer" + } + } + }, + "listTagResponse": { + "type": "object", + "required": [ + "Tags", + "Errors" + ], + "properties": { + "Errors": { + "type": "array", + "items": { + "type": "string" + } + }, + "Tags": { + "type": "array", + "items": { + "type": "string" + } } } }, diff --git a/internal/frontend/gen/restapi/operations/dags/delete_dag_parameters.go b/internal/frontend/gen/restapi/operations/dags/delete_dag_parameters.go index a102dcda4..0e40f3f06 100644 --- a/internal/frontend/gen/restapi/operations/dags/delete_dag_parameters.go +++ b/internal/frontend/gen/restapi/operations/dags/delete_dag_parameters.go @@ -56,7 +56,7 @@ func (o *DeleteDagParams) BindRequest(r *http.Request, route *middleware.Matched return nil } -// bindDagID binds and validates parameter DagID from filepath. +// bindDagID binds and validates parameter DagID from path. func (o *DeleteDagParams) bindDagID(rawData []string, hasKey bool, formats strfmt.Registry) error { var raw string if len(rawData) > 0 { diff --git a/internal/frontend/gen/restapi/operations/dags/get_dag_details_parameters.go b/internal/frontend/gen/restapi/operations/dags/get_dag_details_parameters.go index 42cdd5102..20d6f398b 100644 --- a/internal/frontend/gen/restapi/operations/dags/get_dag_details_parameters.go +++ b/internal/frontend/gen/restapi/operations/dags/get_dag_details_parameters.go @@ -86,7 +86,7 @@ func (o *GetDagDetailsParams) BindRequest(r *http.Request, route *middleware.Mat return nil } -// bindDagID binds and validates parameter DagID from filepath. +// bindDagID binds and validates parameter DagID from path. func (o *GetDagDetailsParams) bindDagID(rawData []string, hasKey bool, formats strfmt.Registry) error { var raw string if len(rawData) > 0 { diff --git a/internal/frontend/gen/restapi/operations/dags/list_dags_parameters.go b/internal/frontend/gen/restapi/operations/dags/list_dags_parameters.go index 7175a3e98..3a102b9c3 100644 --- a/internal/frontend/gen/restapi/operations/dags/list_dags_parameters.go +++ b/internal/frontend/gen/restapi/operations/dags/list_dags_parameters.go @@ -9,7 +9,11 @@ import ( "net/http" "github.com/go-openapi/errors" + "github.com/go-openapi/runtime" "github.com/go-openapi/runtime/middleware" + "github.com/go-openapi/strfmt" + "github.com/go-openapi/swag" + "github.com/go-openapi/validate" ) // NewListDagsParams creates a new ListDagsParams object @@ -28,6 +32,25 @@ type ListDagsParams struct { // HTTP Request Object HTTPRequest *http.Request `json:"-"` + + /* + Required: true + In: query + */ + Limit int64 + /* + Required: true + In: query + */ + Page int64 + /* + In: query + */ + SearchName *string + /* + In: query + */ + SearchTag *string } // BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface @@ -39,8 +62,117 @@ func (o *ListDagsParams) BindRequest(r *http.Request, route *middleware.MatchedR o.HTTPRequest = r + qs := runtime.Values(r.URL.Query()) + + qLimit, qhkLimit, _ := qs.GetOK("limit") + if err := o.bindLimit(qLimit, qhkLimit, route.Formats); err != nil { + res = append(res, err) + } + + qPage, qhkPage, _ := qs.GetOK("page") + if err := o.bindPage(qPage, qhkPage, route.Formats); err != nil { + res = append(res, err) + } + + qSearchName, qhkSearchName, _ := qs.GetOK("searchName") + if err := o.bindSearchName(qSearchName, qhkSearchName, route.Formats); err != nil { + res = append(res, err) + } + + qSearchTag, qhkSearchTag, _ := qs.GetOK("searchTag") + if err := o.bindSearchTag(qSearchTag, qhkSearchTag, route.Formats); err != nil { + res = append(res, err) + } if len(res) > 0 { return errors.CompositeValidationError(res...) } return nil } + +// bindLimit binds and validates parameter Limit from query. +func (o *ListDagsParams) bindLimit(rawData []string, hasKey bool, formats strfmt.Registry) error { + if !hasKey { + return errors.Required("limit", "query", rawData) + } + var raw string + if len(rawData) > 0 { + raw = rawData[len(rawData)-1] + } + + // Required: true + // AllowEmptyValue: false + + if err := validate.RequiredString("limit", "query", raw); err != nil { + return err + } + + value, err := swag.ConvertInt64(raw) + if err != nil { + return errors.InvalidType("limit", "query", "int64", raw) + } + o.Limit = value + + return nil +} + +// bindPage binds and validates parameter Page from query. +func (o *ListDagsParams) bindPage(rawData []string, hasKey bool, formats strfmt.Registry) error { + if !hasKey { + return errors.Required("page", "query", rawData) + } + var raw string + if len(rawData) > 0 { + raw = rawData[len(rawData)-1] + } + + // Required: true + // AllowEmptyValue: false + + if err := validate.RequiredString("page", "query", raw); err != nil { + return err + } + + value, err := swag.ConvertInt64(raw) + if err != nil { + return errors.InvalidType("page", "query", "int64", raw) + } + o.Page = value + + return nil +} + +// bindSearchName binds and validates parameter SearchName from query. +func (o *ListDagsParams) bindSearchName(rawData []string, hasKey bool, formats strfmt.Registry) error { + var raw string + if len(rawData) > 0 { + raw = rawData[len(rawData)-1] + } + + // Required: false + // AllowEmptyValue: false + + if raw == "" { // empty values pass all other validations + return nil + } + o.SearchName = &raw + + return nil +} + +// bindSearchTag binds and validates parameter SearchTag from query. +func (o *ListDagsParams) bindSearchTag(rawData []string, hasKey bool, formats strfmt.Registry) error { + var raw string + if len(rawData) > 0 { + raw = rawData[len(rawData)-1] + } + + // Required: false + // AllowEmptyValue: false + + if raw == "" { // empty values pass all other validations + return nil + } + o.SearchTag = &raw + + return nil +} diff --git a/internal/frontend/gen/restapi/operations/dags/list_dags_urlbuilder.go b/internal/frontend/gen/restapi/operations/dags/list_dags_urlbuilder.go index eda320eb4..1ce7bdee0 100644 --- a/internal/frontend/gen/restapi/operations/dags/list_dags_urlbuilder.go +++ b/internal/frontend/gen/restapi/operations/dags/list_dags_urlbuilder.go @@ -9,11 +9,20 @@ import ( "errors" "net/url" golangswaggerpaths "path" + + "github.com/go-openapi/swag" ) // ListDagsURL generates an URL for the list dags operation type ListDagsURL struct { + Limit int64 + Page int64 + SearchName *string + SearchTag *string + _basePath string + // avoid unkeyed usage + _ struct{} } // WithBasePath sets the base path for this url builder, only required when it's different from the @@ -43,6 +52,36 @@ func (o *ListDagsURL) Build() (*url.URL, error) { } _result.Path = golangswaggerpaths.Join(_basePath, _path) + qs := make(url.Values) + + limitQ := swag.FormatInt64(o.Limit) + if limitQ != "" { + qs.Set("limit", limitQ) + } + + pageQ := swag.FormatInt64(o.Page) + if pageQ != "" { + qs.Set("page", pageQ) + } + + var searchNameQ string + if o.SearchName != nil { + searchNameQ = *o.SearchName + } + if searchNameQ != "" { + qs.Set("searchName", searchNameQ) + } + + var searchTagQ string + if o.SearchTag != nil { + searchTagQ = *o.SearchTag + } + if searchTagQ != "" { + qs.Set("searchTag", searchTagQ) + } + + _result.RawQuery = qs.Encode() + return &_result, nil } diff --git a/internal/frontend/gen/restapi/operations/dags/list_tags.go b/internal/frontend/gen/restapi/operations/dags/list_tags.go new file mode 100644 index 000000000..267ba0531 --- /dev/null +++ b/internal/frontend/gen/restapi/operations/dags/list_tags.go @@ -0,0 +1,56 @@ +// Code generated by go-swagger; DO NOT EDIT. + +package dags + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the generate command + +import ( + "net/http" + + "github.com/go-openapi/runtime/middleware" +) + +// ListTagsHandlerFunc turns a function with the right signature into a list tags handler +type ListTagsHandlerFunc func(ListTagsParams) middleware.Responder + +// Handle executing the request and returning a response +func (fn ListTagsHandlerFunc) Handle(params ListTagsParams) middleware.Responder { + return fn(params) +} + +// ListTagsHandler interface for that can handle valid list tags params +type ListTagsHandler interface { + Handle(ListTagsParams) middleware.Responder +} + +// NewListTags creates a new http.Handler for the list tags operation +func NewListTags(ctx *middleware.Context, handler ListTagsHandler) *ListTags { + return &ListTags{Context: ctx, Handler: handler} +} + +/* + ListTags swagger:route GET /tags dags listTags + +Returns a list of tags. +*/ +type ListTags struct { + Context *middleware.Context + Handler ListTagsHandler +} + +func (o *ListTags) ServeHTTP(rw http.ResponseWriter, r *http.Request) { + route, rCtx, _ := o.Context.RouteInfo(r) + if rCtx != nil { + *r = *rCtx + } + var Params = NewListTagsParams() + if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params + o.Context.Respond(rw, r, route.Produces, route, err) + return + } + + res := o.Handler.Handle(Params) // actually handle the request + o.Context.Respond(rw, r, route.Produces, route, res) + +} diff --git a/internal/frontend/gen/restapi/operations/dags/list_tags_parameters.go b/internal/frontend/gen/restapi/operations/dags/list_tags_parameters.go new file mode 100644 index 000000000..491253583 --- /dev/null +++ b/internal/frontend/gen/restapi/operations/dags/list_tags_parameters.go @@ -0,0 +1,46 @@ +// Code generated by go-swagger; DO NOT EDIT. + +package dags + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "net/http" + + "github.com/go-openapi/errors" + "github.com/go-openapi/runtime/middleware" +) + +// NewListTagsParams creates a new ListTagsParams object +// +// There are no default values defined in the spec. +func NewListTagsParams() ListTagsParams { + + return ListTagsParams{} +} + +// ListTagsParams contains all the bound params for the list tags operation +// typically these are obtained from a http.Request +// +// swagger:parameters listTags +type ListTagsParams struct { + + // HTTP Request Object + HTTPRequest *http.Request `json:"-"` +} + +// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface +// for simple values it will use straight method calls. +// +// To ensure default values, the struct must have been initialized with NewListTagsParams() beforehand. +func (o *ListTagsParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { + var res []error + + o.HTTPRequest = r + + if len(res) > 0 { + return errors.CompositeValidationError(res...) + } + return nil +} diff --git a/internal/frontend/gen/restapi/operations/dags/list_tags_responses.go b/internal/frontend/gen/restapi/operations/dags/list_tags_responses.go new file mode 100644 index 000000000..f1d101706 --- /dev/null +++ b/internal/frontend/gen/restapi/operations/dags/list_tags_responses.go @@ -0,0 +1,118 @@ +// Code generated by go-swagger; DO NOT EDIT. + +package dags + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "net/http" + + "github.com/go-openapi/runtime" + + "github.com/daguflow/dagu/internal/frontend/gen/models" +) + +// ListTagsOKCode is the HTTP code returned for type ListTagsOK +const ListTagsOKCode int = 200 + +/* +ListTagsOK A successful response. + +swagger:response listTagsOK +*/ +type ListTagsOK struct { + + /* + In: Body + */ + Payload *models.ListTagResponse `json:"body,omitempty"` +} + +// NewListTagsOK creates ListTagsOK with default headers values +func NewListTagsOK() *ListTagsOK { + + return &ListTagsOK{} +} + +// WithPayload adds the payload to the list tags o k response +func (o *ListTagsOK) WithPayload(payload *models.ListTagResponse) *ListTagsOK { + o.Payload = payload + return o +} + +// SetPayload sets the payload to the list tags o k response +func (o *ListTagsOK) SetPayload(payload *models.ListTagResponse) { + o.Payload = payload +} + +// WriteResponse to the client +func (o *ListTagsOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { + + rw.WriteHeader(200) + if o.Payload != nil { + payload := o.Payload + if err := producer.Produce(rw, payload); err != nil { + panic(err) // let the recovery middleware deal with this + } + } +} + +/* +ListTagsDefault Generic error response. + +swagger:response listTagsDefault +*/ +type ListTagsDefault struct { + _statusCode int + + /* + In: Body + */ + Payload *models.APIError `json:"body,omitempty"` +} + +// NewListTagsDefault creates ListTagsDefault with default headers values +func NewListTagsDefault(code int) *ListTagsDefault { + if code <= 0 { + code = 500 + } + + return &ListTagsDefault{ + _statusCode: code, + } +} + +// WithStatusCode adds the status to the list tags default response +func (o *ListTagsDefault) WithStatusCode(code int) *ListTagsDefault { + o._statusCode = code + return o +} + +// SetStatusCode sets the status to the list tags default response +func (o *ListTagsDefault) SetStatusCode(code int) { + o._statusCode = code +} + +// WithPayload adds the payload to the list tags default response +func (o *ListTagsDefault) WithPayload(payload *models.APIError) *ListTagsDefault { + o.Payload = payload + return o +} + +// SetPayload sets the payload to the list tags default response +func (o *ListTagsDefault) SetPayload(payload *models.APIError) { + o.Payload = payload +} + +// WriteResponse to the client +func (o *ListTagsDefault) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { + + rw.WriteHeader(o._statusCode) + if o.Payload != nil { + payload := o.Payload + if err := producer.Produce(rw, payload); err != nil { + panic(err) // let the recovery middleware deal with this + } + } +} diff --git a/internal/frontend/gen/restapi/operations/dags/list_tags_urlbuilder.go b/internal/frontend/gen/restapi/operations/dags/list_tags_urlbuilder.go new file mode 100644 index 000000000..835d94cd1 --- /dev/null +++ b/internal/frontend/gen/restapi/operations/dags/list_tags_urlbuilder.go @@ -0,0 +1,87 @@ +// Code generated by go-swagger; DO NOT EDIT. + +package dags + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the generate command + +import ( + "errors" + "net/url" + golangswaggerpaths "path" +) + +// ListTagsURL generates an URL for the list tags operation +type ListTagsURL struct { + _basePath string +} + +// WithBasePath sets the base path for this url builder, only required when it's different from the +// base path specified in the swagger spec. +// When the value of the base path is an empty string +func (o *ListTagsURL) WithBasePath(bp string) *ListTagsURL { + o.SetBasePath(bp) + return o +} + +// SetBasePath sets the base path for this url builder, only required when it's different from the +// base path specified in the swagger spec. +// When the value of the base path is an empty string +func (o *ListTagsURL) SetBasePath(bp string) { + o._basePath = bp +} + +// Build a url path and query string +func (o *ListTagsURL) Build() (*url.URL, error) { + var _result url.URL + + var _path = "/tags" + + _basePath := o._basePath + if _basePath == "" { + _basePath = "/api/v1" + } + _result.Path = golangswaggerpaths.Join(_basePath, _path) + + return &_result, nil +} + +// Must is a helper function to panic when the url builder returns an error +func (o *ListTagsURL) Must(u *url.URL, err error) *url.URL { + if err != nil { + panic(err) + } + if u == nil { + panic("url can't be nil") + } + return u +} + +// String returns the string representation of the path with query string +func (o *ListTagsURL) String() string { + return o.Must(o.Build()).String() +} + +// BuildFull builds a full url with scheme, host, path and query string +func (o *ListTagsURL) BuildFull(scheme, host string) (*url.URL, error) { + if scheme == "" { + return nil, errors.New("scheme is required for a full url on ListTagsURL") + } + if host == "" { + return nil, errors.New("host is required for a full url on ListTagsURL") + } + + base, err := o.Build() + if err != nil { + return nil, err + } + + base.Scheme = scheme + base.Host = host + return base, nil +} + +// StringFull returns the string representation of a complete url +func (o *ListTagsURL) StringFull(scheme, host string) string { + return o.Must(o.BuildFull(scheme, host)).String() +} diff --git a/internal/frontend/gen/restapi/operations/dags/post_dag_action_parameters.go b/internal/frontend/gen/restapi/operations/dags/post_dag_action_parameters.go index 08432746a..284fb244e 100644 --- a/internal/frontend/gen/restapi/operations/dags/post_dag_action_parameters.go +++ b/internal/frontend/gen/restapi/operations/dags/post_dag_action_parameters.go @@ -84,7 +84,7 @@ func (o *PostDagActionParams) BindRequest(r *http.Request, route *middleware.Mat return nil } -// bindDagID binds and validates parameter DagID from filepath. +// bindDagID binds and validates parameter DagID from path. func (o *PostDagActionParams) bindDagID(rawData []string, hasKey bool, formats strfmt.Registry) error { var raw string if len(rawData) > 0 { diff --git a/internal/frontend/gen/restapi/operations/dagu_api.go b/internal/frontend/gen/restapi/operations/dagu_api.go index 774fe3d0e..92d636e2e 100644 --- a/internal/frontend/gen/restapi/operations/dagu_api.go +++ b/internal/frontend/gen/restapi/operations/dagu_api.go @@ -56,6 +56,9 @@ func NewDaguAPI(spec *loads.Document) *DaguAPI { DagsListDagsHandler: dags.ListDagsHandlerFunc(func(params dags.ListDagsParams) middleware.Responder { return middleware.NotImplemented("operation dags.ListDags has not yet been implemented") }), + DagsListTagsHandler: dags.ListTagsHandlerFunc(func(params dags.ListTagsParams) middleware.Responder { + return middleware.NotImplemented("operation dags.ListTags has not yet been implemented") + }), DagsPostDagActionHandler: dags.PostDagActionHandlerFunc(func(params dags.PostDagActionParams) middleware.Responder { return middleware.NotImplemented("operation dags.PostDagAction has not yet been implemented") }), @@ -109,6 +112,8 @@ type DaguAPI struct { DagsGetDagDetailsHandler dags.GetDagDetailsHandler // DagsListDagsHandler sets the operation handler for the list dags operation DagsListDagsHandler dags.ListDagsHandler + // DagsListTagsHandler sets the operation handler for the list tags operation + DagsListTagsHandler dags.ListTagsHandler // DagsPostDagActionHandler sets the operation handler for the post dag action operation DagsPostDagActionHandler dags.PostDagActionHandler // DagsSearchDagsHandler sets the operation handler for the search dags operation @@ -202,6 +207,9 @@ func (o *DaguAPI) Validate() error { if o.DagsListDagsHandler == nil { unregistered = append(unregistered, "dags.ListDagsHandler") } + if o.DagsListTagsHandler == nil { + unregistered = append(unregistered, "dags.ListTagsHandler") + } if o.DagsPostDagActionHandler == nil { unregistered = append(unregistered, "dags.PostDagActionHandler") } @@ -312,6 +320,10 @@ func (o *DaguAPI) initHandlerCache() { o.handlers["GET"] = make(map[string]http.Handler) } o.handlers["GET"]["/dags"] = dags.NewListDags(o.context, o.DagsListDagsHandler) + if o.handlers["GET"] == nil { + o.handlers["GET"] = make(map[string]http.Handler) + } + o.handlers["GET"]["/tags"] = dags.NewListTags(o.context, o.DagsListTagsHandler) if o.handlers["POST"] == nil { o.handlers["POST"] = make(map[string]http.Handler) } diff --git a/internal/frontend/gen/restapi/server.go b/internal/frontend/gen/restapi/server.go index 35adb360b..63e5e546a 100644 --- a/internal/frontend/gen/restapi/server.go +++ b/internal/frontend/gen/restapi/server.go @@ -215,13 +215,13 @@ func (s *Server) Serve() (err error) { servers = append(servers, httpServer) wg.Add(1) - s.Logf("Serving frontend at http://%s", s.httpServerL.Addr()) + s.Logf("Serving dagu at http://%s", s.httpServerL.Addr()) go func(l net.Listener) { defer wg.Done() if err := httpServer.Serve(l); err != nil && err != http.ErrServerClosed { s.Fatalf("%v", err) } - s.Logf("Stopped serving frontend at http://%s", l.Addr()) + s.Logf("Stopped serving dagu at http://%s", l.Addr()) }(s.httpServerL) } @@ -308,13 +308,13 @@ func (s *Server) Serve() (err error) { servers = append(servers, httpsServer) wg.Add(1) - s.Logf("Serving frontend at https://%s", s.httpsServerL.Addr()) + s.Logf("Serving dagu at https://%s", s.httpsServerL.Addr()) go func(l net.Listener) { defer wg.Done() if err := httpsServer.Serve(l); err != nil && err != http.ErrServerClosed { s.Fatalf("%v", err) } - s.Logf("Stopped serving frontend at https://%s", l.Addr()) + s.Logf("Stopped serving dagu at https://%s", l.Addr()) }(tls.NewListener(s.httpsServerL, httpsServer.TLSConfig)) } diff --git a/internal/persistence/interface.go b/internal/persistence/interface.go index b591f1e7c..b52b49b2f 100644 --- a/internal/persistence/interface.go +++ b/internal/persistence/interface.go @@ -20,6 +20,7 @@ import ( "time" "github.com/daguflow/dagu/internal/dag" + "github.com/daguflow/dagu/internal/frontend/gen/restapi/operations/dags" "github.com/daguflow/dagu/internal/persistence/grep" "github.com/daguflow/dagu/internal/persistence/model" ) @@ -53,6 +54,7 @@ type DAGStore interface { Create(name string, spec []byte) (string, error) Delete(name string) error List() (ret []*dag.DAG, errs []string, err error) + ListPagination(params dags.ListDagsParams) (*DagListPaginationResult, error) GetMetadata(name string) (*dag.DAG, error) GetDetails(name string) (*dag.DAG, error) Grep(pattern string) (ret []*GrepResult, errs []string, err error) @@ -61,6 +63,7 @@ type DAGStore interface { GetSpec(name string) (string, error) UpdateSpec(name string, spec []byte) error Find(name string) (*dag.DAG, error) + TagList() ([]string, []string, error) } type GrepResult struct { @@ -69,6 +72,12 @@ type GrepResult struct { Matches []*grep.Match } +type DagListPaginationResult struct { + DagList []*dag.DAG + Count int64 + ErrorList []string +} + type FlagStore interface { ToggleSuspend(id string, suspend bool) error IsSuspended(id string) bool diff --git a/internal/persistence/local/dag_store.go b/internal/persistence/local/dag_store.go index 158bc75b0..e74a72a27 100644 --- a/internal/persistence/local/dag_store.go +++ b/internal/persistence/local/dag_store.go @@ -18,16 +18,17 @@ package local import ( "errors" "fmt" + "io/fs" "os" "path" "path/filepath" "strings" "time" + "github.com/daguflow/dagu/internal/dag" + "github.com/daguflow/dagu/internal/frontend/gen/restapi/operations/dags" "github.com/daguflow/dagu/internal/persistence" "github.com/daguflow/dagu/internal/persistence/filecache" - - "github.com/daguflow/dagu/internal/dag" "github.com/daguflow/dagu/internal/persistence/grep" "github.com/daguflow/dagu/internal/util" ) @@ -162,6 +163,88 @@ func (d *dagStoreImpl) ensureDirExist() error { return nil } +func (d *dagStoreImpl) getFileNameDagMeta() { + +} + +func (d *dagStoreImpl) searchName(fileName string, searchText *string) bool { + if searchText == nil { + return true + } + + fileName = strings.TrimSuffix(fileName, path.Ext(fileName)) + + return strings.Contains(fileName, *searchText) +} + +func (d *dagStoreImpl) searchTags(tags []string, searchTag *string) bool { + if searchTag == nil { + return true + } + + for _, tag := range tags { + if tag == *searchTag { + return true + } + } + + return false +} + +func (d *dagStoreImpl) getTagList(tagSet map[string]struct{}) []string { + tagList := make([]string, 0, len(tagSet)) + for tag := range tagSet { + tagList = append(tagList, tag) + } + return tagList +} + +func (d *dagStoreImpl) ListPagination(params dags.ListDagsParams) (*persistence.DagListPaginationResult, error) { + var ( + dagList = make([]*dag.DAG, 0) + errList = make([]string, 0) + count int64 + currentDag *dag.DAG + ) + + if err := filepath.WalkDir(d.dir, func(path string, dir fs.DirEntry, err error) error { + if err != nil { + return err + } + + if dir.IsDir() || !checkExtension(dir.Name()) { + return nil + } + + if currentDag, err = d.GetMetadata(dir.Name()); err != nil { + errList = append(errList, fmt.Sprintf("reading %s failed: %s", dir.Name(), err)) + } + + if !d.searchName(dir.Name(), params.SearchName) || currentDag == nil || !d.searchTags(currentDag.Tags, params.SearchTag) { + return nil + } + + count++ + if count > (params.Page-1)*params.Limit && int64(len(dagList)) < params.Limit { + dagList = append(dagList, currentDag) + } + + return nil + }); err != nil { + return &persistence.DagListPaginationResult{ + DagList: dagList, + Count: count, + ErrorList: append(errList, err.Error()), + }, err + } + + return &persistence.DagListPaginationResult{ + DagList: dagList, + Count: count, + ErrorList: errList, + }, nil +} + func (d *dagStoreImpl) List() (ret []*dag.DAG, errs []string, err error) { if err = d.ensureDirExist(); err != nil { errs = append(errs, err.Error()) @@ -321,3 +404,40 @@ func find(name string) (string, error) { } return "", fmt.Errorf("sub workflow %s not found", name) } + +func (d *dagStoreImpl) TagList() ([]string, []string, error) { + var ( + errList = make([]string, 0) + tagSet = make(map[string]struct{}) + currentDag *dag.DAG + err error + ) + + if err = filepath.WalkDir(d.dir, func(path string, dir fs.DirEntry, err error) error { + if err != nil { + return err + } + + if dir.IsDir() || !checkExtension(dir.Name()) { + return nil + } + + if currentDag, err = d.GetMetadata(dir.Name()); err != nil { + errList = append(errList, fmt.Sprintf("reading %s failed: %s", dir.Name(), err)) + } + + if currentDag == nil { + return nil + } + + for _, tag := range currentDag.Tags { + tagSet[tag] = struct{}{} + } + + return nil + }); err != nil { + return nil, append(errList, err.Error()), err + } + + return d.getTagList(tagSet), errList, nil +} diff --git a/swagger.yaml b/swagger.yaml index 70a38cb88..b71f170ad 100644 --- a/swagger.yaml +++ b/swagger.yaml @@ -27,6 +27,23 @@ paths: produces: - application/json operationId: listDags + parameters: + - name: page + in: query + required: true + type: integer + - name: limit + in: query + required: true + type: integer + - name: searchName + in: query + required: false + type: string + - name: searchTag + in: query + required: false + type: string responses: "200": description: A successful response. @@ -192,6 +209,23 @@ paths: $ref: "#/definitions/ApiError" tags: - dags + /tags: + get: + description: Returns a list of tags. + produces: + - application/json + operationId: listTags + responses: + "200": + description: A successful response. + schema: + $ref: "#/definitions/listTagResponse" + default: + description: Generic error response. + schema: + $ref: "#/definitions/ApiError" + tags: + - dags definitions: ApiError: @@ -218,10 +252,13 @@ definitions: type: string HasError: type: boolean + PageCount: + type: integer required: - DAGs - Errors - HasError + - PageCount createDagResponse: type: object @@ -733,3 +770,18 @@ definitions: type: boolean Interval: type: integer + + listTagResponse: + type: object + properties: + Tags: + type: array + items: + type: string + Errors: + type: array + items: + type: string + required: + - Tags + - Errors \ No newline at end of file diff --git a/ui/package.json b/ui/package.json index 6cc0494f4..240edd872 100644 --- a/ui/package.json +++ b/ui/package.json @@ -64,6 +64,7 @@ "@mui/icons-material": "^5.8.0", "@mui/material": "^5.8.1", "@tanstack/react-table": "^8.5.11", + "@types/lodash": "^4.17.7", "cron-parser": "^4.5.0", "fontsource-roboto": "^4.0.0", "mermaid": "^9.1.1", diff --git a/ui/src/components/molecules/CreateDAGButton.tsx b/ui/src/components/molecules/CreateDAGButton.tsx index 9ddcb4a6c..7dbc61952 100644 --- a/ui/src/components/molecules/CreateDAGButton.tsx +++ b/ui/src/components/molecules/CreateDAGButton.tsx @@ -11,10 +11,14 @@ function CreateDAGButton() { }} onClick={async () => { const name = window.prompt('Please input the new DAG name', ''); - if (name == '') { + if (name === null) { return; } - if (name?.indexOf(' ') != -1) { + if (name === '') { + alert('File name cannot be empty'); + return; + } + if (name.indexOf(' ') != -1) { alert('File name cannot contain space'); return; } diff --git a/ui/src/components/molecules/DAGPagination.tsx b/ui/src/components/molecules/DAGPagination.tsx new file mode 100644 index 000000000..0745a8436 --- /dev/null +++ b/ui/src/components/molecules/DAGPagination.tsx @@ -0,0 +1,34 @@ +import { Box, Pagination } from "@mui/material"; +import React from "react"; + +type DAGPaginationProps = { + totalPages: number; + page: number; + pageChange: (page: number) => void; +}; + +const DAGPagination = ({ totalPages, page, pageChange }: DAGPaginationProps) => { + const handleChange = (event: React.ChangeEvent, value: number) => { + pageChange(value); + }; + + return ( + + + + ); +} + +export default DAGPagination; \ No newline at end of file diff --git a/ui/src/components/molecules/DAGTable.tsx b/ui/src/components/molecules/DAGTable.tsx index d34e85189..33b96c876 100644 --- a/ui/src/components/molecules/DAGTable.tsx +++ b/ui/src/components/molecules/DAGTable.tsx @@ -1,4 +1,4 @@ -import React, { useEffect } from 'react'; +import React from 'react'; import { flexRender, useReactTable, @@ -27,7 +27,7 @@ import { TableRow, TextField, } from '@mui/material'; -import { Link, useSearchParams } from 'react-router-dom'; +import { Link } from 'react-router-dom'; import { getFirstTag, getStatus, @@ -48,11 +48,17 @@ import moment from 'moment'; import 'moment-duration-format'; import Ticker from '../atoms/Ticker'; import VisuallyHidden from '../atoms/VisuallyHidden'; +import useSWR from 'swr'; +import { GetListTagsResponse } from '../../models/api'; type Props = { DAGs: DAGItem[]; group: string; refreshFn: () => void; + searchText: string; + handleSearchTextChange: (searchText: string) => void; + searchTag: string; + handleSearchTagChange: (tag: string) => void; }; type DAGRow = DAGItem & { subRows?: DAGItem[] }; @@ -385,7 +391,7 @@ const defaultColumns = [ }), ]; -function DAGTable({ DAGs = [], group = '', refreshFn }: Props) { +function DAGTable({ DAGs = [], group = '', refreshFn, searchText, handleSearchTextChange, searchTag, handleSearchTagChange }: Props) { const [columns] = React.useState(() => [ ...defaultColumns, ]); @@ -400,41 +406,6 @@ function DAGTable({ DAGs = [], group = '', refreshFn }: Props) { }, ]); - const [searchParams, setSearchParams] = useSearchParams(); - useEffect(() => { - const searchText = searchParams.get('search'); - if (searchText) { - instance.getColumn('Name')?.setFilterValue(searchText); - } - const t = searchParams.get('tag'); - if (t) { - instance.getColumn('Tags')?.setFilterValue(t); - } - }, []); - - const addSearchParam = React.useCallback( - (key: string, value: string) => { - const ret: { [key: string]: string } = {}; - searchParams.forEach((v, k) => { - if (v && k !== key) { - ret[k] = v; - } - }); - if (value) { - ret[key] = value; - } - setSearchParams(ret); - }, - [searchParams, setSearchParams] - ); - - const selectedTag = React.useMemo(() => { - return ( - (columnFilters.find((filter) => filter.id == 'Tags')?.value as string) || - '' - ); - }, [columnFilters]); - const [expanded, setExpanded] = React.useState({}); const data = React.useMemo(() => { @@ -472,18 +443,11 @@ function DAGTable({ DAGs = [], group = '', refreshFn }: Props) { ]; }, [DAGs, group]); - const tagOptions = React.useMemo(() => { - const map: { [key: string]: boolean } = { '': true }; - DAGs.forEach((data) => { - if (data.Type == DAGDataType.DAG) { - data.DAGStatus.DAG.Tags?.forEach((tag) => { - map[tag] = true; - }); - } - }); - const ret = Object.keys(map).sort(); - return ret; - }, []); + const tagsRes = useSWR('/tags', null, { + refreshInterval: 10000, + }); + + const tagOptions = tagsRes.data?.Tags || []; const instance = useReactTable({ data, @@ -528,11 +492,10 @@ function DAGTable({ DAGs = [], group = '', refreshFn }: Props) { size="small" variant="filled" InputProps={{ - value: instance.getColumn('Name')?.getFilterValue(), + value: searchText, onChange: (e) => { const value = e.target.value || ''; - addSearchParam('search', value); - instance.getColumn('Name')?.setFilterValue(value); + handleSearchTextChange(value); }, type: 'search', }} @@ -540,12 +503,11 @@ function DAGTable({ DAGs = [], group = '', refreshFn }: Props) { size="small" limitTags={1} - value={selectedTag} + value={searchTag} options={tagOptions} onChange={(_, value) => { const v = value || ''; - addSearchParam('tag', v); - instance.getColumn('Tags')?.setFilterValue(v); + handleSearchTagChange(v); }} renderInput={(params) => ( @@ -589,9 +551,9 @@ function DAGTable({ DAGs = [], group = '', refreshFn }: Props) { {header.isPlaceholder ? null : flexRender( - header.column.columnDef.header, - header.getContext() - )} + header.column.columnDef.header, + header.getContext() + )} {{ asc: ( new URLSearchParams(useLocation().search); const query = useQuery(); const group = query.get('group') || ''; const appBarContext = React.useContext(AppBarContext); + const [searchText, setSearchText] = React.useState(query.get('search') || ''); + const [searchTag, setSearchTag] = React.useState(query.get('tag') || ''); + const [page, setPage] = React.useState(parseInt(query.get('page') || '1')); + const [apiSearchText, setAPISearchText] = React.useState(query.get('search') || ''); + const [apiSearchTag, setAPISearchTag] = React.useState(query.get('tag') || ''); const { cache, mutate } = useSWRConfig(); - const { data } = useSWR(`/dags`, null, { + const endPoint =`/dags?${new URLSearchParams( + { + page: page.toString(), + limit: '50', + searchName: apiSearchText, + searchTag: apiSearchTag, + } + ).toString()}` + const { data } = useSWR(endPoint, null, { refreshInterval: 10000, + revalidateIfStale: false, }); + const addSearchParam = (key: string, value: string) => { + const locationQuery = new URLSearchParams(window.location.search); + locationQuery.set(key, value); + window.history.pushState({}, '', `${window.location.pathname}?${locationQuery.toString()}`); + } + const refreshFn = React.useCallback(() => { - setTimeout(() => mutate(`/dags`), 500); + setTimeout(() => mutate(endPoint), 500); }, [mutate, cache]); React.useEffect(() => { @@ -46,6 +68,33 @@ function DAGs() { return ret; }, [data]); + const pageChange = (page: number) => { + addSearchParam('page', page.toString()); + setPage(page); + }; + + const debouncedAPISearchText = React.useMemo(() => debounce((searchText: string) => { + setAPISearchText(searchText); + }, 500), []); + + const debouncedAPISearchTag = React.useMemo(() => debounce((searchTag: string) => { + setAPISearchTag(searchTag); + }, 500), []); + + const searchTextChange = (searchText: string) => { + addSearchParam('search', searchText); + setSearchText(searchText); + setPage(1); + debouncedAPISearchText(searchText); + } + + const searchTagChange = (searchTag: string) => { + addSearchParam('tag', searchTag); + setSearchTag(searchTag); + setPage(1); + debouncedAPISearchTag(searchTag); + } + return ( + )} diff --git a/ui/src/pages/search/index.tsx b/ui/src/pages/search/index.tsx index 9c3444178..bfa5caa29 100644 --- a/ui/src/pages/search/index.tsx +++ b/ui/src/pages/search/index.tsx @@ -69,18 +69,30 @@ function Search() { - {!data && !error ? : null} - {data && data.Results.length == 0 ? ( - No results found - ) : null} - {data && data.Results.length > 0 ? ( - - - {data.Results.length} results found - - - - ) : null} + { + (() => { + if (!data && !error) { + return ; + } + + if (data && data.Results && data.Results.length > 0) { + return ( + + + {data.Results.length} results found + + + + ); + } + + if ((data && !data.Results) || (data && data.Results && data.Results.length === 0)) { + return No results found; + } + + return null + })() + } diff --git a/ui/yarn.lock b/ui/yarn.lock index 447db71cc..cd1d2f46b 100644 --- a/ui/yarn.lock +++ b/ui/yarn.lock @@ -927,6 +927,11 @@ resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.12.tgz#d70faba7039d5fca54c83c7dbab41051d2b6f6cb" integrity sha512-Hr5Jfhc9eYOQNPYO5WLDq/n4jqijdHNlDXjuAQkkt+mWdQR+XJToOHrsD4cPaMXpn6KO7y2+wM8AZEs8VpBLVA== +"@types/lodash@^4.17.7": + version "4.17.7" + resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.17.7.tgz#2f776bcb53adc9e13b2c0dfd493dfcbd7de43612" + integrity sha512-8wTvZawATi/lsmNu10/j2hk1KEP0IvjubqPE3cu1Xz7xfXXt5oCq3SNUz4fMIP4XGF9Ky+Ue2tBA3hcS7LSBlA== + "@types/mermaid@^8.2.9": version "8.2.9" resolved "https://registry.yarnpkg.com/@types/mermaid/-/mermaid-8.2.9.tgz#1844505dcffcd47703e94628a6200583d35c2c76"