From 39169e8fcd768a17c94b95eb3ccaf6acf2c9749d Mon Sep 17 00:00:00 2001 From: Yota Hamada Date: Fri, 6 Dec 2024 16:19:13 +0900 Subject: [PATCH] [#730] Add Remote-Node support (#731) --- README.md | 30 +++ docs/source/config_remote.rst | 42 +++++ docs/source/index.rst | 4 + internal/config/config.go | 13 ++ internal/frontend/dag/handler.go | 172 ++++++++++++++++++ internal/frontend/frontend.go | 8 + internal/frontend/server/server.go | 2 + internal/frontend/server/templates.go | 5 + internal/frontend/templates/base.gohtml | 50 ++--- ui/index.html | 1 + ui/src/App.tsx | 12 ++ ui/src/Layout.tsx | 47 ++++- .../components/molecules/CreateDAGButton.tsx | 25 ++- ui/src/components/molecules/DAGActions.tsx | 6 +- .../components/molecules/DAGEditButtons.tsx | 6 +- ui/src/components/molecules/LiveSwitch.tsx | 6 +- ui/src/components/organizations/DAGSpec.tsx | 4 + ui/src/contexts/AppBarContext.ts | 8 + ui/src/contexts/ConfigContext.tsx | 1 + ui/src/hooks/useDAGPostAPI.ts | 6 +- ui/src/pages/dags/dag/index.tsx | 2 +- ui/src/pages/dags/index.tsx | 1 + ui/src/pages/index.tsx | 12 +- ui/src/pages/search/index.tsx | 49 ++--- 24 files changed, 448 insertions(+), 64 deletions(-) create mode 100644 docs/source/config_remote.rst diff --git a/README.md b/README.md index 43ed4a25c..7f30a3165 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,8 @@ Dagu is a powerful Cron alternative that comes with a Web UI. It allows you to d - [3. Edit the DAG](#3-edit-the-dag) - [4. Execute the DAG](#4-execute-the-dag) - [**CLI**](#cli) +- [**Remote Node Management support**](#remote-node-management-support) + - [Configuration](#configuration) - [**Localized Documentation**](#localized-documentation) - [**Documentation**](#documentation) - [**Running as a daemon**](#running-as-a-daemon) @@ -92,6 +94,10 @@ Dagu is a powerful Cron alternative that comes with a Web UI. It allows you to d - Sending emails - Running jq command - Executing remote commands via SSH +- Remote Node support for managing multiple Dagu instances: + - Monitor DAGs across different environments + - Switch between nodes through UI dropdown + - Centralized management interface - Email notification - Scheduling with Cron expressions - REST API Interface @@ -253,6 +259,29 @@ dagu scheduler [--dags=] dagu version ``` +## **Remote Node Management support** + +Dagu supports managing multiple Dagu servers from a single UI through its remote node feature. This allows you to: + +- Monitor and manage DAGs across different environments (dev, staging, prod) +- Access multiple Dagu instances from a centralized UI +- Switch between nodes easily through the UI dropdown + +See [Remote Node Configuration](https://dagu.readthedocs.io/en/latest/config_remote.html) for more details. + +### Configuration + +Remote nodes can be configured by creating `admin.yaml` in `$HOME/.config/dagu/`: + +```yaml +# admin.yaml +remoteNodes: + - name: "prod" # Name of the remote node + apiBaseUrl: "https://prod.example.com/api/v1" # Base URL of the remote node API + - name: "staging" + apiBaseUrl: "https://staging.example.com/api/v1" +``` + ## **Localized Documentation** - [中文文档 (Chinese Documentation)](https://dagu.readthedocs.io/zh) @@ -289,6 +318,7 @@ dagu version - [JSON Processing](https://dagu.readthedocs.io/en/latest/examples.html#querying-json-data-with-jq) - [Email](https://dagu.readthedocs.io/en/latest/examples.html#sending-email) - [Configurations](https://dagu.readthedocs.io/en/latest/config.html) +- [Remote Node](https://dagu.readthedocs.io/en/latest/config_remote.html) - [Scheduler](https://dagu.readthedocs.io/en/latest/scheduler.html) - [Docker Compose](https://dagu.readthedocs.io/en/latest/docker-compose.html) - [REST API Documentation](https://app.swaggerhub.com/apis/YOHAMTA_1/dagu) diff --git a/docs/source/config_remote.rst b/docs/source/config_remote.rst new file mode 100644 index 000000000..73442a174 --- /dev/null +++ b/docs/source/config_remote.rst @@ -0,0 +1,42 @@ +.. _Remote Node Configuration: + +Remote Node +=========== + +.. contents:: + :local: + +Introduction +------------- +Dagu UI can be configured to connect to remote nodes, allowing management of DAGs across different environments from a single interface. + +How to configure +---------------- +Create ``admin.yaml`` in ``$HOME/.config/dagu/`` to configure remote nodes. Example configuration: + +.. code-block:: yaml + + # Remote Node Configuration + remoteNodes: + - name: "dev" # name of the remote node + apiBaseUrl: "http://localhost:8080/api/v1" # Base API URL of the remote node it must end with /api/v1 + + # Authentication settings for the remote node + # Basic authentication + isBasicAuth: true # Enable basic auth (optional) + basicAuthUsername: "admin" # Basic auth username (optional) + basicAuthPassword: "secret" # Basic auth password (optional) + + # api token authentication + isAuthToken: true # Enable API token (optional) + authToken: "your-secret-token" # API token value (optional) + +Using Remote Nodes +----------------- +Once configured, remote nodes can be selected from the dropdown menu in the top right corner of the UI. This allows you to: + +- Switch between different environments +- View and manage DAGs on remote nodes +- Monitor execution status across nodes + +The UI will maintain all functionality while operating on the selected remote node. diff --git a/docs/source/index.rst b/docs/source/index.rst index c07a38e91..97be21a65 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -44,6 +44,9 @@ Quick Start :ref:`Configuration Options` Configuration options. +:ref:`Remote Node Configuration` + Remote Node Configuration. + .. toctree:: :caption: Installation :hidden: @@ -75,6 +78,7 @@ Quick Start :hidden: config + config_remote scheduler auth email diff --git a/internal/config/config.go b/internal/config/config.go index a08a825e8..1e2a5df34 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -32,6 +32,7 @@ import ( // Config represents the configuration for the server. type Config struct { + RemoteNodes []RemoteNode // Remote node API URLs (e.g., http://localhost:8080/api/v1) Host string // Server host Port int // Server port DAGs string // Location of DAG files @@ -62,6 +63,18 @@ type Config struct { MaxDashboardPageLimit int // The default page limit for the dashboard } +// RemoteNode is the configuration for a remote host that can be proxied by the server. +// This is useful for fetching data from a remote host and displaying it on the server. +type RemoteNode struct { + Name string // Name of the remote host + APIBaseURL string // Base URL for the remote host API (e.g., http://localhost:9090/api/v1) + IsBasicAuth bool // Enable basic auth + BasicAuthUsername string // Basic auth username + BasicAuthPassword string // Basic auth password + IsAuthToken bool // Enable auth token for API + AuthToken string // Auth token for API +} + type TLS struct { CertFile string KeyFile string diff --git a/internal/frontend/dag/handler.go b/internal/frontend/dag/handler.go index 66a9ca2be..bd3de45f1 100644 --- a/internal/frontend/dag/handler.go +++ b/internal/frontend/dag/handler.go @@ -16,14 +16,17 @@ package dag import ( + "encoding/json" "errors" "fmt" "io" + "net/http" "os" "sort" "strings" "github.com/dagu-org/dagu/internal/client" + "github.com/dagu-org/dagu/internal/config" "github.com/dagu-org/dagu/internal/dag" "github.com/dagu-org/dagu/internal/dag/scheduler" "github.com/dagu-org/dagu/internal/frontend/gen/models" @@ -32,6 +35,7 @@ import ( "github.com/dagu-org/dagu/internal/frontend/server" "github.com/dagu-org/dagu/internal/persistence/jsondb" "github.com/dagu-org/dagu/internal/persistence/model" + "github.com/go-openapi/runtime" "github.com/go-openapi/runtime/middleware" "github.com/go-openapi/swag" "github.com/samber/lo" @@ -59,23 +63,36 @@ var ( type Handler struct { client client.Client logEncodingCharset string + remoteNodes map[string]config.RemoteNode + apiBasePath string } type NewHandlerArgs struct { Client client.Client LogEncodingCharset string + RemoteNodes []config.RemoteNode + ApiBasePath string } func NewHandler(args *NewHandlerArgs) server.Handler { + remoteNodes := make(map[string]config.RemoteNode) + for _, node := range args.RemoteNodes { + remoteNodes[node.Name] = node + } return &Handler{ client: args.Client, logEncodingCharset: args.LogEncodingCharset, + remoteNodes: remoteNodes, + apiBasePath: args.ApiBasePath, } } func (h *Handler) Configure(api *operations.DaguAPI) { api.DagsListDagsHandler = dags.ListDagsHandlerFunc( func(params dags.ListDagsParams) middleware.Responder { + if resp := h.handleRemoteNodeProxy(nil, params.HTTPRequest); resp != nil { + return resp + } resp, err := h.getList(params) if err != nil { return dags.NewListDagsDefault(err.Code). @@ -86,6 +103,9 @@ func (h *Handler) Configure(api *operations.DaguAPI) { api.DagsGetDagDetailsHandler = dags.GetDagDetailsHandlerFunc( func(params dags.GetDagDetailsParams) middleware.Responder { + if resp := h.handleRemoteNodeProxy(nil, params.HTTPRequest); resp != nil { + return resp + } resp, err := h.getDetail(params) if err != nil { return dags.NewGetDagDetailsDefault(err.Code). @@ -96,6 +116,9 @@ func (h *Handler) Configure(api *operations.DaguAPI) { api.DagsPostDagActionHandler = dags.PostDagActionHandlerFunc( func(params dags.PostDagActionParams) middleware.Responder { + if resp := h.handleRemoteNodeProxy(params.Body, params.HTTPRequest); resp != nil { + return resp + } resp, err := h.postAction(params) if err != nil { return dags.NewPostDagActionDefault(err.Code). @@ -106,6 +129,9 @@ func (h *Handler) Configure(api *operations.DaguAPI) { api.DagsCreateDagHandler = dags.CreateDagHandlerFunc( func(params dags.CreateDagParams) middleware.Responder { + if resp := h.handleRemoteNodeProxy(params.Body, params.HTTPRequest); resp != nil { + return resp + } resp, err := h.createDAG(params) if err != nil { return dags.NewCreateDagDefault(err.Code). @@ -116,6 +142,9 @@ func (h *Handler) Configure(api *operations.DaguAPI) { api.DagsDeleteDagHandler = dags.DeleteDagHandlerFunc( func(params dags.DeleteDagParams) middleware.Responder { + if resp := h.handleRemoteNodeProxy(nil, params.HTTPRequest); resp != nil { + return resp + } err := h.deleteDAG(params) if err != nil { return dags.NewDeleteDagDefault(err.Code). @@ -126,6 +155,9 @@ func (h *Handler) Configure(api *operations.DaguAPI) { api.DagsSearchDagsHandler = dags.SearchDagsHandlerFunc( func(params dags.SearchDagsParams) middleware.Responder { + if resp := h.handleRemoteNodeProxy(nil, params.HTTPRequest); resp != nil { + return resp + } resp, err := h.searchDAGs(params) if err != nil { return dags.NewSearchDagsDefault(err.Code). @@ -136,6 +168,9 @@ func (h *Handler) Configure(api *operations.DaguAPI) { api.DagsListTagsHandler = dags.ListTagsHandlerFunc( func(params dags.ListTagsParams) middleware.Responder { + if resp := h.handleRemoteNodeProxy(nil, params.HTTPRequest); resp != nil { + return resp + } tags, err := h.getTagList(params) if err != nil { return dags.NewListTagsDefault(err.Code). @@ -145,6 +180,143 @@ func (h *Handler) Configure(api *operations.DaguAPI) { }) } +// handleRemoteNodeProxy checks if 'remoteNode' is present in the query parameters. +// If yes, it proxies the request to the remote node and returns the remote response. +// If not, it returns nil, indicating to proceed locally. +func (h *Handler) handleRemoteNodeProxy(body any, r *http.Request) middleware.Responder { + if r == nil { + return nil + } + + remoteNodeName := r.URL.Query().Get("remoteNode") + if remoteNodeName == "" || remoteNodeName == "local" { + return nil // No remote node specified, handle locally + } + + node, ok := h.remoteNodes[remoteNodeName] + if !ok { + // remote node not found, return bad request + return dags.NewListDagsDefault(400) + } + + // forward the request to the remote node + return h.doRemoteProxy(body, r, node) +} + +// doRemoteProxy performs the actual proxying of the request to the remote node. +func (h *Handler) doRemoteProxy(body any, originalReq *http.Request, node config.RemoteNode) middleware.Responder { + // Copy original query parameters except remoteNode + q := originalReq.URL.Query() + q.Del("remoteNode") + + // Build the new remote URL + urlComponents := strings.Split(originalReq.URL.Path, h.apiBasePath) + if len(urlComponents) < 2 { + return h.responderWithCodedError(&codedError{ + Code: 400, + APIError: &models.APIError{ + Message: swag.String("invalid API path"), + }}) + } + remoteURL := fmt.Sprintf("%s%s?%s", strings.TrimSuffix(node.APIBaseURL, "/"), urlComponents[1], q.Encode()) + + method := originalReq.Method + var bodyJSON io.Reader + if body != nil { + // Forward the request body if needed + // originalReq.Body is a ReadCloser; ensure we can read it only once. + // Typically, you'd buffer it or ensure it's reusable. + // For simplicity, let's assume we can read it directly. + data, err := json.Marshal(body) + if err != nil { + return h.responderWithCodedError(&codedError{ + Code: 502, + APIError: &models.APIError{ + Message: swag.String(fmt.Sprintf("failed to read request body: %v", err)), + }}) + } + bodyJSON = strings.NewReader(string(data)) + } + + req, err := http.NewRequest(method, remoteURL, bodyJSON) + if err != nil { + return h.responderWithCodedError(&codedError{ + Code: 502, + APIError: &models.APIError{ + Message: swag.String(fmt.Sprintf("failed to create request to remote node: %v", err)), + }}) + } + + // Copy headers from the original request if needed + // But we need to overwrite authorization headers + if node.IsBasicAuth { + req.SetBasicAuth(node.BasicAuthUsername, node.BasicAuthPassword) + } else if node.IsAuthToken { + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", node.AuthToken)) + } + for k, v := range originalReq.Header { + if k == "Authorization" { + continue + } + for _, vv := range v { + req.Header.Add(k, vv) + } + } + + client := &http.Client{} + resp, err := client.Do(req) + defer func() { + _ = resp.Body.Close() + }() // Ensure we close the response body + if err != nil { + return h.responderWithCodedError(&codedError{ + Code: 502, + APIError: &models.APIError{ + Message: swag.String(fmt.Sprintf("failed to send request to remote node: %v", err)), + }}) + } + defer resp.Body.Close() + + // If not status 200, return an error + if resp.StatusCode < 200 || resp.StatusCode > 299 { + // Try to decode remote error if it's JSON + var remoteErr models.APIError + if err := json.NewDecoder(resp.Body).Decode(&remoteErr); err == nil && remoteErr.Message != nil { + return h.responderWithCodedError(&codedError{ + Code: resp.StatusCode, + APIError: &remoteErr, + }) + } + // If we cannot decode a proper error, return a generic one + payload := &models.APIError{ + Message: swag.String(fmt.Sprintf("remote node responded with status %d", resp.StatusCode)), + } + return h.responderWithCodedError(&codedError{ + Code: resp.StatusCode, + APIError: payload, + }) + } + + respData, err := io.ReadAll(resp.Body) + if err != nil { + return h.responderWithCodedError(&codedError{ + Code: 502, + APIError: &models.APIError{ + Message: swag.String(fmt.Sprintf("failed to read response from remote node: %v", err)), + }}) + } + + return middleware.ResponderFunc(func(w http.ResponseWriter, _ runtime.Producer) { + w.WriteHeader(resp.StatusCode) + _, _ = w.Write(respData) + }) +} + +func (h *Handler) responderWithCodedError(err *codedError) middleware.Responder { + return dags.NewListDagsDefault(err.Code). + WithPayload(err.APIError) +} + func (h *Handler) createDAG( params dags.CreateDagParams, ) (*models.CreateDagResponse, *codedError) { diff --git a/internal/frontend/frontend.go b/internal/frontend/frontend.go index 881d61224..5971c58d8 100644 --- a/internal/frontend/frontend.go +++ b/internal/frontend/frontend.go @@ -30,9 +30,16 @@ func New(cfg *config.Config, lg logger.Logger, cli client.Client) *server.Server &dag.NewHandlerArgs{ Client: cli, LogEncodingCharset: cfg.LogEncodingCharset, + RemoteNodes: cfg.RemoteNodes, + ApiBasePath: cfg.APIBaseURL, }, )) + var remoteNodes []string + for _, n := range cfg.RemoteNodes { + remoteNodes = append(remoteNodes, n.Name) + } + serverParams := server.NewServerArgs{ Host: cfg.Host, Port: cfg.Port, @@ -45,6 +52,7 @@ func New(cfg *config.Config, lg logger.Logger, cli client.Client) *server.Server APIBaseURL: cfg.APIBaseURL, MaxDashboardPageLimit: cfg.MaxDashboardPageLimit, TimeZone: cfg.TZ, + RemoteNodes: remoteNodes, } if cfg.IsAuthToken { diff --git a/internal/frontend/server/server.go b/internal/frontend/server/server.go index 650448c83..6bb357215 100644 --- a/internal/frontend/server/server.go +++ b/internal/frontend/server/server.go @@ -66,6 +66,7 @@ type NewServerArgs struct { APIBaseURL string TimeZone string MaxDashboardPageLimit int + RemoteNodes []string } type BasicAuth struct { @@ -98,6 +99,7 @@ func New(params NewServerArgs) *Server { APIBaseURL: params.APIBaseURL, TZ: params.TimeZone, MaxDashboardPageLimit: params.MaxDashboardPageLimit, + RemoteNodes: params.RemoteNodes, }, } } diff --git a/internal/frontend/server/templates.go b/internal/frontend/server/templates.go index 50063e363..016d72a33 100644 --- a/internal/frontend/server/templates.go +++ b/internal/frontend/server/templates.go @@ -21,6 +21,7 @@ import ( "net/http" "path" "path/filepath" + "strings" "text/template" "github.com/dagu-org/dagu/internal/constants" @@ -61,6 +62,7 @@ type funcsConfig struct { APIBaseURL string TZ string MaxDashboardPageLimit int + RemoteNodes []string } func defaultFunctions(cfg funcsConfig) template.FuncMap { @@ -93,6 +95,9 @@ func defaultFunctions(cfg funcsConfig) template.FuncMap { "maxDashboardPageLimit": func() int { return cfg.MaxDashboardPageLimit }, + "remoteNodes": func() string { + return strings.Join(cfg.RemoteNodes, ",") + }, } } diff --git a/internal/frontend/templates/base.gohtml b/internal/frontend/templates/base.gohtml index 694a04789..e389dcf82 100644 --- a/internal/frontend/templates/base.gohtml +++ b/internal/frontend/templates/base.gohtml @@ -1,28 +1,32 @@ {{define "base"}} - - - - - {{ navbarTitle }} - - - - - {{template "content" .}} - + + + + + {{ navbarTitle }} + + + + + {{template "content" .}} + {{ end }} diff --git a/ui/index.html b/ui/index.html index cce3a38e7..2a507f49f 100644 --- a/ui/index.html +++ b/ui/index.html @@ -14,6 +14,7 @@ version: '', tz: '', maxDashboardPageLimit: '', + remoteNodes: '', }; } diff --git a/ui/src/App.tsx b/ui/src/App.tsx index afe94ce4f..7c7916fb6 100644 --- a/ui/src/App.tsx +++ b/ui/src/App.tsx @@ -19,6 +19,15 @@ type Props = { function App({ config }: Props) { const [title, setTitle] = React.useState(''); config.tz ||= moment.tz.guess(); + const remoteNodes = config.remoteNodes + .split(',') + .filter(Boolean) + .map((node) => node.trim()); + if (!remoteNodes.includes('local')) { + remoteNodes.unshift('local'); + } + const [selectedRemoteNode, setSelectedRemoteNode] = + React.useState('local'); return ( diff --git a/ui/src/Layout.tsx b/ui/src/Layout.tsx index d933f692a..db4e56bc4 100644 --- a/ui/src/Layout.tsx +++ b/ui/src/Layout.tsx @@ -8,7 +8,7 @@ import Toolbar from '@mui/material/Toolbar'; import List from '@mui/material/List'; import Typography from '@mui/material/Typography'; import { mainListItems } from './menu'; -import { Grid } from '@mui/material'; +import { Grid, MenuItem, Select } from '@mui/material'; import { AppBarContext } from './contexts/AppBarContext'; const drawerWidthClosed = 64; @@ -137,7 +137,50 @@ function Content({ title, navbarColor, children }: DashboardContentProps) { )} - {title || 'Dagu'} + + {title || 'Dagu'} + + {(context) => { + if ( + !context.remoteNodes || + context.remoteNodes.length === 0 + ) { + return null; + } + return ( + + ); + }} + + { - const url = `${getConfig().apiURL}/dags/${params.name}`; + const url = `${getConfig().apiURL}/dags/${params.name}?remoteNode=${ + appBarContext.selectedRemoteNode || 'local' + }`; const ret = await fetch(url, { method: 'POST', headers: { diff --git a/ui/src/components/molecules/DAGEditButtons.tsx b/ui/src/components/molecules/DAGEditButtons.tsx index 753e7d998..9d6c070f0 100644 --- a/ui/src/components/molecules/DAGEditButtons.tsx +++ b/ui/src/components/molecules/DAGEditButtons.tsx @@ -1,11 +1,13 @@ import React from 'react'; import { Button, Stack } from '@mui/material'; +import { AppBarContext } from '../../contexts/AppBarContext'; type Props = { name: string; }; function DAGEditButtons({ name }: Props) { + const appBarContext = React.useContext(AppBarContext); return (