tutorial/Tutorial7_auto_tuning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Automatic Hyperparameter Tuning for DeepScalper\n",
    "This tutorial shows how automatic hyperparameter tuning is conducted with Optuna library.\n",
    "## Step 1: Import Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-02 13:16:54,316\tINFO services.py:1476 -- View the Ray dashboard at \u001b[1m\u001b[32mhttp://127.0.0.1:8265\u001b[39m\u001b[22m\n",
      "2023-03-02 13:16:56,578\tINFO worker.py:973 -- Calling ray.init() again after it has already been called.\n"
     ]
    }
   ],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "import sys\n",
    "import os\n",
    "\n",
    "ROOT = os.path.dirname(os.path.abspath(\".\"))\n",
    "sys.path.append(ROOT)\n",
    "\n",
    "import torch\n",
    "import argparse\n",
    "import os.path as osp\n",
    "from mmcv import Config\n",
    "from trademaster.utils import replace_cfg_vals\n",
    "from trademaster.nets.builder import build_net\n",
    "from trademaster.environments.builder import build_environment\n",
    "from trademaster.datasets.builder import build_dataset\n",
    "from trademaster.agents.builder import build_agent\n",
    "from trademaster.optimizers.builder import build_optimizer\n",
    "from trademaster.losses.builder import build_loss\n",
    "from trademaster.trainers.builder import build_trainer\n",
    "from trademaster.transition.builder import build_transition\n",
    "import optuna\n",
    "from shutil import copyfile\n",
    "import joblib\n",
    "\n",
    "from trademaster.utils import set_seed\n",
    "set_seed(12345)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 2: Load Configs\n",
    "Load default config from the folder configs/algorithmic_trading/algorithmic_trading_BTC_dqn_dqn_adam_mse.py\n",
    "Set the flag auto_tuning to True and n_trials as the number of hyperparameter groups wanted to be searched. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [],
   "source": [
    "def parse_args():\n",
    "    parser = argparse.ArgumentParser(description='Download Alpaca Datasets')\n",
    "    parser.add_argument(\"--config\", default=osp.join(ROOT, \"configs\", \"algorithmic_trading\", \"algorithmic_trading_BTC_deepscalper_deepscalper_adam_mse.py\"),\n",
    "                        help=\"download datasets config file path\")\n",
    "    parser.add_argument(\"--task_name\", type=str, default=\"train\")\n",
    "    parser.add_argument(\"--auto_tuning\", default=True, type=bool)\n",
    "    parser.add_argument(\"--n_trials\", default=5, type=int)\n",
    "    args = parser.parse_args([])\n",
    "    return args"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 3: Build RL agent, environment and trainer\n",
    "Build the environment of algorithmic trading, agent and trainer of DeepScalper following the previous tutorial"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [],
   "source": [
    "def test_dqn_builder(args, sampled_params):\n",
    "    cfg = Config.fromfile(args.config)\n",
    "    task_name = args.task_name\n",
    "    cfg = replace_cfg_vals(cfg)\n",
    "    \n",
    "    dataset = build_dataset(cfg)\n",
    "\n",
    "    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "    train_environment = build_environment(cfg, default_args=dict(dataset=dataset, task=\"train\"))\n",
    "    valid_environment = build_environment(cfg, default_args=dict(dataset=dataset, task=\"valid\"))\n",
    "    test_environment = build_environment(cfg, default_args=dict(dataset=dataset, task=\"test\"))\n",
    "\n",
    "    action_dim = train_environment.action_dim\n",
    "    state_dim = train_environment.state_dim\n",
    "\n",
    "    cfg.act.update(dict(action_dim=action_dim, state_dim=state_dim))\n",
    "\n",
    "    # hyperparameters sampled from sampler \n",
    "    if args.auto_tuning == True:\n",
    "        # lr = sampled_params['lr']\n",
    "        explore_rate = sampled_params['explore_rate']\n",
    "        dims = sampled_params['dims']\n",
    "        # cfg.optimizer.update(dict(lr=lr))\n",
    "        cfg.act.update(dict(explore_rate=explore_rate, dims = dims))\n",
    "\n",
    "    print(cfg)\n",
    "\n",
    "    act = build_net(cfg.act)\n",
    "    act_optimizer = build_optimizer(cfg, default_args=dict(params=act.parameters()))\n",
    "\n",
    "    if cfg.cri:\n",
    "        cfg.cri.update(dict(action_dim=action_dim, state_dim=state_dim))\n",
    "        cri = build_net(cfg.cri)\n",
    "        cri_optimizer = build_optimizer(cfg, default_args=dict(params=cri.parameters()))\n",
    "    else:\n",
    "        cri = None\n",
    "        cri_optimizer = None\n",
    "\n",
    "    if_remove = cfg.trainer.if_remove\n",
    "    work_dir = os.path.join(ROOT, cfg.work_dir)\n",
    "    \n",
    "\n",
    "    if not os.path.exists(work_dir):\n",
    "        os.makedirs(work_dir)\n",
    "    \n",
    "    cfg.dump(osp.join(ROOT, cfg.work_dir, osp.basename(args.config)))\n",
    "\n",
    "\n",
    "    criterion = build_loss(cfg)\n",
    "\n",
    "    transition = build_transition(cfg)\n",
    "\n",
    "    agent = build_agent(cfg, default_args=dict(action_dim = action_dim,\n",
    "                                               state_dim = state_dim,\n",
    "                                               act = act,\n",
    "                                               cri = cri,\n",
    "                                               act_optimizer = act_optimizer,\n",
    "                                               cri_optimizer = cri_optimizer,\n",
    "                                               criterion = criterion,\n",
    "                                               transition = transition))\n",
    "\n",
    "    trainer = build_trainer(cfg, default_args=dict(train_environment = train_environment,\n",
    "                                               valid_environment = valid_environment,\n",
    "                                               test_environment = test_environment,\n",
    "                                               agent = agent,\n",
    "                                               device = device))\n",
    "    return trainer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 4: Sample hyperparameters using Optuna\n",
    "Optuna library provides functions to sample different types of hyperparameters. In this tutorial, learning rate, explore rate, number of hidden nodes of DQN are tuned. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [],
   "source": [
    "def sample_params(trial: optuna.Trial):\n",
    "    # lr = trial.suggest_float(\"lr\", 1e-4, 1e-3, log=True)\n",
    "    explore_rate = trial.suggest_float(\"explore_rate\", 0.15, 0.3, step=0.05)\n",
    "    dims = trial.suggest_categorical(\"hidden_nodes\", [(64,32), (128,64)])\n",
    "    trial_number = trial.number\n",
    "    sampled_params = dict(explore_rate=explore_rate, dims=dims, trial_number=trial_number)\n",
    "    return sampled_params"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 5: Define objective function of the optimization procedure\n",
    "Optuna tries to find the best hyperparameter pair by optimizing a specific objective function. \n",
    "In this tutorial, the original training procedure is repeated, and the sum of episode reward on valid dataset is used as the objective function. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [],
   "source": [
    "def objective(trial: optuna.Trial) -> float:\n",
    "    args = parse_args()\n",
    "    sampled_params = sample_params(trial)\n",
    "    trainer = test_dqn_builder(args, sampled_params)\n",
    "    valid_score = trainer.train_and_valid_trial(sampled_params['trial_number'])\n",
    "    return valid_score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 6: Run the whole tuning procedure\n",
    "Due to the fact that the original training pipeline will be run for n_trials of times, it might take time to be finished. Please use smaller n_trials to reduce time for tuning."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2023-03-02 13:17:44,149]\u001b[0m A new study created in memory with name: no-name-430a284e-1940-4a4e-80c8-cf68aff54a4b\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Config (path: /home/cqzong/TradeMaster/configs/algorithmic_trading/algorithmic_trading_BTC_deepscalper_deepscalper_adam_mse.py): {'data': {'type': 'AlgorithmicTradingDataset', 'data_path': 'data/algorithmic_trading/BTC', 'train_path': 'data/algorithmic_trading/BTC/train.csv', 'valid_path': 'data/algorithmic_trading/BTC/valid.csv', 'test_path': 'data/algorithmic_trading/BTC/test.csv', 'test_dynamic_path': 'data/algorithmic_trading/BTC/test_labeled_3_24_-0.15_0.15.csv', 'tech_indicator_list': ['high', 'low', 'open', 'close', 'adjcp', 'zopen', 'zhigh', 'zlow', 'zadjcp', 'zclose', 'zd_5', 'zd_10', 'zd_15', 'zd_20', 'zd_25', 'zd_30'], 'backward_num_day': 5, 'forward_num_day': 5, 'test_dynamic': '-1'}, 'environment': {'type': 'AlgorithmicTradingEnvironment'}, 'agent': {'type': 'AlgorithmicTradingDQN', 'max_step': 12345, 'reward_scale': 1, 'repeat_times': 1, 'gamma': 0.9, 'batch_size': 64, 'clip_grad_norm': 3.0, 'soft_update_tau': 0, 'state_value_tau': 0.005}, 'trainer': {'type': 'AlgorithmicTradingTrainer', 'epochs': 20, 'work_dir': 'work_dir/algorithmic_trading_BTC_deepscalper_deepscalper_adam_mse', 'seeds_list': (12345,), 'batch_size': 64, 'horizon_len': 128, 'buffer_size': 1000000.0, 'num_threads': 8, 'if_remove': False, 'if_discrete': True, 'if_off_policy': True, 'if_keep_save': True, 'if_over_write': False, 'if_save_buffer': False}, 'loss': {'type': 'MSELoss'}, 'optimizer': {'type': 'Adam', 'lr': 0.001}, 'act': {'type': 'QNet', 'state_dim': 82, 'action_dim': 3, 'dims': (64, 32), 'explore_rate': 0.3}, 'cri': None, 'transition': {'type': 'Transition'}, 'task_name': 'algorithmic_trading', 'dataset_name': 'BTC', 'optimizer_name': 'adam', 'loss_name': 'mse', 'net_name': 'deepscalper', 'agent_name': 'deepscalper', 'work_dir': 'work_dir/algorithmic_trading_BTC_deepscalper_deepscalper_adam_mse', 'batch_size': 64}\n",
      "| Arguments Keep work_dir: /home/cqzong/TradeMaster/work_dir/algorithmic_trading_BTC_deepscalper_deepscalper_adam_mse\n",
      "Train Episode: [1/20]\n",
      "+---------------+-------------+------------+--------------+\n",
      "|  Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+---------------+-------------+------------+--------------+\n",
      "| -1432.253960% |  -6.911822  | 67.399534% | 2264.159660% |\n",
      "+---------------+-------------+------------+--------------+\n",
      "Valid Episode: [1/20]\n",
      "+--------------+-------------+-------------+--------------+\n",
      "| Total Return | Sharp Ratio |  Volatility | Max Drawdown |\n",
      "+--------------+-------------+-------------+--------------+\n",
      "| -18.269981%  |  -0.068640  | 248.547725% | 111.377168%  |\n",
      "+--------------+-------------+-------------+--------------+\n",
      "Train Episode: [2/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -673.369147% |  -6.386928  | 34.291823% | 195.257119%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Valid Episode: [2/20]\n",
      "+--------------+-------------+-------------+--------------+\n",
      "| Total Return | Sharp Ratio |  Volatility | Max Drawdown |\n",
      "+--------------+-------------+-------------+--------------+\n",
      "| -14.917323%  |  -0.101390  | 137.387124% | 111.504621%  |\n",
      "+--------------+-------------+-------------+--------------+\n",
      "Train Episode: [3/20]\n",
      "+---------------+-------------+------------+--------------+\n",
      "|  Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+---------------+-------------+------------+--------------+\n",
      "| -1200.559899% |  -55.531160 | 7.031961%  | 2410.669300% |\n",
      "+---------------+-------------+------------+--------------+\n",
      "Valid Episode: [3/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  30.521555%  |   7.687130  | 3.707608%  |  51.052408%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [4/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -430.353134% |  -3.541119  | 39.528819% | 205.451565%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Valid Episode: [4/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  6.473204%   |   0.497972  | 12.138529% |  59.040496%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [5/20]\n",
      "+---------------+-------------+------------+--------------+\n",
      "|  Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+---------------+-------------+------------+--------------+\n",
      "| -2902.624710% |  -48.932019 | 19.294216% | 6426.153701% |\n",
      "+---------------+-------------+------------+--------------+\n",
      "Valid Episode: [5/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  2.980972%   |   0.433495  | 6.421331%  |  73.044790%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [6/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -266.669789% |  -7.476699  | 11.600944% | 366.127580%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Valid Episode: [6/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -57.982892%  |  -6.145012  | 8.811067%  |  83.657880%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [7/20]\n",
      "+---------------+-------------+------------+--------------+\n",
      "|  Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+---------------+-------------+------------+--------------+\n",
      "| -2785.448210% |  -72.096347 | 12.566410% | 6099.958142% |\n",
      "+---------------+-------------+------------+--------------+\n",
      "Valid Episode: [7/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  5.932907%   |   1.573329  | 3.521271%  |  49.419858%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [8/20]\n",
      "+---------------+-------------+--------------+--------------+\n",
      "|  Total Return | Sharp Ratio |  Volatility  | Max Drawdown |\n",
      "+---------------+-------------+--------------+--------------+\n",
      "| -2386.857587% |  -0.553500  | 1402.613794% | 4275.213281% |\n",
      "+---------------+-------------+--------------+--------------+\n",
      "Valid Episode: [8/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -34.271965%  |  -2.167177  | 14.767125% |  91.739656%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [9/20]\n",
      "+--------------+-------------+-------------+--------------+\n",
      "| Total Return | Sharp Ratio |  Volatility | Max Drawdown |\n",
      "+--------------+-------------+-------------+--------------+\n",
      "| -634.592205% |  -1.265574  | 163.093496% | 616.200630%  |\n",
      "+--------------+-------------+-------------+--------------+\n",
      "Valid Episode: [9/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  6.167382%   |   1.639328  | 3.513066%  |  50.060936%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [10/20]\n",
      "+---------------+-------------+------------+--------------+\n",
      "|  Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+---------------+-------------+------------+--------------+\n",
      "| -2839.116674% |  -19.801659 | 46.634901% | 3398.835750% |\n",
      "+---------------+-------------+------------+--------------+\n",
      "Valid Episode: [10/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  16.783844%  |   4.422834  | 3.543579%  |  50.061053%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [11/20]\n",
      "+---------------+-------------+------------+--------------+\n",
      "|  Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+---------------+-------------+------------+--------------+\n",
      "| -2159.848733% |  -58.013042 | 12.109524% | 4171.172901% |\n",
      "+---------------+-------------+------------+--------------+\n",
      "Valid Episode: [11/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  4.734262%   |   0.562702  | 7.856435%  |  80.393945%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [12/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -418.956559% |  -4.193499  | 32.495398% | 346.920120%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Valid Episode: [12/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -14.013641%  |  -1.278057  | 10.238869% |  87.113019%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [13/20]\n",
      "+---------------+-------------+------------+--------------+\n",
      "|  Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+---------------+-------------+------------+--------------+\n",
      "| -1132.070249% |  -6.235043  | 59.055907% | 2273.136478% |\n",
      "+---------------+-------------+------------+--------------+\n",
      "Valid Episode: [13/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  23.554368%  |   5.245140  | 4.193394%  |  49.430532%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [14/20]\n",
      "+---------------+-------------+------------+--------------+\n",
      "|  Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+---------------+-------------+------------+--------------+\n",
      "| -1583.828226% |  -5.204213  | 98.987971% | 3060.142581% |\n",
      "+---------------+-------------+------------+--------------+\n",
      "Valid Episode: [14/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  36.243173%  |   8.935169  | 3.787693%  |  50.707941%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [15/20]\n",
      "+---------------+-------------+------------+--------------+\n",
      "|  Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+---------------+-------------+------------+--------------+\n",
      "| -1173.505427% |  -49.300506 | 7.742177%  | 1893.105259% |\n",
      "+---------------+-------------+------------+--------------+\n",
      "Valid Episode: [15/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  6.939353%   |   1.711267  | 3.786627%  |  49.702266%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [16/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -819.328503% |  -36.974003 | 7.207605%  | 1348.309491% |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Valid Episode: [16/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  25.700654%  |   6.494632  | 3.695226%  |  50.808096%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [17/20]\n",
      "+---------------+-------------+------------+--------------+\n",
      "|  Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+---------------+-------------+------------+--------------+\n",
      "| -1553.146667% |  -9.953999  | 50.750961% | 2501.106663% |\n",
      "+---------------+-------------+------------+--------------+\n",
      "Valid Episode: [17/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -13.916212%  |  -0.922812  | 14.081832% |  91.667173%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [18/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -281.226293% |  -1.556474  | 58.768333% | 207.069215%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Valid Episode: [18/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -35.962402%  |  -3.700096  | 9.075844%  |  82.850782%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [19/20]\n",
      "+--------------+-------------+-------------+--------------+\n",
      "| Total Return | Sharp Ratio |  Volatility | Max Drawdown |\n",
      "+--------------+-------------+-------------+--------------+\n",
      "| 1176.853862% |   2.646098  | 144.659161% | 2375.445821% |\n",
      "+--------------+-------------+-------------+--------------+\n",
      "Valid Episode: [19/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  39.637850%  |  10.223068  | 3.620597%  |  49.654296%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [20/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| 1635.491304% |  26.445782  | 20.115055% | 102.216219%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Valid Episode: [20/20]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2023-03-02 13:27:35,996]\u001b[0m Trial 0 finished with value: 62326.314575009914 and parameters: {'explore_rate': 0.3, 'hidden_nodes': (64, 32)}. Best is trial 0 with value: 62326.314575009914.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  3.010911%   |   0.472666  | 5.948328%  |  71.901604%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Config (path: /home/cqzong/TradeMaster/configs/algorithmic_trading/algorithmic_trading_BTC_deepscalper_deepscalper_adam_mse.py): {'data': {'type': 'AlgorithmicTradingDataset', 'data_path': 'data/algorithmic_trading/BTC', 'train_path': 'data/algorithmic_trading/BTC/train.csv', 'valid_path': 'data/algorithmic_trading/BTC/valid.csv', 'test_path': 'data/algorithmic_trading/BTC/test.csv', 'test_dynamic_path': 'data/algorithmic_trading/BTC/test_labeled_3_24_-0.15_0.15.csv', 'tech_indicator_list': ['high', 'low', 'open', 'close', 'adjcp', 'zopen', 'zhigh', 'zlow', 'zadjcp', 'zclose', 'zd_5', 'zd_10', 'zd_15', 'zd_20', 'zd_25', 'zd_30'], 'backward_num_day': 5, 'forward_num_day': 5, 'test_dynamic': '-1'}, 'environment': {'type': 'AlgorithmicTradingEnvironment'}, 'agent': {'type': 'AlgorithmicTradingDQN', 'max_step': 12345, 'reward_scale': 1, 'repeat_times': 1, 'gamma': 0.9, 'batch_size': 64, 'clip_grad_norm': 3.0, 'soft_update_tau': 0, 'state_value_tau': 0.005}, 'trainer': {'type': 'AlgorithmicTradingTrainer', 'epochs': 20, 'work_dir': 'work_dir/algorithmic_trading_BTC_deepscalper_deepscalper_adam_mse', 'seeds_list': (12345,), 'batch_size': 64, 'horizon_len': 128, 'buffer_size': 1000000.0, 'num_threads': 8, 'if_remove': False, 'if_discrete': True, 'if_off_policy': True, 'if_keep_save': True, 'if_over_write': False, 'if_save_buffer': False}, 'loss': {'type': 'MSELoss'}, 'optimizer': {'type': 'Adam', 'lr': 0.001}, 'act': {'type': 'QNet', 'state_dim': 82, 'action_dim': 3, 'dims': (128, 64), 'explore_rate': 0.2}, 'cri': None, 'transition': {'type': 'Transition'}, 'task_name': 'algorithmic_trading', 'dataset_name': 'BTC', 'optimizer_name': 'adam', 'loss_name': 'mse', 'net_name': 'deepscalper', 'agent_name': 'deepscalper', 'work_dir': 'work_dir/algorithmic_trading_BTC_deepscalper_deepscalper_adam_mse', 'batch_size': 64}\n",
      "| Arguments Keep work_dir: /home/cqzong/TradeMaster/work_dir/algorithmic_trading_BTC_deepscalper_deepscalper_adam_mse\n",
      "Train Episode: [1/20]\n",
      "+---------------+-------------+-------------+--------------+\n",
      "|  Total Return | Sharp Ratio |  Volatility | Max Drawdown |\n",
      "+---------------+-------------+-------------+--------------+\n",
      "| -1467.694330% |  -1.438143  | 331.942558% | 3170.478843% |\n",
      "+---------------+-------------+-------------+--------------+\n",
      "Valid Episode: [1/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  -0.000000%  |  -0.000159  | 0.000000%  |  0.000000%   |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [2/20]\n",
      "+---------------+-------------+-------------+--------------+\n",
      "|  Total Return | Sharp Ratio |  Volatility | Max Drawdown |\n",
      "+---------------+-------------+-------------+--------------+\n",
      "| -1345.259828% |  -1.507394  | 290.274515% | 2137.059561% |\n",
      "+---------------+-------------+-------------+--------------+\n",
      "Valid Episode: [2/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -189.953899% |  -7.670739  | 23.123967% | 212.054056%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [3/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -334.221817% |  -12.355647 | 8.798299%  | 179.136186%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Valid Episode: [3/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -173.362232% |  -7.128005  | 22.711083% | 193.453284%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [4/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| 720.638399%  |   2.593904  | 90.363483% | 188.956757%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Valid Episode: [4/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -149.558373% |  -5.931561  | 23.544697% | 186.181733%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [5/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -804.838557% |  -9.279765  | 28.209876% | 542.564150%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Valid Episode: [5/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -174.689102% |  -7.086729  | 23.018198% | 194.780155%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [6/20]\n",
      "+---------------+-------------+-------------+--------------+\n",
      "|  Total Return | Sharp Ratio |  Volatility | Max Drawdown |\n",
      "+---------------+-------------+-------------+--------------+\n",
      "| -4130.142207% |  -3.351020  | 400.882788% | 4707.532529% |\n",
      "+---------------+-------------+-------------+--------------+\n",
      "Valid Episode: [6/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -190.503675% |  -7.769157  | 22.897118% | 212.603833%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [7/20]\n",
      "+---------------+-------------+------------+--------------+\n",
      "|  Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+---------------+-------------+------------+--------------+\n",
      "| -1442.143815% |  -12.543536 | 37.395383% | 1531.796470% |\n",
      "+---------------+-------------+------------+--------------+\n",
      "Valid Episode: [7/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -209.829789% |  -8.721146  | 22.466992% | 233.939052%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [8/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| 1145.982128% |   8.104130  | 45.993965% | 238.924588%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Valid Episode: [8/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -230.015278% |  -9.653198  | 22.250349% | 256.133646%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [9/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| 1284.015069% |  181.481721 | 2.301265%  |  64.519610%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Valid Episode: [9/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -229.781970% |  -9.611025  | 22.325315% | 255.900338%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [10/20]\n",
      "+--------------+-------------+-------------+--------------+\n",
      "| Total Return | Sharp Ratio |  Volatility | Max Drawdown |\n",
      "+--------------+-------------+-------------+--------------+\n",
      "| -330.865808% |  -0.494609  | 217.580213% | 200.390133%  |\n",
      "+--------------+-------------+-------------+--------------+\n",
      "Valid Episode: [10/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -211.175834% |  -6.628011  | 29.751737% | 233.672637%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [11/20]\n",
      "+--------------+-------------+-------------+--------------+\n",
      "| Total Return | Sharp Ratio |  Volatility | Max Drawdown |\n",
      "+--------------+-------------+-------------+--------------+\n",
      "| 1991.289944% |   3.599589  | 179.933059% | 204.087283%  |\n",
      "+--------------+-------------+-------------+--------------+\n",
      "Valid Episode: [11/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -228.830327% |  -9.353198  | 22.845717% | 255.211095%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [12/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| 742.982932%  |  94.768603  | 2.550021%  |  70.693294%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Valid Episode: [12/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -215.702309% |  -8.129846  | 24.775580% | 239.811572%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [13/20]\n",
      "+---------------+-------------+------------+--------------+\n",
      "|  Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+---------------+-------------+------------+--------------+\n",
      "| -1996.284997% |  -39.810048 | 16.310198% | 4213.554643% |\n",
      "+---------------+-------------+------------+--------------+\n",
      "Valid Episode: [13/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -215.702309% |  -8.129846  | 24.775580% | 239.811572%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [14/20]\n",
      "+---------------+-------------+------------+--------------+\n",
      "|  Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+---------------+-------------+------------+--------------+\n",
      "| -1104.233233% |  -45.238320 | 7.939328%  | 928.083442%  |\n",
      "+---------------+-------------+------------+--------------+\n",
      "Valid Episode: [14/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -200.464469% |  -6.617423  | 28.287844% | 245.142734%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [15/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| 2437.007091% |  339.203461 | 2.336823%  |  66.024765%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Valid Episode: [15/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -174.528258% |  -5.478848  | 29.745949% | 234.865801%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [16/20]\n",
      "+---------------+-------------+------------+--------------+\n",
      "|  Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+---------------+-------------+------------+--------------+\n",
      "| -1695.346350% |  -38.947767 | 14.158109% | 3613.916947% |\n",
      "+---------------+-------------+------------+--------------+\n",
      "Valid Episode: [16/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -172.840730% |  -5.408309  | 29.842549% | 236.346598%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [17/20]\n",
      "+---------------+-------------+------------+--------------+\n",
      "|  Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+---------------+-------------+------------+--------------+\n",
      "| -2055.136473% |  -51.341173 | 13.019799% | 4435.606206% |\n",
      "+---------------+-------------+------------+--------------+\n",
      "Valid Episode: [17/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -191.369968% |  -6.698765  | 26.676595% | 236.661944%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [18/20]\n",
      "+---------------+-------------+------------+--------------+\n",
      "|  Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+---------------+-------------+------------+--------------+\n",
      "| -1021.814628% |  -7.297505  | 45.543574% | 2032.494122% |\n",
      "+---------------+-------------+------------+--------------+\n",
      "Valid Episode: [18/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -191.412419% |  -6.162390  | 29.004959% | 245.194399%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [19/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -920.475519% |  -5.939832  | 50.404290% | 596.620478%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Valid Episode: [19/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -229.810164% |  -9.426909  | 22.764141% | 255.928532%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [20/20]\n",
      "+---------------+-------------+------------+--------------+\n",
      "|  Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+---------------+-------------+------------+--------------+\n",
      "| -3168.720438% |  -53.162281 | 19.386960% | 4312.459147% |\n",
      "+---------------+-------------+------------+--------------+\n",
      "Valid Episode: [20/20]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2023-03-02 13:37:23,414]\u001b[0m Trial 1 finished with value: 0.0 and parameters: {'explore_rate': 0.2, 'hidden_nodes': (128, 64)}. Best is trial 0 with value: 62326.314575009914.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -230.134541% |  -9.660176  | 22.245805% | 256.252909%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Config (path: /home/cqzong/TradeMaster/configs/algorithmic_trading/algorithmic_trading_BTC_deepscalper_deepscalper_adam_mse.py): {'data': {'type': 'AlgorithmicTradingDataset', 'data_path': 'data/algorithmic_trading/BTC', 'train_path': 'data/algorithmic_trading/BTC/train.csv', 'valid_path': 'data/algorithmic_trading/BTC/valid.csv', 'test_path': 'data/algorithmic_trading/BTC/test.csv', 'test_dynamic_path': 'data/algorithmic_trading/BTC/test_labeled_3_24_-0.15_0.15.csv', 'tech_indicator_list': ['high', 'low', 'open', 'close', 'adjcp', 'zopen', 'zhigh', 'zlow', 'zadjcp', 'zclose', 'zd_5', 'zd_10', 'zd_15', 'zd_20', 'zd_25', 'zd_30'], 'backward_num_day': 5, 'forward_num_day': 5, 'test_dynamic': '-1'}, 'environment': {'type': 'AlgorithmicTradingEnvironment'}, 'agent': {'type': 'AlgorithmicTradingDQN', 'max_step': 12345, 'reward_scale': 1, 'repeat_times': 1, 'gamma': 0.9, 'batch_size': 64, 'clip_grad_norm': 3.0, 'soft_update_tau': 0, 'state_value_tau': 0.005}, 'trainer': {'type': 'AlgorithmicTradingTrainer', 'epochs': 20, 'work_dir': 'work_dir/algorithmic_trading_BTC_deepscalper_deepscalper_adam_mse', 'seeds_list': (12345,), 'batch_size': 64, 'horizon_len': 128, 'buffer_size': 1000000.0, 'num_threads': 8, 'if_remove': False, 'if_discrete': True, 'if_off_policy': True, 'if_keep_save': True, 'if_over_write': False, 'if_save_buffer': False}, 'loss': {'type': 'MSELoss'}, 'optimizer': {'type': 'Adam', 'lr': 0.001}, 'act': {'type': 'QNet', 'state_dim': 82, 'action_dim': 3, 'dims': (64, 32), 'explore_rate': 0.2}, 'cri': None, 'transition': {'type': 'Transition'}, 'task_name': 'algorithmic_trading', 'dataset_name': 'BTC', 'optimizer_name': 'adam', 'loss_name': 'mse', 'net_name': 'deepscalper', 'agent_name': 'deepscalper', 'work_dir': 'work_dir/algorithmic_trading_BTC_deepscalper_deepscalper_adam_mse', 'batch_size': 64}\n",
      "| Arguments Keep work_dir: /home/cqzong/TradeMaster/work_dir/algorithmic_trading_BTC_deepscalper_deepscalper_adam_mse\n",
      "Train Episode: [1/20]\n",
      "+---------------+-------------+------------+--------------+\n",
      "|  Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+---------------+-------------+------------+--------------+\n",
      "| -1227.692613% |  -38.805900 | 10.290140% | 182.711888%  |\n",
      "+---------------+-------------+------------+--------------+\n",
      "Valid Episode: [1/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  -0.000000%  |  -0.000159  | 0.000000%  |  0.000000%   |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [2/20]\n",
      "+---------------+-------------+------------+--------------+\n",
      "|  Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+---------------+-------------+------------+--------------+\n",
      "| -1451.195365% |  -6.501880  | 72.596604% | 2879.850299% |\n",
      "+---------------+-------------+------------+--------------+\n",
      "Valid Episode: [2/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  -0.000000%  |  -0.000159  | 0.000000%  |  0.000000%   |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [3/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| 2553.705128% |  230.961385 | 3.596340%  |  82.865816%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Valid Episode: [3/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  -0.000000%  |  -0.000159  | 0.000000%  |  0.000000%   |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [4/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| 1089.456328% |  111.048676 | 3.190993%  |  78.705632%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Valid Episode: [4/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  -0.000000%  |  -0.000159  | 0.000000%  |  0.000000%   |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [5/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -884.143513% |  -5.203680  | 55.263905% | 1389.449672% |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Valid Episode: [5/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -127.531005% |  -1.639202  | 72.649836% | 164.862175%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [6/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| 2247.843053% |  226.640580 | 3.225951%  |  82.267887%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Valid Episode: [6/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  -0.000000%  |  -0.000159  | 0.000000%  |  0.000000%   |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [7/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| 1608.360103% |  177.496276 | 2.947294%  |  79.658299%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Valid Episode: [7/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  -0.000000%  |  -0.000159  | 0.000000%  |  0.000000%   |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [8/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| 145.941526%  |  19.277600  | 2.462381%  |  82.719997%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Valid Episode: [8/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  -0.000000%  |  -0.000159  | 0.000000%  |  0.000000%   |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [9/20]\n",
      "+--------------+-------------+-------------+--------------+\n",
      "| Total Return | Sharp Ratio |  Volatility | Max Drawdown |\n",
      "+--------------+-------------+-------------+--------------+\n",
      "| 3178.741763% |   7.194647  | 143.706079% | 108.837125%  |\n",
      "+--------------+-------------+-------------+--------------+\n",
      "Valid Episode: [9/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  26.475428%  |  20.494220  | 1.206321%  |  10.085728%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [10/20]\n",
      "+--------------+-------------+--------------+--------------+\n",
      "| Total Return | Sharp Ratio |  Volatility  | Max Drawdown |\n",
      "+--------------+-------------+--------------+--------------+\n",
      "| -443.698051% |  -0.030406  | 4746.380100% | 660.513221%  |\n",
      "+--------------+-------------+--------------+--------------+\n",
      "Valid Episode: [10/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  19.955979%  |  24.185952  | 0.770480%  |  6.278827%   |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [11/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| 2793.632019% |  262.751899 | 3.458221%  |  82.192177%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Valid Episode: [11/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  23.108687%  |  22.189838  | 0.972462%  |  7.544280%   |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [12/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| 2965.919200% |  280.866338 | 3.434702%  |  82.742907%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Valid Episode: [12/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  25.879111%  |  14.980203  | 1.613181%  |  19.642463%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [13/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| 2459.875101% |  237.339779 | 3.371102%  |  82.537049%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Valid Episode: [13/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  28.297712%  |  17.915010  | 1.474978%  |  15.361942%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [14/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| 3589.771941% |  360.466909 | 3.239148%  |  82.956871%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Valid Episode: [14/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  28.619951%  |  14.156757  | 1.887802%  |  23.933411%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [15/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -190.831869% |  -0.947283  | 65.524015% | 356.070886%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Valid Episode: [15/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  -0.000000%  |  -0.000159  | 0.000000%  |  0.000000%   |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [16/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| 166.393790%  |  17.191742  | 3.148086%  |  88.379715%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Valid Episode: [16/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  21.639735%  |  19.141254  | 1.055681%  |  10.051232%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [17/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| 219.445491%  |  12.232178  | 5.835153%  |  93.291937%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Valid Episode: [17/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  -0.000000%  |  -0.000159  | 0.000000%  |  0.000000%   |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [18/20]\n",
      "+---------------+-------------+------------+--------------+\n",
      "|  Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+---------------+-------------+------------+--------------+\n",
      "| -1314.661785% |  -28.568422 | 14.967773% | 3764.088506% |\n",
      "+---------------+-------------+------------+--------------+\n",
      "Valid Episode: [18/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  -0.000000%  |  -0.000159  | 0.000000%  |  0.000000%   |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [19/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  41.564233%  |   0.305778  | 44.212243% | 183.815798%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Valid Episode: [19/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  -0.000000%  |  -0.000159  | 0.000000%  |  0.000000%   |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [20/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| 352.133397%  |  39.314905  | 2.913261%  |  92.037194%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Valid Episode: [20/20]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2023-03-02 13:46:39,278]\u001b[0m Trial 2 finished with value: 47405.00923414003 and parameters: {'explore_rate': 0.2, 'hidden_nodes': (64, 32)}. Best is trial 0 with value: 62326.314575009914.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  26.185687%  |  15.326190  | 1.595442%  |  19.251972%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Config (path: /home/cqzong/TradeMaster/configs/algorithmic_trading/algorithmic_trading_BTC_deepscalper_deepscalper_adam_mse.py): {'data': {'type': 'AlgorithmicTradingDataset', 'data_path': 'data/algorithmic_trading/BTC', 'train_path': 'data/algorithmic_trading/BTC/train.csv', 'valid_path': 'data/algorithmic_trading/BTC/valid.csv', 'test_path': 'data/algorithmic_trading/BTC/test.csv', 'test_dynamic_path': 'data/algorithmic_trading/BTC/test_labeled_3_24_-0.15_0.15.csv', 'tech_indicator_list': ['high', 'low', 'open', 'close', 'adjcp', 'zopen', 'zhigh', 'zlow', 'zadjcp', 'zclose', 'zd_5', 'zd_10', 'zd_15', 'zd_20', 'zd_25', 'zd_30'], 'backward_num_day': 5, 'forward_num_day': 5, 'test_dynamic': '-1'}, 'environment': {'type': 'AlgorithmicTradingEnvironment'}, 'agent': {'type': 'AlgorithmicTradingDQN', 'max_step': 12345, 'reward_scale': 1, 'repeat_times': 1, 'gamma': 0.9, 'batch_size': 64, 'clip_grad_norm': 3.0, 'soft_update_tau': 0, 'state_value_tau': 0.005}, 'trainer': {'type': 'AlgorithmicTradingTrainer', 'epochs': 20, 'work_dir': 'work_dir/algorithmic_trading_BTC_deepscalper_deepscalper_adam_mse', 'seeds_list': (12345,), 'batch_size': 64, 'horizon_len': 128, 'buffer_size': 1000000.0, 'num_threads': 8, 'if_remove': False, 'if_discrete': True, 'if_off_policy': True, 'if_keep_save': True, 'if_over_write': False, 'if_save_buffer': False}, 'loss': {'type': 'MSELoss'}, 'optimizer': {'type': 'Adam', 'lr': 0.001}, 'act': {'type': 'QNet', 'state_dim': 82, 'action_dim': 3, 'dims': (64, 32), 'explore_rate': 0.25}, 'cri': None, 'transition': {'type': 'Transition'}, 'task_name': 'algorithmic_trading', 'dataset_name': 'BTC', 'optimizer_name': 'adam', 'loss_name': 'mse', 'net_name': 'deepscalper', 'agent_name': 'deepscalper', 'work_dir': 'work_dir/algorithmic_trading_BTC_deepscalper_deepscalper_adam_mse', 'batch_size': 64}\n",
      "| Arguments Keep work_dir: /home/cqzong/TradeMaster/work_dir/algorithmic_trading_BTC_deepscalper_deepscalper_adam_mse\n",
      "Train Episode: [1/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| 723.935829%  |  14.501896  | 16.236958% | 1332.625841% |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Valid Episode: [1/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  34.458228%  |   8.519827  | 3.776709%  |  51.375878%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [2/20]\n",
      "+--------------+-------------+-------------+--------------+\n",
      "| Total Return | Sharp Ratio |  Volatility | Max Drawdown |\n",
      "+--------------+-------------+-------------+--------------+\n",
      "| 153.789484%  |   0.125799  | 397.629096% | 146.664344%  |\n",
      "+--------------+-------------+-------------+--------------+\n",
      "Valid Episode: [2/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  29.169858%  |   8.226430  | 3.311115%  |  45.111550%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [3/20]\n",
      "+--------------+-------------+-------------+--------------+\n",
      "| Total Return | Sharp Ratio |  Volatility | Max Drawdown |\n",
      "+--------------+-------------+-------------+--------------+\n",
      "| 396.159754%  |   0.916931  | 140.528045% | 137.795561%  |\n",
      "+--------------+-------------+-------------+--------------+\n",
      "Valid Episode: [3/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  28.175734%  |   7.165852  | 3.671629%  |  50.877754%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [4/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| 341.530819%  |   4.996744  | 22.231675% | 100.792688%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Valid Episode: [4/20]\n",
      "+--------------+-------------+-------------+--------------+\n",
      "| Total Return | Sharp Ratio |  Volatility | Max Drawdown |\n",
      "+--------------+-------------+-------------+--------------+\n",
      "| -50.063761%  |  -0.101190  | 461.995724% | 127.480255%  |\n",
      "+--------------+-------------+-------------+--------------+\n",
      "Train Episode: [5/20]\n",
      "+--------------+-------------+-------------+--------------+\n",
      "| Total Return | Sharp Ratio |  Volatility | Max Drawdown |\n",
      "+--------------+-------------+-------------+--------------+\n",
      "| -438.389201% |  -0.597701  | 238.564235% | 174.191839%  |\n",
      "+--------------+-------------+-------------+--------------+\n",
      "Valid Episode: [5/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -230.134541% |  -9.660176  | 22.245805% | 256.252909%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [6/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| 206.614615%  |  38.366169  | 1.751628%  |  66.655327%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Valid Episode: [6/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -229.767118% |  -9.608902  | 22.328805% | 255.885486%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [7/20]\n",
      "+---------------+-------------+-------------+--------------+\n",
      "|  Total Return | Sharp Ratio |  Volatility | Max Drawdown |\n",
      "+---------------+-------------+-------------+--------------+\n",
      "| -1303.422235% |  -2.893741  | 146.505799% | 2764.336071% |\n",
      "+---------------+-------------+-------------+--------------+\n",
      "Valid Episode: [7/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -229.488351% |  -9.461009  | 22.650330% | 255.606719%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [8/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| 476.354347%  |   2.526974  | 61.313867% | 178.924191%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Valid Episode: [8/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -240.062838% |  -9.335660  | 24.012161% | 268.190311%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [9/20]\n",
      "+--------------+-------------+-------------+--------------+\n",
      "| Total Return | Sharp Ratio |  Volatility | Max Drawdown |\n",
      "+--------------+-------------+-------------+--------------+\n",
      "| -874.482606% |  -1.978899  | 143.733130% | 1190.227009% |\n",
      "+--------------+-------------+-------------+--------------+\n",
      "Valid Episode: [9/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -231.152237% |  -8.085138  | 26.696974% | 257.179500%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [10/20]\n",
      "+--------------+-------------+-------------+--------------+\n",
      "| Total Return | Sharp Ratio |  Volatility | Max Drawdown |\n",
      "+--------------+-------------+-------------+--------------+\n",
      "| 983.420918%  |   1.406512  | 227.418294% | 331.348226%  |\n",
      "+--------------+-------------+-------------+--------------+\n",
      "Valid Episode: [10/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -210.594539% |  -8.507908  | 23.114031% | 234.703802%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [11/20]\n",
      "+--------------+-------------+-------------+--------------+\n",
      "| Total Return | Sharp Ratio |  Volatility | Max Drawdown |\n",
      "+--------------+-------------+-------------+--------------+\n",
      "| -431.619703% |  -1.396051  | 100.560944% | 172.424093%  |\n",
      "+--------------+-------------+-------------+--------------+\n",
      "Valid Episode: [11/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -229.595897% |  -9.601337  | 22.329744% | 255.714265%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [12/20]\n",
      "+---------------+-------------+-------------+--------------+\n",
      "|  Total Return | Sharp Ratio |  Volatility | Max Drawdown |\n",
      "+---------------+-------------+-------------+--------------+\n",
      "| -1112.357135% |  -1.545945  | 234.034399% | 1297.200297% |\n",
      "+---------------+-------------+-------------+--------------+\n",
      "Valid Episode: [12/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -204.748469% |  -6.694469  | 28.559846% | 251.435839%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [13/20]\n",
      "+---------------+-------------+------------+--------------+\n",
      "|  Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+---------------+-------------+------------+--------------+\n",
      "| -2170.499911% |  -7.476380  | 94.427354% | 4683.701471% |\n",
      "+---------------+-------------+------------+--------------+\n",
      "Valid Episode: [13/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -204.108030% |  -6.658392  | 28.624775% | 250.795400%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [14/20]\n",
      "+---------------+-------------+-------------+--------------+\n",
      "|  Total Return | Sharp Ratio |  Volatility | Max Drawdown |\n",
      "+---------------+-------------+-------------+--------------+\n",
      "| -1898.969388% |  -0.985109  | 626.993321% | 3989.499587% |\n",
      "+---------------+-------------+-------------+--------------+\n",
      "Valid Episode: [14/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -135.372781% |  -2.451739  | 51.559469% | 186.855352%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [15/20]\n",
      "+---------------+-------------+------------+--------------+\n",
      "|  Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+---------------+-------------+------------+--------------+\n",
      "| -2580.562353% |  -17.634232 | 47.597835% | 5009.434629% |\n",
      "+---------------+-------------+------------+--------------+\n",
      "Valid Episode: [15/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -229.798444% |  -9.598953  | 22.354995% | 255.916812%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [16/20]\n",
      "+---------------+-------------+------------+--------------+\n",
      "|  Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+---------------+-------------+------------+--------------+\n",
      "| -1515.823451% |  -26.639736 | 18.507514% | 3767.726994% |\n",
      "+---------------+-------------+------------+--------------+\n",
      "Valid Episode: [16/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  36.243173%  |   8.935169  | 3.787693%  |  50.707941%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [17/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -707.078142% |  -6.329213  | 36.336837% | 1299.305151% |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Valid Episode: [17/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -214.557299% |  -7.909450  | 25.330769% | 238.666562%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [18/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| 174.232021%  |   0.732168  | 77.401021% | 214.871638%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Valid Episode: [18/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  36.243173%  |   8.935169  | 3.787693%  |  50.707941%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [19/20]\n",
      "+---------------+-------------+--------------+--------------+\n",
      "|  Total Return | Sharp Ratio |  Volatility  | Max Drawdown |\n",
      "+---------------+-------------+--------------+--------------+\n",
      "| -1037.189447% |  -0.064948  | 5194.259572% | 1580.838098% |\n",
      "+---------------+-------------+--------------+--------------+\n",
      "Valid Episode: [19/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -72.324010%  |  -16.929096 | 3.989331%  |  80.360431%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [20/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -555.852819% |  -17.477805 | 10.344329% | 796.605692%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Valid Episode: [20/20]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2023-03-02 13:56:33,372]\u001b[0m Trial 3 finished with value: 56390.91407490198 and parameters: {'explore_rate': 0.25, 'hidden_nodes': (64, 32)}. Best is trial 0 with value: 62326.314575009914.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------+-------------+--------------+--------------+\n",
      "| Total Return | Sharp Ratio |  Volatility  | Max Drawdown |\n",
      "+--------------+-------------+--------------+--------------+\n",
      "| -52.609006%  |  -0.026569  | 1848.999094% | 146.310408%  |\n",
      "+--------------+-------------+--------------+--------------+\n",
      "Config (path: /home/cqzong/TradeMaster/configs/algorithmic_trading/algorithmic_trading_BTC_deepscalper_deepscalper_adam_mse.py): {'data': {'type': 'AlgorithmicTradingDataset', 'data_path': 'data/algorithmic_trading/BTC', 'train_path': 'data/algorithmic_trading/BTC/train.csv', 'valid_path': 'data/algorithmic_trading/BTC/valid.csv', 'test_path': 'data/algorithmic_trading/BTC/test.csv', 'test_dynamic_path': 'data/algorithmic_trading/BTC/test_labeled_3_24_-0.15_0.15.csv', 'tech_indicator_list': ['high', 'low', 'open', 'close', 'adjcp', 'zopen', 'zhigh', 'zlow', 'zadjcp', 'zclose', 'zd_5', 'zd_10', 'zd_15', 'zd_20', 'zd_25', 'zd_30'], 'backward_num_day': 5, 'forward_num_day': 5, 'test_dynamic': '-1'}, 'environment': {'type': 'AlgorithmicTradingEnvironment'}, 'agent': {'type': 'AlgorithmicTradingDQN', 'max_step': 12345, 'reward_scale': 1, 'repeat_times': 1, 'gamma': 0.9, 'batch_size': 64, 'clip_grad_norm': 3.0, 'soft_update_tau': 0, 'state_value_tau': 0.005}, 'trainer': {'type': 'AlgorithmicTradingTrainer', 'epochs': 20, 'work_dir': 'work_dir/algorithmic_trading_BTC_deepscalper_deepscalper_adam_mse', 'seeds_list': (12345,), 'batch_size': 64, 'horizon_len': 128, 'buffer_size': 1000000.0, 'num_threads': 8, 'if_remove': False, 'if_discrete': True, 'if_off_policy': True, 'if_keep_save': True, 'if_over_write': False, 'if_save_buffer': False}, 'loss': {'type': 'MSELoss'}, 'optimizer': {'type': 'Adam', 'lr': 0.001}, 'act': {'type': 'QNet', 'state_dim': 82, 'action_dim': 3, 'dims': (128, 64), 'explore_rate': 0.15}, 'cri': None, 'transition': {'type': 'Transition'}, 'task_name': 'algorithmic_trading', 'dataset_name': 'BTC', 'optimizer_name': 'adam', 'loss_name': 'mse', 'net_name': 'deepscalper', 'agent_name': 'deepscalper', 'work_dir': 'work_dir/algorithmic_trading_BTC_deepscalper_deepscalper_adam_mse', 'batch_size': 64}\n",
      "| Arguments Keep work_dir: /home/cqzong/TradeMaster/work_dir/algorithmic_trading_BTC_deepscalper_deepscalper_adam_mse\n",
      "Train Episode: [1/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| 1339.715669% |  132.544945 | 3.287599%  |  79.509789%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Valid Episode: [1/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  36.243173%  |   8.935169  | 3.787693%  |  50.707941%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [2/20]\n",
      "+---------------+-------------+------------+--------------+\n",
      "|  Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+---------------+-------------+------------+--------------+\n",
      "| -2155.769920% |  -24.900422 | 28.159510% | 4625.996043% |\n",
      "+---------------+-------------+------------+--------------+\n",
      "Valid Episode: [2/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  36.440345%  |  12.663205  | 2.687139%  |  36.573957%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [3/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -935.021143% |  -5.190204  | 58.595783% | 3093.157076% |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Valid Episode: [3/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  39.144488%  |  13.852160  | 2.638788%  |  35.485706%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [4/20]\n",
      "+--------------+-------------+-------------+--------------+\n",
      "| Total Return | Sharp Ratio |  Volatility | Max Drawdown |\n",
      "+--------------+-------------+-------------+--------------+\n",
      "| 757.309688%  |   0.448320  | 549.433424% | 1662.466193% |\n",
      "+--------------+-------------+-------------+--------------+\n",
      "Valid Episode: [4/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  5.056221%   |   0.275712  | 17.124631% |  62.129153%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [5/20]\n",
      "+---------------+-------------+------------+--------------+\n",
      "|  Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+---------------+-------------+------------+--------------+\n",
      "| -1837.489507% |  -31.839621 | 18.770955% | 3196.478624% |\n",
      "+---------------+-------------+------------+--------------+\n",
      "Valid Episode: [5/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  30.758165%  |   9.080536  | 3.163008%  |  44.158719%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [6/20]\n",
      "+---------------+-------------+------------+--------------+\n",
      "|  Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+---------------+-------------+------------+--------------+\n",
      "| -2628.363735% | -133.355162 | 6.410694%  | 3830.232023% |\n",
      "+---------------+-------------+------------+--------------+\n",
      "Valid Episode: [6/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  29.912781%  |   8.759810  | 3.188698%  |  44.806665%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [7/20]\n",
      "+--------------+-------------+-------------+--------------+\n",
      "| Total Return | Sharp Ratio |  Volatility | Max Drawdown |\n",
      "+--------------+-------------+-------------+--------------+\n",
      "|  63.815742%  |   0.031300  | 663.143668% | 2937.854288% |\n",
      "+--------------+-------------+-------------+--------------+\n",
      "Valid Episode: [7/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  25.005963%  |   9.064706  | 2.575972%  |  36.763242%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [8/20]\n",
      "+---------------+-------------+------------+--------------+\n",
      "|  Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+---------------+-------------+------------+--------------+\n",
      "| -2336.093900% |  -46.114287 | 16.477229% | 6317.065783% |\n",
      "+---------------+-------------+------------+--------------+\n",
      "Valid Episode: [8/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  36.243173%  |   8.935169  | 3.787693%  |  50.707941%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [9/20]\n",
      "+--------------+-------------+-------------+--------------+\n",
      "| Total Return | Sharp Ratio |  Volatility | Max Drawdown |\n",
      "+--------------+-------------+-------------+--------------+\n",
      "| -678.799911% |  -2.104784  | 104.897118% | 4197.417477% |\n",
      "+--------------+-------------+-------------+--------------+\n",
      "Valid Episode: [9/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  31.109725%  |   7.844726  | 3.703137%  |  50.877754%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [10/20]\n",
      "+---------------+-------------+------------+--------------+\n",
      "|  Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+---------------+-------------+------------+--------------+\n",
      "| -3486.238551% |  -24.716921 | 45.876695% | 7243.330456% |\n",
      "+---------------+-------------+------------+--------------+\n",
      "Valid Episode: [10/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  36.007137%  |   8.893701  | 3.780571%  |  50.707941%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [11/20]\n",
      "+--------------+-------------+-------------+--------------+\n",
      "| Total Return | Sharp Ratio |  Volatility | Max Drawdown |\n",
      "+--------------+-------------+-------------+--------------+\n",
      "| 638.492280%  |   0.818409  | 253.755147% | 464.239696%  |\n",
      "+--------------+-------------+-------------+--------------+\n",
      "Valid Episode: [11/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  36.243173%  |   8.935169  | 3.787693%  |  50.707941%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [12/20]\n",
      "+---------------+-------------+------------+--------------+\n",
      "|  Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+---------------+-------------+------------+--------------+\n",
      "| -1566.578592% |  -6.452058  | 78.973849% | 2583.554316% |\n",
      "+---------------+-------------+------------+--------------+\n",
      "Valid Episode: [12/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  35.894527%  |   7.122431  | 4.705994%  |  50.837038%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [13/20]\n",
      "+--------------+-------------+-------------+--------------+\n",
      "| Total Return | Sharp Ratio |  Volatility | Max Drawdown |\n",
      "+--------------+-------------+-------------+--------------+\n",
      "| -457.606944% |  -1.458517  | 102.049431% | 864.767003%  |\n",
      "+--------------+-------------+-------------+--------------+\n",
      "Valid Episode: [13/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -12.018446%  |  -0.142602  | 78.699708% | 100.758509%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [14/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| 514.647279%  |   2.161081  | 77.458299% | 138.733453%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Valid Episode: [14/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -230.134541% |  -9.660176  | 22.245805% | 256.252909%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [15/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| 973.233546%  |  113.226899 | 2.795740%  |  78.057661%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Valid Episode: [15/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  22.571188%  |   5.124698  | 4.112799%  |  58.191024%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [16/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| 917.151770%  |  25.323542  | 11.780024% | 108.874788%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Valid Episode: [16/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -208.681248% |  -2.842102  | 68.563843% | 230.371292%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [17/20]\n",
      "+--------------+-------------+-------------+--------------+\n",
      "| Total Return | Sharp Ratio |  Volatility | Max Drawdown |\n",
      "+--------------+-------------+-------------+--------------+\n",
      "| 589.680750%  |   0.601715  | 318.753771% | 168.621037%  |\n",
      "+--------------+-------------+-------------+--------------+\n",
      "Valid Episode: [17/20]\n",
      "+--------------+-------------+-------------+--------------+\n",
      "| Total Return | Sharp Ratio |  Volatility | Max Drawdown |\n",
      "+--------------+-------------+-------------+--------------+\n",
      "| -46.539682%  |  -0.076217  | 570.192436% | 120.062273%  |\n",
      "+--------------+-------------+-------------+--------------+\n",
      "Train Episode: [18/20]\n",
      "+---------------+-------------+------------+--------------+\n",
      "|  Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+---------------+-------------+------------+--------------+\n",
      "| -1132.438596% |  -26.331469 | 13.988430% | 1983.507060% |\n",
      "+---------------+-------------+------------+--------------+\n",
      "Valid Episode: [18/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -229.488351% |  -9.461009  | 22.650330% | 255.606719%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [19/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| 1986.102269% |  219.533373 | 2.942595%  |  79.413669%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Valid Episode: [19/20]\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -230.134541% |  -9.660176  | 22.245805% | 256.252909%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Train Episode: [20/20]\n",
      "+---------------+-------------+------------+--------------+\n",
      "|  Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+---------------+-------------+------------+--------------+\n",
      "| -2127.814861% |  -16.568515 | 41.771459% | 4816.894884% |\n",
      "+---------------+-------------+------------+--------------+\n",
      "Valid Episode: [20/20]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2023-03-02 14:06:45,936]\u001b[0m Trial 4 finished with value: 68162.89112228403 and parameters: {'explore_rate': 0.15, 'hidden_nodes': (128, 64)}. Best is trial 4 with value: 68162.89112228403.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "|  42.588022%  |  11.648698  | 3.413984%  |  43.602653%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "Best trial:\n",
      "  Value:  68162.89112228403\n",
      "  Params: \n",
      "    explore_rate: 0.15\n",
      "    hidden_nodes: (128, 64)\n",
      "  Best trial number:  4\n",
      "Config (path: /home/cqzong/TradeMaster/configs/algorithmic_trading/algorithmic_trading_BTC_deepscalper_deepscalper_adam_mse.py): {'data': {'type': 'AlgorithmicTradingDataset', 'data_path': 'data/algorithmic_trading/BTC', 'train_path': 'data/algorithmic_trading/BTC/train.csv', 'valid_path': 'data/algorithmic_trading/BTC/valid.csv', 'test_path': 'data/algorithmic_trading/BTC/test.csv', 'test_dynamic_path': 'data/algorithmic_trading/BTC/test_labeled_3_24_-0.15_0.15.csv', 'tech_indicator_list': ['high', 'low', 'open', 'close', 'adjcp', 'zopen', 'zhigh', 'zlow', 'zadjcp', 'zclose', 'zd_5', 'zd_10', 'zd_15', 'zd_20', 'zd_25', 'zd_30'], 'backward_num_day': 5, 'forward_num_day': 5, 'test_dynamic': '-1'}, 'environment': {'type': 'AlgorithmicTradingEnvironment'}, 'agent': {'type': 'AlgorithmicTradingDQN', 'max_step': 12345, 'reward_scale': 1, 'repeat_times': 1, 'gamma': 0.9, 'batch_size': 64, 'clip_grad_norm': 3.0, 'soft_update_tau': 0, 'state_value_tau': 0.005}, 'trainer': {'type': 'AlgorithmicTradingTrainer', 'epochs': 20, 'work_dir': 'work_dir/algorithmic_trading_BTC_deepscalper_deepscalper_adam_mse', 'seeds_list': (12345,), 'batch_size': 64, 'horizon_len': 128, 'buffer_size': 1000000.0, 'num_threads': 8, 'if_remove': False, 'if_discrete': True, 'if_off_policy': True, 'if_keep_save': True, 'if_over_write': False, 'if_save_buffer': False}, 'loss': {'type': 'MSELoss'}, 'optimizer': {'type': 'Adam', 'lr': 0.001}, 'act': {'type': 'QNet', 'state_dim': 82, 'action_dim': 3, 'dims': (128, 64), 'explore_rate': 0.15}, 'cri': None, 'transition': {'type': 'Transition'}, 'task_name': 'algorithmic_trading', 'dataset_name': 'BTC', 'optimizer_name': 'adam', 'loss_name': 'mse', 'net_name': 'deepscalper', 'agent_name': 'deepscalper', 'work_dir': 'work_dir/algorithmic_trading_BTC_deepscalper_deepscalper_adam_mse', 'batch_size': 64}\n",
      "| Arguments Keep work_dir: /home/cqzong/TradeMaster/work_dir/algorithmic_trading_BTC_deepscalper_deepscalper_adam_mse\n",
      "Resume checkpoint /home/cqzong/TradeMaster/work_dir/algorithmic_trading_BTC_deepscalper_deepscalper_adam_mse/checkpoints/best.pth\n",
      "Test Best Episode\n",
      "+--------------+-------------+------------+--------------+\n",
      "| Total Return | Sharp Ratio | Volatility | Max Drawdown |\n",
      "+--------------+-------------+------------+--------------+\n",
      "| -240.799357% |  -2.589529  | 86.833210% | 329.778719%  |\n",
      "+--------------+-------------+------------+--------------+\n",
      "train end\n"
     ]
    }
   ],
   "source": [
    "args = parse_args()\n",
    "cfg = Config.fromfile(args.config)\n",
    "\n",
    "if args.task_name.startswith(\"train\"):\n",
    "    study = optuna.create_study(direction=\"maximize\")\n",
    "    study.optimize(objective, n_trials=args.n_trials)\n",
    "    print(\"Best trial:\")\n",
    "    trial = study.best_trial\n",
    "    print(\"  Value: \", trial.value)\n",
    "\n",
    "    print(\"  Params: \")\n",
    "    for key, value in trial.params.items():\n",
    "        print(\"    {}: {}\".format(key, value))\n",
    "    print(\"  Best trial number: \", trial.number)\n",
    "    best_model_path = os.path.join(ROOT, cfg.work_dir, \"checkpoints\", \"trial-{:05d}.pth\".format(trial.number))\n",
    "    copyfile(best_model_path, os.path.join(ROOT, cfg.work_dir, \"checkpoints\", \"best.pth\"))\n",
    "    joblib.dump(study, os.path.join(ROOT, cfg.work_dir, \"auto_tuning.pkl\"))\n",
    "    trainer = test_dqn_builder(args, sampled_params = sample_params(study.best_trial))\n",
    "    trainer.test()\n",
    "    print(\"train end\")\n",
    "elif args.task_name.startswith(\"test\"):\n",
    "    study = joblib.load(os.path.join(ROOT, cfg.work_dir, \"auto_tuning.pkl\"))\n",
    "    trainer = test_dqn_builder(args, sampled_params = sample_params(study.best_trial))\n",
    "    trainer.test()\n",
    "    print(\"test end\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "trademaster",
   "language": "python",
   "name": "trademaster"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}