diff --git a/cases/template.result b/cases/template.result index 09640b244..bd6e4171c 100644 --- a/cases/template.result +++ b/cases/template.result @@ -1,3 +1,4 @@ +select sleep(10); create table t1 ( a int not null default 1, int32 int primary key); insert into t1 (int32) values (-1),(1234567890),(2147483647); insert into t1 (int32) values (-1),(1234567890),(2147483647); diff --git a/cases/template.sql b/cases/template.sql index 85aaf2fdc..d8f9e1c13 100644 --- a/cases/template.sql +++ b/cases/template.sql @@ -1,3 +1,4 @@ +select sleep(10); create table t1 ( a int not null default 1, int32 int primary key); insert into t1 (int32) values (-1),(1234567890),(2147483647); -- @pattern diff --git a/lib/mo-tester-1.0-SNAPSHOT.jar b/lib/mo-tester-1.0-SNAPSHOT.jar index edcbb88f2..c545aa25f 100644 Binary files a/lib/mo-tester-1.0-SNAPSHOT.jar and b/lib/mo-tester-1.0-SNAPSHOT.jar differ diff --git a/mo.yml b/mo.yml index 4ede5a474..63b07e7b7 100755 --- a/mo.yml +++ b/mo.yml @@ -21,3 +21,8 @@ jdbc: user: name: "dump" password: "111" + +#debug info +debug: + serverIP: "127.0.0.1" + port: 6060 diff --git a/pprof.sh b/pprof.sh new file mode 100755 index 000000000..58d559a72 --- /dev/null +++ b/pprof.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +if [ $# -eq 0 ] ; then + echo "No parameters provided,please use -H to get help. " >&2 + exit 2 +fi + +PORT=6060 +SERVER="127.0.0.1" +DURATION=30 +SLEEP=60 +WORKSPACE=$(cd `dirname $0`; pwd) + +while getopts ":h:p:d:s:H" opt +do + case $opt in + h) + SERVER="${OPTARG}" + echo -e "`date +'%Y-%m-%d %H:%M:%S'` The server addr is : ${OPTARG}" + ;; + p) + expr ${OPTARG} "+" 10 &> /dev/null + if [ $? -ne 0 ]; then + echo "`date +'%Y-%m-%d %H:%M:%S'` The port [${OPTARG}] is not a number" + exit 1 + fi + PORT=${OPTARG} + echo -e "`date +'%Y-%m-%d %H:%M:%S'` The debug port is : ${OPTARG}" + ;; + d) + DURATION=${OPTARG} + echo -e "`date +'%Y-%m-%d %H:%M:%S'` The collection duration is : ${OPTARG} s" + ;; + s) + SLEEP=${OPTARG} + echo -e "`date +'%Y-%m-%d %H:%M:%S'` The sleep time for each turn is : ${OPTARG} s" + ;; + H) + echo -e "Usage:　bash pprof.sh [option] [param] ...\nExcute pprof task" + echo -e " -h server address, default value is 127.0.0.1" + echo -e " -p server port, default value is 6060" + echo -e " -d the duration that pprof collection will last once, unit s" + echo -e " -s the sleeptime for each turn, unit s" + echo -e "Examples:" + echo " bash pprof.sh -h 127.0.0.1 -p 6060 -d 10" + echo "For more support,please email to sudong@matrixorigin.cn" + exit 1 + ;; + ?) + echo "Unkown parameter,please use -H to get help." + exit 1;; + esac +done + +if [ ! -d ${WORKSPACE}/report/prof/${SERVER} ] ; then + mkdir ${WORKSPACE}/report/prof/${SERVER} +fi + +time=`date +'%Y-%m-%d_%H:%M:%S'` +mkdir -p ${WORKSPACE}/report/prof/${SERVER}/${time} +curl http://${SERVER}:${PORT}/debug/pprof/goroutine?debug=2 -o ${WORKSPACE}/report/prof/${SERVER}/${time}/goroutine.log +curl http://${SERVER}:${PORT}/debug/pprof/trace?seconds=30 -o ${WORKSPACE}/report/prof/${SERVER}/${time}/trace.out +if [ $? -ne 0 ];then + echo -e "`date +'%Y-%m-%d_%H:%M:%S'` The MO debug service can not be reached, the pprof operation was failed." + exit 1 +fi + diff --git a/run.sh b/run.sh index a77c2d0b5..6cca4966c 100755 --- a/run.sh +++ b/run.sh @@ -7,7 +7,7 @@ fi TIMES=1 -while getopts ":p:m:t:r:i:e:s:gfnch" opt +while getopts ":p:m:t:r:i:e:s:ogfnch" opt do case $opt in p) @@ -58,6 +58,10 @@ do CHECK="check" echo -e "The meta data of the resultset will be ignored when comparing the resut" ;; + o) + PPROF="pprof" + echo -e "If a query timeout, mo-tester will collect the pprof info from mo" + ;; h) echo -e "Usage:　bash run.sh [option] [param] ...\nExcute test cases task" echo -e " -p set the path of test cases needed to be executed by mo-tester" @@ -75,6 +79,7 @@ do echo -e " Those two sql commands are associated with the issue#3236,and they will not been executed in bvt test,until the flag is removed when the issue#3236 is fixed." echo -e " -n means the meta data of the resultset will be ignored when comparing the resut" echo -e " -c check whether the case scripts match the result file" + echo -e " -o if a query timeout, mo-tester will collect the pprof info from mo" echo -e "Examples:" echo " bash run.sh -p case -m run -t script -r 100 -i select,subquery -e substring -g" echo "For more support,please email to dong.su@matrixorigin.io" @@ -104,7 +109,7 @@ if [ ${TIMES} -eq 1 ]; then java -Xms1024M -Xmx1024M -cp ${libJars} \ -Dconf.yml=${MO_YAML} \ -Drun.yml=${RUN_YAML} \ - io.mo.Tester ${PATHC} ${METHOD} ${TYPE} ${RATE} ${INCLUDE} ${EXCLUDE} ${IGNORE} ${NOMETA} ${CHECK} ${RESOURCE} ${FORCE} + io.mo.Tester ${PATHC} ${METHOD} ${TYPE} ${RATE} ${INCLUDE} ${EXCLUDE} ${IGNORE} ${NOMETA} ${CHECK} ${RESOURCE} ${FORCE} ${PPROF} else echo "This test will be run for ${TIMES} times" for i in $(seq 1 ${TIMES}) @@ -113,7 +118,7 @@ else java -Xms1024M -Xmx1024M -cp ${libJars} \ -Dconf.yml=${MO_YAML} \ -Drun.yml=${RUN_YAML} \ - io.mo.Tester ${PATHC} ${METHOD} ${TYPE} ${RATE} ${INCLUDE} ${EXCLUDE} ${IGNORE} ${NOMETA} ${CHECK} ${RESOURCE} ${FORCE} + io.mo.Tester ${PATHC} ${METHOD} ${TYPE} ${RATE} ${INCLUDE} ${EXCLUDE} ${IGNORE} ${NOMETA} ${CHECK} ${RESOURCE} ${FORCE} ${PPROF} echo "The ${i} turn test has ended, and test report is in ./report/${i} dir." | tee -a ${WORKSPACE}/run.log mkdir -p ${WORKSPACE}/${MOTESTER_DIR}/report/${i}/ mv ${WORKSPACE}/${MOTESTER_DIR}/report/*.txt ${WORKSPACE}/${MOTESTER_DIR}/report/${i}/ diff --git a/src/main/java/io/mo/Tester.java b/src/main/java/io/mo/Tester.java index 6f3c2a148..cd9de4e00 100755 --- a/src/main/java/io/mo/Tester.java +++ b/src/main/java/io/mo/Tester.java @@ -118,6 +118,11 @@ public static void main(String[] args){ if (arg.equalsIgnoreCase("check")) { method = "check"; } + + //get pprof info + if (arg.equalsIgnoreCase("pprof")) { + COMMON.NEEDPPROF = true; + } } } diff --git a/src/main/java/io/mo/constant/COMMON.java b/src/main/java/io/mo/constant/COMMON.java index e8dcd08c1..e2109467a 100755 --- a/src/main/java/io/mo/constant/COMMON.java +++ b/src/main/java/io/mo/constant/COMMON.java @@ -62,6 +62,8 @@ public class COMMON { public static String START_TRX = "begin"; public static String COMMIT_TRX = "commit"; public static String ROLLBACK_TRX = "rollback"; + + public static boolean NEEDPPROF = false; public static String[] SPECIAL_CHARS = new String[]{"+","-","*","/","%","&",">","<","(",")","!","=","\'","\""}; diff --git a/src/main/java/io/mo/db/Executor.java b/src/main/java/io/mo/db/Executor.java index 07edd952e..b1b6d32ba 100755 --- a/src/main/java/io/mo/db/Executor.java +++ b/src/main/java/io/mo/db/Executor.java @@ -203,6 +203,12 @@ public static void run(TestScript script){ LOG.error("[EXPECT RESULT]:\n" + command.getTestResult().getExpResult()); LOG.error("[ACTUAL RESULT]:\n" + command.getTestResult().getActResult()); + if(COMMON.NEEDPPROF){ + LOG.info("Start to collect pprof information,please wait........"); + pprof(); + LOG.info("Finish to collect pprof information,the test will continue"); + } + //reconnect to mo, and set db to last use db LOG.warn(String.format("The mo-tester tries to re-connect to mo, con[id=%d, user=%s, pwd=%s, db=%s], please wait.....", command.getConn_id(),command.getConn_user(),command.getConn_pswd(),command.getUseDB())); @@ -657,6 +663,68 @@ public static void syncCommit(Connection connection){ } } + public static void pprof(){ + String[] debugServers = MoConfUtil.getDebugServers(); + if(debugServers != null){ + int port = MoConfUtil.getDebugPort(); + Thread[] threads = new Thread[debugServers.length]; + for(int i = 0; i < debugServers.length;i++){ + int index = i; + threads[index] = new Thread(new Runnable() { + @Override + public void run() { + try { + Process p = Runtime.getRuntime().exec(String.format("./pprof.sh -h %s -p %d",debugServers[index],port)); + InputStream is = p.getInputStream(); + BufferedReader reader = new BufferedReader(new InputStreamReader(is)); + p.waitFor(); + + StringBuffer execResut = new StringBuffer(); + String str = reader.readLine(); + while ( str != null) { + execResut.append(str+"\n"); + str = reader.readLine(); + } + + if (p.exitValue() != 0) { + LOG.error(String.format("The pprof operation has been executed failed.\n%s",execResut.toString())); + }else { + LOG.info(String.format("The pprof operation has been executed successfully.\n%s", execResut.toString())); + LOG.info(String.format("The result is in the dir ./report/pprof/%s/",debugServers[index])); + } + + } catch (IOException e) { + LOG.error("The pprof operation has been executed failed."); + LOG.error(String.format("The output of pprof operation is \n%s.",e.getMessage())); + } catch (InterruptedException e) { + LOG.error("The pprof operation has been executed failed."); + LOG.error(String.format("The output of pprof operation is \n%s.",e.getMessage())); + } + } + }); + threads[index].start(); + } + + boolean finished = false; + while(!finished){ + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + + for(Thread thread:threads){ + if(thread.isAlive()){ + finished = false; + break; + } + finished = true; + } + } + } + + + } public static void executeSysCmd(String cmd){ try { diff --git a/src/main/java/io/mo/util/MoConfUtil.java b/src/main/java/io/mo/util/MoConfUtil.java index ae275abcc..c04787424 100755 --- a/src/main/java/io/mo/util/MoConfUtil.java +++ b/src/main/java/io/mo/util/MoConfUtil.java @@ -85,8 +85,30 @@ public static int getSocketTimeout(){ return COMMON.DEFAULT_MAX_EXECUTE_TIME; } + public static String[] getDebugServers(){ + Map debug = (Map)conf.get("debug"); + if(debug != null){ + String serverStr = debug.get("serverIP").toString(); + if(serverStr != null){ + return serverStr.split(","); + } + } + + return null; + } + + public static int getDebugPort(){ + Map debug = (Map)conf.get("debug"); + if(debug != null){ + String portStr = debug.get("port").toString(); + if(portStr != null){ + return Integer.parseInt(portStr); + } + } - + return 0; + } + public static void main(String[] args){ System.out.println(getDriver()); System.out.println(getURL());