Add pprof (#153)

* modify socketTimeout to 120s

* add regular match

* support rc + Pessimistic

* revert Fix pessimistic commit wait error

* revert revert

* add pprof

* add pprof scripts

cases/template.result

@@ -1,3 +1,4 @@
+select sleep(10);
 create table t1 ( a int not null default 1, int32 int primary key);
 insert into t1 (int32) values (-1),(1234567890),(2147483647);
 insert into t1 (int32) values (-1),(1234567890),(2147483647);

cases/template.sql

@@ -1,3 +1,4 @@
+select sleep(10);
 create table t1 ( a int not null default 1, int32 int primary key);
 insert into t1 (int32) values (-1),(1234567890),(2147483647);
 -- @pattern

mo.yml

@@ -21,3 +21,8 @@ jdbc:
 user:
   name: "dump"
   password: "111"
+
+#debug info
+debug:
+  serverIP: "127.0.0.1"
+  port: 6060

pprof.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+if [ $# -eq 0 ] ; then
+    echo "No parameters provided,please use -H to get help. " >&2
+    exit 2
+fi
+
+PORT=6060
+SERVER="127.0.0.1"
+DURATION=30
+SLEEP=60
+WORKSPACE=$(cd `dirname $0`; pwd)
+
+while getopts ":h:p:d:s:H" opt
+do
+    case $opt in
+        h)
+        SERVER="${OPTARG}"
+        echo -e "`date +'%Y-%m-%d %H:%M:%S'` The server addr is : ${OPTARG}"
+        ;;
+        p)
+        expr ${OPTARG} "+" 10 &> /dev/null
+        if [ $? -ne 0 ]; then
+          echo "`date +'%Y-%m-%d %H:%M:%S'` The port [${OPTARG}] is not a number"
+          exit 1
+        fi
+        PORT=${OPTARG}
+        echo -e "`date +'%Y-%m-%d %H:%M:%S'` The debug port is : ${OPTARG}"
+        ;;
+        d)
+        DURATION=${OPTARG}
+        echo -e "`date +'%Y-%m-%d %H:%M:%S'` The collection duration is : ${OPTARG} s"
+        ;;
+        s)
+        SLEEP=${OPTARG}
+        echo -e "`date +'%Y-%m-%d %H:%M:%S'` The sleep time for each turn is : ${OPTARG} s"
+        ;;
+        H)
+        echo -e "Usage:　bash pprof.sh [option] [param] ...\nExcute pprof task"
+        echo -e "   -h  server address, default value is 127.0.0.1"
+        echo -e "   -p  server port, default value is 6060"
+        echo -e "   -d  the duration that pprof collection will last once, unit s"
+        echo -e "   -s  the sleeptime for each turn, unit s"
+        echo -e "Examples:"
+        echo "   bash pprof.sh -h 127.0.0.1 -p 6060 -d 10"
+	      echo "For more support,please email to [email protected]"
+        exit 1
+        ;;
+        ?)
+        echo "Unkown parameter,please use -H to get help."
+        exit 1;;
+    esac
+done
+
+if [ ! -d ${WORKSPACE}/report/prof/${SERVER} ] ; then
+    mkdir ${WORKSPACE}/report/prof/${SERVER}
+fi
+
+time=`date +'%Y-%m-%d_%H:%M:%S'`
+mkdir -p ${WORKSPACE}/report/prof/${SERVER}/${time}
+curl http://${SERVER}:${PORT}/debug/pprof/goroutine?debug=2 -o ${WORKSPACE}/report/prof/${SERVER}/${time}/goroutine.log
+curl http://${SERVER}:${PORT}/debug/pprof/trace?seconds=30 -o ${WORKSPACE}/report/prof/${SERVER}/${time}/trace.out
+if [ $? -ne 0 ];then
+  echo -e "`date +'%Y-%m-%d_%H:%M:%S'` The MO debug service can not be reached, the pprof operation was failed."
+  exit 1
+fi
+

run.sh

@@ -7,7 +7,7 @@ fi
 
 TIMES=1
 
-while getopts ":p:m:t:r:i:e:s:gfnch" opt
+while getopts ":p:m:t:r:i:e:s:ogfnch" opt
 do
     case $opt in
         p)
@@ -58,6 +58,10 @@ do
         CHECK="check"
         echo -e "The meta data of the resultset will be ignored when comparing the resut"
         ;;
+        o)
+        PPROF="pprof"
+        echo -e "If a query timeout, mo-tester will collect the pprof info from mo"
+        ;;
         h)
         echo -e "Usage:　bash run.sh [option] [param] ...\nExcute test cases task"
         echo -e "   -p  set the path of test cases needed to be executed by mo-tester"
@@ -75,6 +79,7 @@ do
         echo -e "       Those two sql commands are associated with the issue#3236,and they will not been executed in bvt test,until the flag is removed when the issue#3236 is fixed."
         echo -e "   -n  means the meta data of the resultset will be ignored when comparing the resut"
         echo -e "   -c  check whether the case scripts match the result file"
+        echo -e "   -o  if a query timeout, mo-tester will collect the pprof info from mo"
         echo -e "Examples:"
         echo "   bash run.sh -p case -m run -t script -r 100 -i select,subquery -e substring -g"
         echo "For more support,please email to [email protected]"
@@ -104,7 +109,7 @@ if [ ${TIMES} -eq 1 ]; then
   java -Xms1024M -Xmx1024M -cp ${libJars} \
           -Dconf.yml=${MO_YAML} \
           -Drun.yml=${RUN_YAML} \
-          io.mo.Tester ${PATHC} ${METHOD} ${TYPE} ${RATE} ${INCLUDE} ${EXCLUDE} ${IGNORE} ${NOMETA} ${CHECK} ${RESOURCE} ${FORCE}
+          io.mo.Tester ${PATHC} ${METHOD} ${TYPE} ${RATE} ${INCLUDE} ${EXCLUDE} ${IGNORE} ${NOMETA} ${CHECK} ${RESOURCE} ${FORCE} ${PPROF}
 else
   echo "This test will be run for ${TIMES} times"
   for i in $(seq 1 ${TIMES})
@@ -113,7 +118,7 @@ else
       java -Xms1024M -Xmx1024M -cp ${libJars} \
                 -Dconf.yml=${MO_YAML} \
                 -Drun.yml=${RUN_YAML} \
-                io.mo.Tester ${PATHC} ${METHOD} ${TYPE} ${RATE} ${INCLUDE} ${EXCLUDE} ${IGNORE} ${NOMETA} ${CHECK} ${RESOURCE} ${FORCE}
+                io.mo.Tester ${PATHC} ${METHOD} ${TYPE} ${RATE} ${INCLUDE} ${EXCLUDE} ${IGNORE} ${NOMETA} ${CHECK} ${RESOURCE} ${FORCE} ${PPROF}
       echo "The ${i} turn test has ended, and test report is in ./report/${i} dir." | tee -a ${WORKSPACE}/run.log
       mkdir -p ${WORKSPACE}/${MOTESTER_DIR}/report/${i}/
       mv ${WORKSPACE}/${MOTESTER_DIR}/report/*.txt ${WORKSPACE}/${MOTESTER_DIR}/report/${i}/

src/main/java/io/mo/Tester.java

@@ -118,6 +118,11 @@ public static void main(String[] args){
                 if (arg.equalsIgnoreCase("check")) {
                     method = "check";
                 }
+
+                //get pprof info
+                if (arg.equalsIgnoreCase("pprof")) {
+                    COMMON.NEEDPPROF = true;
+                }
             }
         }
 

src/main/java/io/mo/constant/COMMON.java

@@ -62,6 +62,8 @@ public class COMMON {
     public static String START_TRX = "begin";
     public static String COMMIT_TRX = "commit";
     public static String ROLLBACK_TRX = "rollback";
+
+    public static boolean NEEDPPROF = false;
 
     public static String[] SPECIAL_CHARS = new String[]{"+","-","*","/","%","&",">","<","(",")","!","=","\'","\""};
 

src/main/java/io/mo/db/Executor.java

@@ -203,6 +203,12 @@ public static void run(TestScript script){
                         LOG.error("[EXPECT RESULT]:\n" + command.getTestResult().getExpResult());
                         LOG.error("[ACTUAL RESULT]:\n" + command.getTestResult().getActResult());
 
+                        if(COMMON.NEEDPPROF){
+                            LOG.info("Start to collect pprof information,please wait........");
+                            pprof();
+                            LOG.info("Finish to collect pprof information,the test will continue");
+                        }
+
                         //reconnect to mo, and set db to last use db
                         LOG.warn(String.format("The mo-tester tries to re-connect to mo, con[id=%d, user=%s, pwd=%s, db=%s], please wait.....",
                                 command.getConn_id(),command.getConn_user(),command.getConn_pswd(),command.getUseDB()));
@@ -657,6 +663,68 @@ public static void syncCommit(Connection connection){
         }
     }
 
+    public static void pprof(){
+        String[] debugServers = MoConfUtil.getDebugServers();
+        if(debugServers != null){
+            int port = MoConfUtil.getDebugPort();
+            Thread[] threads = new Thread[debugServers.length];
+            for(int i = 0; i < debugServers.length;i++){
+                int index = i;
+                threads[index] = new Thread(new Runnable() {
+                    @Override
+                    public void run() {
+                        try {
+                            Process p = Runtime.getRuntime().exec(String.format("./pprof.sh -h %s -p %d",debugServers[index],port));
+                            InputStream is = p.getInputStream();
+                            BufferedReader reader = new BufferedReader(new InputStreamReader(is));
+                            p.waitFor();
+
+                            StringBuffer execResut = new StringBuffer();
+                            String str = reader.readLine();
+                            while ( str != null) {
+                                execResut.append(str+"\n");
+                                str = reader.readLine();
+                            }
+
+                            if (p.exitValue() != 0) {
+                                LOG.error(String.format("The pprof operation has been executed failed.\n%s",execResut.toString()));
+                            }else {
+                                LOG.info(String.format("The pprof operation has been executed successfully.\n%s", execResut.toString()));
+                                LOG.info(String.format("The result is in the dir ./report/pprof/%s/",debugServers[index]));
+                            }
+
+                        } catch (IOException e) {
+                            LOG.error("The pprof operation has been executed failed.");
+                            LOG.error(String.format("The output of pprof operation is \n%s.",e.getMessage()));
+                        } catch (InterruptedException e) {
+                            LOG.error("The pprof operation has been executed failed.");
+                            LOG.error(String.format("The output of pprof operation is \n%s.",e.getMessage()));
+                        }
+                    }
+                });
+                threads[index].start();
+            }
+
+            boolean finished = false;
+            while(!finished){
+                try {
+                    Thread.sleep(1000);
+                } catch (InterruptedException e) {
+                    throw new RuntimeException(e);
+                }
+
+                for(Thread thread:threads){
+                    if(thread.isAlive()){
+                        finished = false;
+                        break;
+                    }
+                    finished = true;
+                }
+            }
+        }
+
+
+    }
 
     public static void executeSysCmd(String cmd){
         try {

src/main/java/io/mo/util/MoConfUtil.java

@@ -85,8 +85,30 @@ public static int getSocketTimeout(){
         return COMMON.DEFAULT_MAX_EXECUTE_TIME;
     }
 
+    public static String[] getDebugServers(){
+        Map debug = (Map)conf.get("debug");
+        if(debug != null){
+            String serverStr = debug.get("serverIP").toString();
+            if(serverStr != null){
+                return serverStr.split(",");
+            }
+        }
+
+        return null;
+    }
+
+    public static int getDebugPort(){
+        Map debug = (Map)conf.get("debug");
+        if(debug != null){
+            String portStr = debug.get("port").toString();
+            if(portStr != null){
+                return Integer.parseInt(portStr);
+            }
+        }
 
-
+        return 0;
+    }
+
     public static void main(String[] args){
         System.out.println(getDriver());
         System.out.println(getURL());