From 9905d8ef603b51c2ce7814f1283024a5991eec82 Mon Sep 17 00:00:00 2001 From: xuyifan <675434007@qq.com> Date: Wed, 21 Dec 2022 15:08:53 +0800 Subject: [PATCH 1/3] fix incorrect datetime value when loading stats --- statistics/handle/handle.go | 2 +- statistics/handle/handle_test.go | 39 ++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/statistics/handle/handle.go b/statistics/handle/handle.go index c85656ff99d4b..f8354df1470f5 100644 --- a/statistics/handle/handle.go +++ b/statistics/handle/handle.go @@ -1906,7 +1906,7 @@ func (h *Handle) histogramFromStorage(reader *statsReader, tableID int64, colID lowerBound = rows[i].GetDatum(2, &fields[2].Column.FieldType) upperBound = rows[i].GetDatum(3, &fields[3].Column.FieldType) } else { - sc := &stmtctx.StatementContext{TimeZone: time.UTC} + sc := &stmtctx.StatementContext{TimeZone: time.UTC, AllowInvalidDate: true, IgnoreZeroInDate: true} d := rows[i].GetDatum(2, &fields[2].Column.FieldType) // For new collation data, when storing the bounds of the histogram, we store the collate key instead of the // original value. diff --git a/statistics/handle/handle_test.go b/statistics/handle/handle_test.go index 5e41b1c1279a2..9122ad816302b 100644 --- a/statistics/handle/handle_test.go +++ b/statistics/handle/handle_test.go @@ -3543,3 +3543,42 @@ func TestStatsLockAndUnlockTables(t *testing.T) { tbl2Stats2 := handle.GetTableStats(tbl2.Meta()) require.Equal(t, int64(2), tbl2Stats2.Count) } + +func TestIssue39336(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec(` +create table t1 ( + a datetime(3) default null, + b int +) partition by range (b) ( + partition p0 values less than (1000), + partition p1 values less than (2000), + partition p3 values less than (maxvalue) +)`) + tk.MustExec("set @@sql_mode=''") + tk.MustExec(` +insert into t1 values +('2022-11-23 14:25:08.000', 1001), +('1000-00-09 00:00:00.000', 1001), +('1000-00-06 00:00:00.000', 1001), +('1000-00-06 00:00:00.000', 1001), +('2022-11-23 14:24:30.000', 1), +('2022-11-23 14:24:32.000', 1), +('2022-11-23 14:24:33.000', 1), +('2022-11-23 14:24:35.000', 1), +('1000-00-09 00:00:00.000', 1), +('1000-00-06 00:00:00.000', 1), +('1000-00-06 00:00:00.000', 1), +('2022-11-23 14:25:11.000', 2001), +('2022-11-23 14:25:16.000', 3001), +('1000-00-09 00:00:00.000', 3001), +('1000-00-09 00:00:00.000', 2001), +('1000-00-06 00:00:00.000', 2001), +('1000-00-09 00:00:00.000', 2001)`) + tk.MustExec("analyze table t1") + rows := tk.MustQuery("show analyze status where job_info like 'merge global stats%'").Rows() + require.Len(t, rows, 1) + require.Equal(t, "finished", rows[0][7]) +} From f88f0dac36ba45185f9894285e2b35a12b897797 Mon Sep 17 00:00:00 2001 From: xuyifan <675434007@qq.com> Date: Fri, 23 Dec 2022 14:02:36 +0800 Subject: [PATCH 2/3] upd --- statistics/handle/handle.go | 6 +++++- statistics/handle/handle_test.go | 4 +++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/statistics/handle/handle.go b/statistics/handle/handle.go index f8354df1470f5..1378ce50be8d1 100644 --- a/statistics/handle/handle.go +++ b/statistics/handle/handle.go @@ -1906,7 +1906,11 @@ func (h *Handle) histogramFromStorage(reader *statsReader, tableID int64, colID lowerBound = rows[i].GetDatum(2, &fields[2].Column.FieldType) upperBound = rows[i].GetDatum(3, &fields[3].Column.FieldType) } else { - sc := &stmtctx.StatementContext{TimeZone: time.UTC, AllowInvalidDate: true, IgnoreZeroInDate: true} + // Invalid date values or zero date values may be inserted into table under some relaxed sql mode. Those values + // may exist in statistics. Hence, when reading statistics, we should skip invalid date check and zero date check. + // See #39336. + //sc := &stmtctx.StatementContext{TimeZone: time.UTC, AllowInvalidDate: true, IgnoreZeroInDate: true} + sc := &stmtctx.StatementContext{TimeZone: time.UTC} d := rows[i].GetDatum(2, &fields[2].Column.FieldType) // For new collation data, when storing the bounds of the histogram, we store the collate key instead of the // original value. diff --git a/statistics/handle/handle_test.go b/statistics/handle/handle_test.go index 9122ad816302b..3a1bae029b72d 100644 --- a/statistics/handle/handle_test.go +++ b/statistics/handle/handle_test.go @@ -3558,6 +3558,8 @@ create table t1 ( partition p3 values less than (maxvalue) )`) tk.MustExec("set @@sql_mode=''") + tk.MustExec("set @@tidb_analyze_version=2") + tk.MustExec("set @@tidb_partition_prune_mode='dynamic'") tk.MustExec(` insert into t1 values ('2022-11-23 14:25:08.000', 1001), @@ -3577,7 +3579,7 @@ insert into t1 values ('1000-00-09 00:00:00.000', 2001), ('1000-00-06 00:00:00.000', 2001), ('1000-00-09 00:00:00.000', 2001)`) - tk.MustExec("analyze table t1") + tk.MustExec("analyze table t1 with 0 topn") rows := tk.MustQuery("show analyze status where job_info like 'merge global stats%'").Rows() require.Len(t, rows, 1) require.Equal(t, "finished", rows[0][7]) From 597a32ed2455351e20826fbba338513788ed9090 Mon Sep 17 00:00:00 2001 From: xuyifan <675434007@qq.com> Date: Fri, 23 Dec 2022 16:07:31 +0800 Subject: [PATCH 3/3] upd --- statistics/handle/handle.go | 8 +++----- statistics/handle/handle_test.go | 21 ++++++--------------- 2 files changed, 9 insertions(+), 20 deletions(-) diff --git a/statistics/handle/handle.go b/statistics/handle/handle.go index 1378ce50be8d1..f53f075301acb 100644 --- a/statistics/handle/handle.go +++ b/statistics/handle/handle.go @@ -1906,11 +1906,9 @@ func (h *Handle) histogramFromStorage(reader *statsReader, tableID int64, colID lowerBound = rows[i].GetDatum(2, &fields[2].Column.FieldType) upperBound = rows[i].GetDatum(3, &fields[3].Column.FieldType) } else { - // Invalid date values or zero date values may be inserted into table under some relaxed sql mode. Those values - // may exist in statistics. Hence, when reading statistics, we should skip invalid date check and zero date check. - // See #39336. - //sc := &stmtctx.StatementContext{TimeZone: time.UTC, AllowInvalidDate: true, IgnoreZeroInDate: true} - sc := &stmtctx.StatementContext{TimeZone: time.UTC} + // Invalid date values may be inserted into table under some relaxed sql mode. Those values may exist in statistics. + // Hence, when reading statistics, we should skip invalid date check. See #39336. + sc := &stmtctx.StatementContext{TimeZone: time.UTC, AllowInvalidDate: true, IgnoreZeroInDate: true} d := rows[i].GetDatum(2, &fields[2].Column.FieldType) // For new collation data, when storing the bounds of the histogram, we store the collate key instead of the // original value. diff --git a/statistics/handle/handle_test.go b/statistics/handle/handle_test.go index 3a1bae029b72d..9bb80498bc90f 100644 --- a/statistics/handle/handle_test.go +++ b/statistics/handle/handle_test.go @@ -3554,31 +3554,22 @@ create table t1 ( b int ) partition by range (b) ( partition p0 values less than (1000), - partition p1 values less than (2000), - partition p3 values less than (maxvalue) + partition p1 values less than (maxvalue) )`) tk.MustExec("set @@sql_mode=''") tk.MustExec("set @@tidb_analyze_version=2") tk.MustExec("set @@tidb_partition_prune_mode='dynamic'") tk.MustExec(` insert into t1 values -('2022-11-23 14:25:08.000', 1001), -('1000-00-09 00:00:00.000', 1001), -('1000-00-06 00:00:00.000', 1001), -('1000-00-06 00:00:00.000', 1001), +('1000-00-09 00:00:00.000', 1), +('1000-00-06 00:00:00.000', 1), +('1000-00-06 00:00:00.000', 1), ('2022-11-23 14:24:30.000', 1), ('2022-11-23 14:24:32.000', 1), ('2022-11-23 14:24:33.000', 1), ('2022-11-23 14:24:35.000', 1), -('1000-00-09 00:00:00.000', 1), -('1000-00-06 00:00:00.000', 1), -('1000-00-06 00:00:00.000', 1), -('2022-11-23 14:25:11.000', 2001), -('2022-11-23 14:25:16.000', 3001), -('1000-00-09 00:00:00.000', 3001), -('1000-00-09 00:00:00.000', 2001), -('1000-00-06 00:00:00.000', 2001), -('1000-00-09 00:00:00.000', 2001)`) +('2022-11-23 14:25:08.000', 1001), +('2022-11-23 14:25:09.000', 1001)`) tk.MustExec("analyze table t1 with 0 topn") rows := tk.MustQuery("show analyze status where job_info like 'merge global stats%'").Rows() require.Len(t, rows, 1)