invalid collation ID using vector

[wjhuang2016]

Bug Report

Please answer these questions before submitting your issue. Thanks!

1. Minimal reproduce step (Required)

CREATE TABLE `tdf6b259c` (
  `col_69` timestamp NULL DEFAULT NULL,
  `col_70` time DEFAULT NULL,
  `col_71` varchar(280) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '!M0x8r*6Ja^*1',
  `col_72` tinyint(1) DEFAULT '0',
  `col_73` vector NOT NULL,
  `col_74` float NOT NULL,
  `col_75` vector DEFAULT NULL,
  `col_76` vector DEFAULT NULL,
  `col_77` vector DEFAULT NULL,
  `col_78` varchar(455) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT 'RDA&C&82j',
  UNIQUE KEY `idx_15_5` (`col_74`,`col_78`(3),`col_69`),
  UNIQUE KEY `idx_16` (`col_72`,`col_78`),
  KEY `idx_17` (`col_74`,`col_71`(2))
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
INSERT INTO `tdf6b259c` VALUES ('2021-06-05 00:00:00','22:22:48','-+9@Dov6e!1p',1,'[0.768367]',459.80948,'[0.875416,0.798303,0.95263,0.549606,0.883848,0.537395,0.697845,0.282063,0.598167]','[0.153242,0.66589,0.86416,0.0374,0.179505]','[0.739426,0.010635,0.014233,0.488696,0.572936,0.458141]','IAR@'), ('1984-10-14 00:00:00','06:44:35','g_2%^',1,'[0.16176,0.158337,0.179023,0.85178,0.393512]',1882.1268,'[0.511331,0.281022,0.108078]','[0.429538,0.10373,0.084632]','[0.530908,0.925691,0.702226,0.165207,0.857029,0.16353,0.478932,0.183449,0.092973]','sJ2lpHm_k7mc#5'), ('1987-03-03 00:00:00','17:22:17','',NULL,'[0.436354,0.240392]',8651.746,'[0.273624,0.718277,0.640185,0.966812,0.241593]','[0.438376]','[0.007709,0.917675,0.534939]','77Js'), (NULL,'09:24:15','dII8!gOP#@gbW',0,'[0.696222,0.714997,0.308708,0.278187,0.771423]',7893.792,'[0.696198,0.399106]','[0.892947,0.384578,0.196303,0.35732,0.467014,0.379903,0.033385,0.176111]','[0.351172,0.382038,0.000892,0.266095,0.321914]','(WBk7JX^re%0V^l'), ('1989-07-10 00:00:00','14:57:25','UPoGz!$y',0,'[0.922199,0.808347,0.539523,0.285251,0.263623]',2990.7754,'[0.717826,0.899709,0.748334,0.779614,0.844035,0.069045,0.143246,0.148318,0.522787]','[0.794194,0.334738,0.253752,0.501697,0.989366]','[0.572076,0.082038,0.866923,0.224718,0.342494,0.363199,0.376946]','Jt^$SXXzr87-'), ('1984-08-11 00:00:00','14:26:37','#az9rBIsHMSKl',1,'[0.132071,0.999432,0.868087]',2299.4421,'[0.635225,0.653566,0.211639,0.32263]','[0.679219]','[0.204091,0.437863,0.810699,0.713632,0.24805,0.327955]','E~I4KwS9'), ('2006-07-17 00:00:00','07:37:53','uj27',0,'[0.614454,0.990473,0.278903]',6268.065,'[0.460278,0.721062,0.450292]','[0.677389,0.565604,0.128649,0.443451,0.76285,0.740928,0.430681,0.393115,0.010335,0.316196]','[0.880352,0.443912,0.677599,0.518225,0.974141,0.589329,0.800139]','$e_8btI4kaHaZo+Q0td'), ('2034-02-04 00:00:00','17:54:13','%LQOZ',1,'[0.97323,0.532345,0.903054,0.25654,0.428855,0.667964,0.785114,0.325878,0.174073]',9246.063,NULL,'[0.005738,0.631881,0.566612,0.712757,0.761189,0.443902,0.211031]','[0.625876,0.061498,0.855065]','Z+XhYG*yHS9yJYXa'), ('2022-07-11 00:00:00','19:09:15','BCuq',0,'[0.005913,0.312031,0.30494,0.149265,0.368831]',9357.192,'[0.761202,0.832853,0.838391,0.949236,0.738126]','[0.254855]','[0.787639,0.555112,0.610335,0.156837,0.131754,0.585172,0.627183,0.304718]','9b'), ('2024-09-30 00:00:00','23:28:39','hVzr',0,'[0.303383,0.830808,0.946549,0.169501,0.233423,0.935685,0.807238]',1420.5289,'[0.270846,0.796107,0.604017]','[0.360708,0.61982,0.403065,0.108743,0.761886,0.347637,0.59551,0.18061,0.71127,0.768534]','[0.09747,0.931537]','yQ_VO+q'), ('1974-12-21 00:00:00','17:36:28','Md9j=x)',0,'[0.680813,0.251938,0.603233,0.077319]',9531.407,'[0.276456,0.702095,0.939563]','[0.975907,0.162346]','[0.537672,0.195429,0.601556,0.260959,0.777523,0.95623,0.390249]','Z'), ('2036-08-21 00:00:00','11:23:20','1J',0,'[0.634106]',125.54578,'[0.135578,0.151275,0.938054,0.843996,0.84646,0.515414,0.338462]','[0.268664,0.761316,0.907708,0.931381]','[0.222805,0.316272,0.964051,0.019543,0.543798,0.598823,0.443615,0.747412,0.37692,0.505909]','L'), ('2001-11-27 00:00:00','09:26:50','Ylc&tUE2KgV',1,'[0.380011,0.616955,0.600541,0.573678,0.977827,0.238978,0.234863,0.797402,0.574637,0.255334]',3390.1177,'[0.379792,0.876047,0.430851,0.023545,0.141399,0.977003,0.942889,0.725326,0.468952]','[0.998623,0.59612,0.10348,0.262907,0.380591,0.26818,0.394373,0.009878]','[0.358095,0.208099,0.600131,0.697941,0.384417,0.440019,0.437179,0.717385,0.680042,0.786112]','OL4=P%JE&'), ('2032-10-31 00:00:00','19:56:26','~+KSj=gypBXny@LfO',0,'[0.144257,0.184732]',3488.1033,'[0.123239,0.697359,0.888311,0.551426,0.502726]','[0.194721,0.432261]','[0.206087,0.854889,0.275608,0.065764,0.563178,0.394006,0.45993]','-ox~C=(RC'), ('2006-12-30 00:00:00','11:48:29','LHo&SZdZ3d',0,'[0.0687,0.120366,0.497344,0.200777,0.388504]',9260.338,'[0.916161,0.093694]','[0.794789,0.274178]','[0.330111,0.32633,0.218215,0.984068,0.479349,0.805201,0.656584,0.087717]','p_=+iTCBkv^'), ('2025-05-21 00:00:00','09:36:09','GoDXz$',0,'[0.710344,0.927059,0.049245,0.881506,0.753727,0.78396,0.434597]',3785.2375,'[0.519477,0.930364]','[0.244884,0.8317,0.751951,0.128865,0.834638,0.463626,0.965154,0.082179,0.32744,0.055279]','[0.456137,0.840085,0.664893,0.653673,0.301459]','no2VoS)D'), ('2011-09-19 00:00:00','16:23:36','gZKUraYLS=VBP7fq',1,'[0.110217,0.12858,0.38088,0.655097,0.283326,0.219191,0.35158,0.561]',8495.709,'[0.450481,0.259553,0.942294,0.8993,0.103823,0.287252,0.033757,0.562437]','[0.61972,0.380139,0.637095,0.294071,0.495599]','[0.612494,0.56813,0.393902,0.633826,0.799152,0.771355]','G0bT*wYAP'), ('1984-05-26 00:00:00','19:58:42','5dfTuUTQtM-fj2^m',0,'[0.654201,0.669526,0.733463,0.911632,0.017419,0.707658,0.180698,0.358572,0.223526]',1991.9677,'[0.807081,0.212535,0.497195,0.808561,0.820641]','[0.491594,0.176452,0.081224,0.238793,0.776373,0.336504,0.04663,0.020001,0.98625,0.173639]','[0.409949,0.398241,0.340868,0.402707,0.948149]','otj6Nhpl'), ('2024-01-29 00:00:00','21:29:00','GL&$+ZY~HES$FzkP',0,'[0.353389,0.13418,0.912908,0.603667,0.839657,0.50152,0.102168,0.488625,0.662079]',4462.4697,'[0.642813,0.384936,0.391114,0.350459]','[0.319128,0.697857,0.169544,0.084868,0.754337,0.702221]','[0.790781,0.351218]','^owmoWmrk1');
CREATE TABLE `t667c50dc` (
  `col_35` vector DEFAULT NULL,
  `col_36` vector NOT NULL,
  `col_37` mediumint(9) NOT NULL DEFAULT '2566516',
  `col_38` vector(1) DEFAULT NULL,
  `col_39` double NOT NULL DEFAULT '9678.469625529695',
  `col_40` tinyblob NOT NULL,
  `col_41` date DEFAULT NULL,
  `col_42` char(210) CHARACTER SET gbk COLLATE gbk_bin NOT NULL,
  `col_43` vector(9) NOT NULL,
  `col_44` vector(4) DEFAULT NULL,
  KEY `idx_8` (`col_37`,`col_42`,`col_40`(4)) /*!80000 INVISIBLE */,
  UNIQUE KEY `idx_9` (`col_37`),
  KEY `idx_10` (`col_37`),
  VECTOR INDEX `idx_11`((VEC_L2_DISTANCE(`col_43`)))
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
INSERT INTO `t667c50dc` VALUES ('[0.383298,0.18879]','[0.516337]',-547990,'[0.852749]',7883.45152187935,x'7741462a403d572b','1991-06-09','T%r叄JU','[0.877401,0.468472,0.597765,0.972251,0.181309,0.613176,0.963745,0.233932,0.546485]','[0.140203,0.709093,0.553438,0.848537]'), ('[0.456176,0.38014,0.327892,0.190015,0.096354,0.62266,0.153024,0.127524,0.15913]','[0.435997,0.36343,0.531185,0.235006]',2716483,'[0.719311]',5504.430739929597,x'77286b435451394c31544747','1973-07-23','o縯b筘墝xYx煨f_a9','[0.879254,0.376472,0.990963,0.263592,0.930676,0.841443,0.788239,0.637674,0.945468]','[0.462101,0.80809,0.428661,0.844338]'), ('[0.011368,0.597524,0.452023]','[0.679119]',-3690882,'[0.818823]',9369.743141924358,x'6c474830367831442872593043214569533d6b','1981-11-14','dq鹴囒2_Ztf0b','[0.99222,0.61804,0.770268,0.04225,0.599467,0.925737,0.783302,0.530401,0.851362]','[0.601979,0.509617,0.626067,0.544254]'), ('[0.52842,0.814156,0.760761,0.812914,0.588171]','[0.075177,0.292542,0.86925]',1761126,'[0.386723]',5155.09855638897,x'543937','2029-02-12','','[0.836403,0.339261,0.924742,0.880582,0.692607,0.017795,0.162952,0.203983,0.167703]','[0.292242,0.671653,0.70387,0.207538]'), ('[0.541232,0.762744,0.298321,0.329372,0.012124,0.031456]','[0.380051,0.470826,0.661199,0.543595]',-3955310,'[0.312153]',3444.2047652535184,x'304347','1989-09-01','弰^&杄ECoOn益','[0.987482,0.064784,0.32232,0.171681,0.106259,0.088317,0.108157,0.324767,0.512666]','[0.674746,0.015006,0.069941,0.912483]'), ('[0.854878,0.176907,0.294815,0.896173,0.167893,0.804607]','[0.240318,0.293221,0.382622,0.528261,0.24109,0.289549,0.717569]',-5408827,'[0.121421]',5212.975400969331,x'573d46676f3179457a79214f2d68','2011-05-24','J倅喂擮颣coQ毻轪f龢敺呜','[0.001037,0.432115,0.471264,0.039859,0.602367,0.12022,0.102074,0.335723,0.236293]','[0.397395,0.328342,0.469473,0.690935]'), ('[0.671412,0.737304,0.45749,0.734991,0.793201]','[0.664807,0.042991,0.86229,0.781814,0.318566,0.780797]',-6533288,'[0.929198]',1344.3285404502353,x'2556744d393221457472','1986-03-16','','[0.194178,0.287801,0.549327,0.578941,0.690463,0.12557,0.232245,0.454096,0.406056]','[0.702292,0.688803,0.467933,0.570569]'), ('[0.962588,0.603691]','[0.786548,0.441419,0.059135,0.449744,0.987237]',-7934462,'[0.841157]',8987.658748884956,x'376576304c48556f','2028-08-06','欏8k%8閛*~ON#睭','[0.442557,0.062884,0.979605,0.955596,0.09224,0.151467,0.116198,0.909936,0.405495]','[0.663901,0.792231,0.299896,0.942555]'), ('[0.597888]','[0.468309,0.939542,0.366326,0.012183,0.73243,0.794514,0.352435,0.267509,0.298331,0.838862]',-7037843,'[0.901442]',6771.6979876299665,x'454b315a65484b262942514d34764c3530','2031-07-01','Sb觱dG埶^2','[0.715405,0.685803,0.576119,0.474778,0.785863,0.710941,0.187822,0.025745,0.418759]','[0.390527,0.094864,0.165406,0.501767]'), ('[0.509335,0.921061,0.235377,0.247205]','[0.298677,0.713486,0.540673,0.384015,0.096588,0.597306,0.874814,0.009815]',8107007,'[0.989311]',2163.623546551584,x'765a6365','2020-06-02','a','[0.562331,0.96549,0.250146,0.732895,0.733039,0.623109,0.821982,0.680988,0.153725]','[0.330236,0.015141,0.930046,0.1691]'), ('[0.906407]','[0.918741,0.101126]',2403389,'[0.842198]',3899.553871381563,x'6d283963442d','2029-12-14','oVw御歍殊','[0.030286,0.193697,0.847513,0.800063,0.975524,0.126351,0.270162,0.833302,0.58671]','[0.451676,0.5232,0.884215,0.709094]'), ('[0.858318]','[0.934645]',-8388607,'[0.185795]',9887.492981996802,x'507644374826634a7a2a28642b6247','2007-11-18','G~顯J巗u!G梪0T琣袻','[0.80894,0.015861,0.168281,0.097462,0.472798,0.579262,0.74031,0.641623,0.546468]','[0.160857,0.704655,0.443463,0.078626]'), ('[0.755475,0.327822,0.780293,0.610684,0.408645,0.040815,0.695149]','[0.75688,0.31899,0.286155,0.722777]',3806664,'[0.169849]',1735.1276043992375,x'2b787870555f3376657370216435','2026-10-30','奊AsC$椉','[0.706697,0.054781,0.741856,0.71834,0.994083,0.264234,0.241546,0.021396,0.754233]','[0.110252,0.679945,0.320649,0.091471]'), ('[0.374679,0.943942,0.58678,0.284305,0.842554,0.740462,0.100487,0.428322]','[0.530125,0.617044,0.960488,0.230743,0.49258,0.593865]',-2159662,'[0.275496]',6588.592251080329,x'656b44','2013-06-24','0彝pp墻o6捖hno重圑値*碟浘E','[0.574832,0.420887,0.854298,0.861727,0.694412,0.377134,0.815708,0.92949,0.72314]','[0.012637,0.887168,0.049051,0.236332]'), ('[0.586334,0.855492,0.510314,0.378086,0.772613,0.944038]','[0.132031,0.148937,0.711282,0.612281,0.586452]',4496898,'[0.597997]',5532.120041672184,x'','2010-07-22','琜L^9釫位6~y噢蟥E)秇h盉&璱S','[0.997543,0.514932,0.136357,0.346065,0.699027,0.333859,0.749956,0.52599,0.106445]','[0.839962,0.951969,0.129571,0.507795]'), ('[0.409649,0.711616,0.451979,0.557916,0.077437,0.02711,0.287762]','[0.304001,0.705944,0.059937,0.082481,0.506452]',5553424,NULL,5328.741221282641,x'6a38535126776c643d263658366163632b67','1986-07-17','+J_鄺锉匏^0_竪僈kOn煨l','[0.307011,0.129233,0.030451,0.682611,0.951263,0.211626,0.849698,0.545163,0.733221]','[0.577903,0.416525,0.783489,0.350081]'), ('[0.856868,0.641526,0.45429,0.501665,0.459246]','[0.278083,0.698274,0.614047,0.933208,0.385731,0.164946,0.003692,0.710457,0.828745,0.763866]',-1533361,'[0.876169]',5570.742417516762,x'4e79416f216b743d487e72546524','2019-01-01','魚龄','[0.362955,0.498105,0.937638,0.657355,0.664758,0.157811,0.343678,0.925668,0.253405]','[0.84016,0.366246,0.376464,0.47579]'), ('[0.240691,0.19257,0.980417,0.956637,0.701167,0.840117,0.935748,0.443001]','[0.509975,0.288122,0.592626,0.140158,0.703394,0.96624,0.689572,0.545013]',2327898,'[0.83344]',2136.220027854472,x'6244386e5a56525333','1987-06-26','','[0.129331,0.630315,0.513899,0.753513,0.971445,0.494609,0.009273,0.120211,0.470958]','[0.96046,0.076018,0.566066,0.757905]'), ('[0.390485,0.460107,0.505047]','[0.19504,0.322512,0.390686,0.488879,0.334366,0.432348,0.04667,0.362289,0.728418,0.702587]',-8388608,'[0.351145]',8236.575540744869,x'6735384e355142755939','1985-10-23','敉赘唺','[0.804303,0.641977,0.844792,0.703239,0.208461,0.913359,0.012677,0.19472,0.056792]','[0.18458,0.048642,0.026425,0.738304]'), ('[0.304831,0.3678,0.101639]','[0.670289,0.41511]',3906319,'[0.284711]',9836.086024419006,x'745a387540','1987-02-05','E^tKj嫛糧uI眼','[0.890103,0.216234,0.164473,0.64673,0.585915,0.16892,0.226846,0.763059,0.393541]','[0.003496,0.919951,0.583768,0.282507]');
alter table tdf6b259c set tiflash replica 1;
alter table t667c50dc set tiflash replica 1;


select   /*+ NO_MERGE_JOIN( tdf6b259c , st_43 */ vec_dims( tdf6b259c.col_73 ) as r0 , instr( tdf6b259c.col_78 , tdf6b259c.col_78 ) as r1 , tdf6b259c.col_78 as
r2 , st_43.r0 as r3 , VEC_NEGATIVE_INNER_PRODUCT( tdf6b259c.col_73 , tdf6b259c.col_76 ) as r4 from tdf6b259c join ( select  /*+ use_index_merge( t667c50dc ) */   t667c50dc.col_42 as r0 , max( distinct  t667c50dc.col_37 ) as r1 from t667c50dc where IsNull( t667c50dc.col_44 ) or t667c50dc.col_43 > '[0.198127, 0.173948, 0.717988, 0.526801, 0.364223, 0.417024, 0.952432, 0.294333, 0.596499]' group by t667c50dc.col_42,t667c50dc.col_36  having t667c50dc.col_42 between 'l甁擺!%莎3籫廷Bs綢褆' and 'Sb觱dG埶^2'  limit 84452813 ) st_43 on tdf6b259c.col_78 = st_43.r0  order by r0,r1,r2,r3,r4 limit 20000000;

2. What did you expect to see? (Required)

No error

3. What did you see instead (Required)

mysql> select   /*+ NO_MERGE_JOIN( tdf6b259c , st_43 */ vec_dims( tdf6b259c.col_73 ) as r0 , instr( tdf6b259c.col_78 , tdf6b259c.col_78 ) as r1 , tdf6b259c.col_78 as r2 , st_43.r0 as r3 , VEC_NEGATIVE_INNER_PRODUCT( tdf6b259c.col_73 , tdf6b259c.col_76 ) as r4 from tdf6b259c join ( select  /*+ use_index_merge( t667c50dc ) */   t667c50dc.col_42 as r0 , max( distinct  t667c50dc.col_37 ) as r1 from t667c50dc where IsNull( t667c50dc.col_44 ) or t667c50dc.col_43 > '[0.198127, 0.173948, 0.717988, 0.526801, 0.364223, 0.417024, 0.952432, 0.294333, 0.596499]' group by t667c50dc.col_42,t667c50dc.col_36  having t667c50dc.col_42 between 'l甁擺!%莎3籫廷Bs綢褆' and 'Sb觱dG埶^2'  limit 84452813 ) st_43 on tdf6b259c.col_78 = st_43.r0  order by r0,r1,r2,r3,r4 limit 20000000;
ERROR 1105 (HY000): other error for mpp stream: Code: 49, e.displayText() = DB::Exception: static TiDB::TiDBCollatorPtr TiDB::ITiDBCollator::getCollator(int32_t): invalid collation ID: 87, e.what() = DB::Exception,

4. What is your TiDB version? (Required)

master

[JaySon-Huang]

Simplified test case:

CREATE TABLE `t667c50dc` (
  `col_42` char(210) CHARACTER SET gbk COLLATE gbk_bin NOT NULL,
  `col_43` vector(9) NOT NULL,
  VECTOR INDEX `idx_11`((VEC_L2_DISTANCE(`col_43`)))
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;

INSERT INTO `t667c50dc` VALUES
('T%r叄JU','[0.877401,0.468472,0.597765,0.972251,0.181309,0.613176,0.963745,0.233932,0.546485]'), 
('o縯b筘墝xYx煨f_a9','[0.879254,0.376472,0.990963,0.263592,0.930676,0.841443,0.788239,0.637674,0.945468]'), 
('dq鹴囒2_Ztf0b','[0.99222,0.61804,0.770268,0.04225,0.599467,0.925737,0.783302,0.530401,0.851362]'), 
('','[0.836403,0.339261,0.924742,0.880582,0.692607,0.017795,0.162952,0.203983,0.167703]'), 
('弰^&杄ECoOn益','[0.987482,0.064784,0.32232,0.171681,0.106259,0.088317,0.108157,0.324767,0.512666]'), 
('J倅喂擮颣coQ毻轪f龢敺呜','[0.001037,0.432115,0.471264,0.039859,0.602367,0.12022,0.102074,0.335723,0.236293]');

SELECT
    t667c50dc.col_42
FROM   t667c50dc
GROUP  BY 1
(1105, 'other error for mpp stream: Code: 49, e.displayText() = DB::Exception: static TiDBCollatorPtr TiDB::ITiDBCollator::getCollator(int32_t): invalid collation ID: 87, e.what() = DB::Exception,')

[JaySon-Huang]

Actually this is because VECTOR INDEX idx_11((VEC_L2_DISTANCE(col_43))) will automatically add tiflash replica. However, tiflash does not support gbk charset, so adding tiflash replica should be blocked.

CREATE TABLE `t667c50dc` (
  `col_42` char(210) CHARACTER SET gbk COLLATE gbk_bin NOT NULL,
  `col_43` vector(9) NOT NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;

alter table t667c50dc set tiflash replica 1;
(8200, 'Unsupported ALTER TiFlash settings for tables not supported by TiFlash: table contains gbk charset')

So actually, we expect that executing this create table statement should be failed

CREATE TABLE `t667c50dc` (
  `col_42` char(210) CHARACTER SET gbk COLLATE gbk_bin NOT NULL,
  `col_43` vector(9) NOT NULL,
  VECTOR INDEX `idx_11`((VEC_L2_DISTANCE(`col_43`)))
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;

[JaySon-Huang]

/cc @zimulala