From 93993d07960f3d0e299c9c5a49c8dfca3fdd4ddf Mon Sep 17 00:00:00 2001 From: Kaili Xu Date: Thu, 10 Jun 2021 18:51:10 +0800 Subject: [PATCH 01/67] [TD-4666]: support max column num to 4096 --- src/client/inc/tsclient.h | 1 + src/client/src/tscSQLParser.c | 11 ++++++++--- src/client/src/tscServer.c | 14 ++++++++------ src/client/src/tscUtil.c | 1 + src/inc/taosdef.h | 4 ++-- tests/examples/lua/luaconnector.so | Bin 0 -> 26736 bytes 6 files changed, 20 insertions(+), 11 deletions(-) create mode 100755 tests/examples/lua/luaconnector.so diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index ec4bf52527..014bcc0bca 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -242,6 +242,7 @@ typedef struct SSqlObj { void * pStream; void * pSubscription; char * sqlstr; + void * pBuf; // tableMeta buffer char parseRetry; char retry; char maxRetry; diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 298bd8dacc..4b6b6d585d 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -7502,8 +7502,13 @@ int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo) { uint32_t maxSize = tscGetTableMetaMaxSize(); char name[TSDB_TABLE_FNAME_LEN] = {0}; - char buf[80 * 1024] = {0}; - assert(maxSize < 80 * 1024); + // char buf[80 * 1024] = {0}; + assert(maxSize < 80 * TSDB_MAX_COLUMNS); + if (!pSql->pBuf) { + if (NULL == (pSql->pBuf = calloc(1, 80 * TSDB_MAX_COLUMNS))) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + } pTableMeta = calloc(1, maxSize); plist = taosArrayInit(4, POINTER_BYTES); @@ -7520,7 +7525,7 @@ int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo) { if (pTableMeta->id.uid > 0) { if (pTableMeta->tableType == TSDB_CHILD_TABLE) { - code = tscCreateTableMetaFromSTableMeta(pTableMeta, name, buf); + code = tscCreateTableMetaFromSTableMeta(pTableMeta, name, pSql->pBuf); // create the child table meta from super table failed, try load it from mnode if (code != TSDB_CODE_SUCCESS) { diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index b4c7a64fd0..dbc53c5429 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -2509,22 +2509,25 @@ int32_t tscGetTableMetaImpl(SSqlObj* pSql, STableMetaInfo *pTableMetaInfo, bool taosHashGetClone(tscTableMetaInfo, name, len, NULL, pTableMetaInfo->pTableMeta, -1); // TODO resize the tableMeta - char buf[80*1024] = {0}; - assert(size < 80*1024); + // char buf[80 * TSDB_MAX_COLUMNS] = {0}; + assert(size < 80 * TSDB_MAX_COLUMNS); + if (!pSql->pBuf) { + if (NULL == (pSql->pBuf = calloc(1, 80 * TSDB_MAX_COLUMNS))) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + } STableMeta* pMeta = pTableMetaInfo->pTableMeta; if (pMeta->id.uid > 0) { // in case of child table, here only get the if (pMeta->tableType == TSDB_CHILD_TABLE) { - int32_t code = tscCreateTableMetaFromSTableMeta(pTableMetaInfo->pTableMeta, name, buf); + int32_t code = tscCreateTableMetaFromSTableMeta(pTableMetaInfo->pTableMeta, name, pSql->pBuf); if (code != TSDB_CODE_SUCCESS) { return getTableMetaFromMnode(pSql, pTableMetaInfo, autocreate); } } - return TSDB_CODE_SUCCESS; } - return getTableMetaFromMnode(pSql, pTableMetaInfo, autocreate); } @@ -2585,7 +2588,6 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, SQueryInfo* pQueryInfo) { if (allVgroupInfoRetrieved(pQueryInfo)) { return TSDB_CODE_SUCCESS; } - SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); pNew->pTscObj = pSql->pTscObj; pNew->signature = pNew; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index c17cd21c42..0b9867971a 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1402,6 +1402,7 @@ void tscFreeSqlObj(SSqlObj* pSql) { pSql->signature = NULL; pSql->fp = NULL; tfree(pSql->sqlstr); + tfree(pSql->pBuf); tfree(pSql->pSubs); pSql->subState.numOfSub = 0; diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index e6eec73681..7c552a0da0 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -178,7 +178,7 @@ do { \ // this is the length of its string representation, including the terminator zero #define TSDB_ACCT_ID_LEN 11 -#define TSDB_MAX_COLUMNS 1024 +#define TSDB_MAX_COLUMNS 4096 #define TSDB_MIN_COLUMNS 2 //PRIMARY COLUMN(timestamp) + other columns #define TSDB_NODE_NAME_LEN 64 @@ -193,7 +193,7 @@ do { \ #define TSDB_APPNAME_LEN TSDB_UNI_LEN -#define TSDB_MAX_BYTES_PER_ROW 16384 +#define TSDB_MAX_BYTES_PER_ROW 65536 #define TSDB_MAX_TAGS_LEN 16384 #define TSDB_MAX_TAGS 128 #define TSDB_MAX_TAG_CONDITIONS 1024 diff --git a/tests/examples/lua/luaconnector.so b/tests/examples/lua/luaconnector.so new file mode 100755 index 0000000000000000000000000000000000000000..08bf6a6156aebe053132545193cd111fb436bc4b GIT binary patch literal 26736 zcmeHQdw5*Mb)U6t5cnY(EMg3X#lZm#EaG7vHU`ON&AM1%z)Bn*U9F^*wPB?d`>16; z9F#Z?Y&HU?1mdJ$h(if=`hEEjd4Qpfk%MiC7N}@}&m_=_w}eeB1WpCPsJ}BaXLjz@ zy-vUOpFVc@*5{tzoO9;PxpU{hHMONrj$7IZsCWISFEwv|ng|-xa>Yt>WNI53@Ss77n&}F+Ucc6J z(^It`Q+D_Ix3kUvmGM?-z0O)?*iDmqJf=L}wa}xu{AZgEr?^g!=XV}9>v=QP;g#_u z8?V(meHlob5UOx(ByBEN*FMFknWQFIrxxW0(!8V z%SBLPU_~`Po4&W<&0UW!TzK%Vv2Mb=+AjeflArBk z=T0A-dND;r{P*(n zC?EV|pK&2Bg7~=GeLi-2eDK9Scn5H=xSfOfdf_vD@On7tHQ)I@51vyyBVj@&DUC^0)fn ze<|(E5O-e$0klR%#B0+38DeQj0k?vuHebu{saE_cyb(sDYui%o(M&d$&PJmmx?*)* zv^k!RuT5mK@$~AtWyw@~e08ib8CQMYJ<+DlShOY49!n-}i;H9~wldn(8gIHKyS_uR z(Y5hxOCp|ZroLz*lgXwN?Q12IO(iWZkx8^?<7?w--KV)u$!6l&Y^pR((!mS9na*FSrN;uZ*Pi9r=m@WnlXVe#A9vIrbY$BEq1GA+N|h2M6elJ9llE3|yS3xD@~B|qT8>$Ut|7k-|$-x^bT95?2BqY6I) zg!>5Tw{X1`O2GjOKUX5^pS>1-oP`fs_-PhCWZ?@ee0GCkjsyuJNezZY_Xt409 z7T#*%(=5Eh!lzq!r-je3@Gc9lu<*MqoUb+N-fZFet0ci%fBP-`cncq}@HrN~*TPS*@Iebd(ZYu; ze6EF$Sold6K4#%3Te#5I7tJeZ;pG;7iiKBN_&f`rZQ-A{@SugCYT>6_xZVn7Z?%Q% zEl=VXTlndgof-?*Tb}HVSooQi{3;8lXGCsmEc|Q+K^rVwKXO^UrNdGLN)aeUpcH{p z1WFMoMW7UcQUpp7C`F(Yfl>ticShjDijzKxY}{WSDNK3hav>s{db5+p2O=AHl|L;f zG`{c+z@y{y-vp^RS&)1y^^6X`K0ZFaS@HqNzhHPue2ot8G<=!lA2mEBx<-e8YIp0XRAwGdv}>Mu)F8JSDV7hieT#Rr2Q=o)TE2!>1UY z5?7fV*ZAw1Qh0PcuG8B{)VT76XtJtN;F~qhNlG6=5Vg81v zL=xt2cuF8){)c$}LCL>v_){eRg5fDqg!vnu5=5B4;VChM`5T@RLYTkdDG`ME8~!xO z-)eYD2x0z)r$i9u&pcM^^d+=fuMl#*fOXX;bG(Bt{(Tq! zwu^tw#lPg@pL6lMUHq?I{I6X6&s_XZT>KAR{CzII+r@8k@f%!x&c(O6__&L|*~Qnp z_&OIKcJWJG{6ZIhwu_(d;^(^fDi=T9#ZPhZhZeiezl(q0#lOvbq$rphSJ@M)I_)s%8zdcjx zzz1%It{I>7hu5K#4(g$Yn@3F~zo9?U7ak)=BZYAPleD4lpu0jO-#2tT_7OrO-BovE zr!x3XjqiZ9aS|K^fcwuJX@YE-oLW`?wxf^Ftux0&4lyhHQ5Kg!UI z$WbY16v+z#ZYO5w(@&H`PZCFeOe6dEkP@8|$i4@OeE7vec%b6(u}Gl~^Qwon967eP zHa|u(EozXl$i@w0qGHn{WTp_7T{p^U?7_Nx1-B_R=$&l7Zx6@eAwY$?7v-ScRsRCf z{PSe_b|orJRZ3I8QI5S#y>|mCYCUMQmg@n`gmyd3-(dvj0w4aCa<|_|f6gJj+(;i# z!II8wq#^RQGXaM)RYvOxsa22IQ9z~N6#t^%BeqKNGOSoYB3(>>^rJ)_hEl%$TsaUR|j>Pc{~_Nq(i(uo}+sUD1y z4?nwc!?U8|8wE0>JMNUj(>YxP*VQ@w1)DFP(*Zz*I*furyQ{|W6r*OjO^IUIzM)H{ zDGDPTld*XcNKxx~qm_o1jLq;iJTLZzd*#HmM{(&YUAfZp`)P!Hm=?X{zc?n>FpXkP z@_7#LVrHD92@k&*X(DUv)D(yCmyGahFic|3%jZs@!#i+FHimlxqZJFny~wdVGABgN6oJ@T?2PdhuI6_?B z4(Ba$I=g5(KJpch)znrz*1K`+=!!f0u=w)578o<#0cb>JBGFB>29 zaj@bs5vkZ=$L+a)MUudTO4D%yWT%lZ93K>i+IdE1qLhUtB4ef)5GeJ8B$yVOT? zV$j!+_;g-!i!jb+Vs_vjfvci#NcIOdZw&7Wz~*9X?BD*Dx}g+hE`SXv$*X5O9p|x~ ztKI!HB`doQp{sbYM)1W(UE;eBdTnGi*vN3z=LYii&%!tk;k zcIsklj< z38+vn>+w%eUfw!jY_e2h>KJIr5V5V6n*U-TC9xG$;77~=a?4# z816LrRpkHx=5WGkWxWt7R5hQY?CpXLve!~^pqK3z>fRy*8qr#aPQ01c@EVNd>-(j7HFLRU{=7I}!((RBU9(1qJl5!L<-Ti;-T|~PJV-Iq z`}&|`U+z<+$gWg5rw6Du4G-901#F%|u_S3#T zh<$xulw&A`(e|$XV3*g7e>F?GqxSzr>B0iXPgQ>>A@-#SZV-xNCgiegIGvr)M zQ{^dB4=(4xT0Z=dd~V6zj1zf3943FviChBdLfuD%;Y8kqM6o(0Q`k!P!edDXZqAca zXVD9d=x;H|z9=ln2~492(g^xK_A2<|0DnYgS`F}!mWy;R?1d>BV5h^EmyGCoh)#?x zLbm}StoldNrd{AJtP3-*`O5#XCn;5K9zzc7lg_ET6h{6L600=wI^fm_GT(19{4h{z zr3jQFP>Mh)0;LF)BJlqz0HO4Y=L9ca*j#wtME|qTPV6?;%$#`=x zn+ocexPp+vyIW@k@m5$S)xKzc=4jEBYK{xMmWLO-NJ=OrhX}tIClc9s8;mc)gA6^= z-H%657ldd z{5G9@@lDw8$T?nx;|LNRp|7MULU2b-;DoZpNgUFjB5eCt%oTLa>*M1+psk?&pgZ5f z+(83F^f=_R20^LuJ1>FT zt`mXI%D@TJjx67dKKd#CM0{q$haRM!#f-}3GiLu;#ng4>UE=aNix!-J>O7!QA3vvy zt%Kv^ln3HQc5lLG3-GO^uN&FS;`7Rz@RhzLhG$fMbJDWuN0ePuhN0z{-v#{fJLBWe zOt$-bCv*QV0Y8ZKAm4d4`mF)(e-Zs3A|}Vu98LexrvD?r?_h1ocVkWeY%~5W=)H?C zk5`f3rhn-qwtoTokD-hax=sHb)T?4f??x{|KiM(;WmC=gS>Pd@lSW&A(wCJ1!b+_a zfl>rY5hz8V6oFC%N)aeU;QxFCxIP5egW&oPT$(m{) zBf4Xjf=x{X{MD9h7wd`XVMUEL%zlpPz##iB8;^%S=ac>iwLJdhLh>9(e4UW|-)qP5 z*Ey1}(02Z(k&&CdJ*o~WyhhVinl@F34X^*D8nht0>sOgBN;uIlUrKUkm zt2M3Bbd{zJns#W~rRio(w`$s>X|JXOnht6@qA3@9s+_0%2x?lbX^p0b?3wbOiy6+usYni$Y=wzbN{bTR0O&oK2Zo$?ie_cjQWyr4Ze(A)k1lJ34;>QWE>$VJ)O1MsrQy#llgN>-;GI6}%b!qpO zVHfYv>(z<#ZeK(LIGij#7%w8~i0&hk#Kbt$t2Ko8;HOEuouYBRFLEEf9V0rINioMi zUh}0BI7v)g|5GqPz+;^pBk_sr@C12+CSDgIZO6T?7D;^K`lQ_2B)a1-yOYiS8J ziDJ3UXcko#so-WtM5E2A=-OneF_w%rXH)4+G?wecFG_74$v6r{hAytYu-aRQ>To8a zv2;4NJ{oV&q9A2UI@T7CHs{*f)dCk(=mCB?|a6{a*bL3IsKsy$g#W7mMB>Dg68 z`R7GdcT-lO233{fwBqb6-{~rO zsw-pa3Y@CG>BMQGT>01(S(PEgm$RR+I|Y%A@vF zHXd5ro(pxPG3j)6y$BhoF_%a-pPgtHQlK@KX%(U7_3bdGC~D=(9vwRe6GdM-o{W)# zKI%Zh*^s;fLpThrO@XC@OuR{ivhhx^@}du=Q?h(+D2@`ia${<3hBfAt4YemJ4L&9< z8WaYkgR!;*EMpuR1_;&U4K-#mB7|MC4LhIN%>Nq)^bLk`KDZsrwoM-}4QiYT=YyCJ z;lS4C@7YXGWkF4GyKuBxaQIn1#p81xjA^x&W0x*gR=P5-H47H&ASGd0+ zvmdHgh;QP+)(>huru2MjH-2Ve`W2rRB>DE6n{$IQF`TrBb zw8Ql<#pAL+KXvJIzJ)30n_1o+zelvbakEBCYT8M`u^OL0h91WOiLgHBZF=ORxn$PoyisQzKs1(9T#;w|U38F( ztk3y`F0J3eikb#pJpD}#t$o(#JQV-`Q3mw<>H4L{ Date: Thu, 10 Jun 2021 19:02:10 +0800 Subject: [PATCH 02/67] code optimization --- src/client/src/tscSQLParser.c | 3 +-- src/client/src/tscServer.c | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 4b6b6d585d..ece2ea5e92 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -7502,10 +7502,9 @@ int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo) { uint32_t maxSize = tscGetTableMetaMaxSize(); char name[TSDB_TABLE_FNAME_LEN] = {0}; - // char buf[80 * 1024] = {0}; assert(maxSize < 80 * TSDB_MAX_COLUMNS); if (!pSql->pBuf) { - if (NULL == (pSql->pBuf = calloc(1, 80 * TSDB_MAX_COLUMNS))) { + if (NULL == (pSql->pBuf = tcalloc(1, 80 * TSDB_MAX_COLUMNS))) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } } diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index dbc53c5429..49cbf8b600 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -2509,10 +2509,9 @@ int32_t tscGetTableMetaImpl(SSqlObj* pSql, STableMetaInfo *pTableMetaInfo, bool taosHashGetClone(tscTableMetaInfo, name, len, NULL, pTableMetaInfo->pTableMeta, -1); // TODO resize the tableMeta - // char buf[80 * TSDB_MAX_COLUMNS] = {0}; assert(size < 80 * TSDB_MAX_COLUMNS); if (!pSql->pBuf) { - if (NULL == (pSql->pBuf = calloc(1, 80 * TSDB_MAX_COLUMNS))) { + if (NULL == (pSql->pBuf = tcalloc(1, 80 * TSDB_MAX_COLUMNS))) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } } From 280f806287c55ca3323242428ea5f7643bdf8474 Mon Sep 17 00:00:00 2001 From: Kaili Xu Date: Fri, 11 Jun 2021 07:38:19 +0800 Subject: [PATCH 03/67] limit TSDB_MAX_BINARY_LEN and TSDB_MAX_NCHAR_LEN by 16K temporarily --- src/inc/taosdef.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 7c552a0da0..77e2155488 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -321,8 +321,8 @@ do { \ #define TSDB_MAX_JOIN_TABLE_NUM 10 #define TSDB_MAX_UNION_CLAUSE 5 -#define TSDB_MAX_BINARY_LEN (TSDB_MAX_BYTES_PER_ROW-TSDB_KEYSIZE) -#define TSDB_MAX_NCHAR_LEN (TSDB_MAX_BYTES_PER_ROW-TSDB_KEYSIZE) +#define TSDB_MAX_BINARY_LEN (16384-TSDB_KEYSIZE) // TD-4666: TODO use TSDB_MAX_BYTES_PER_ROW later. +#define TSDB_MAX_NCHAR_LEN (16384-TSDB_KEYSIZE) // TD-4666: TODO use TSDB_MAX_BYTES_PER_ROW later. #define PRIMARYKEY_TIMESTAMP_COL_INDEX 0 #define TSDB_MAX_RPC_THREADS 5 From 0c7f46e4861dce053a7a153640b50ccbd0b58869 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Tue, 29 Jun 2021 20:10:30 +0800 Subject: [PATCH 04/67] utilize SMemRow --- src/client/inc/tscUtil.h | 52 +++++++++ src/client/src/tscUtil.c | 157 ++++++++++++++++++++++++- src/common/inc/tdataformat.h | 219 ++++++++++++++++++++++++++++++----- src/common/src/tdataformat.c | 178 +++++++++++++++++++++++----- src/cq/src/cqMain.c | 10 +- src/inc/ttype.h | 2 +- src/tsdb/inc/tsdbMemTable.h | 12 +- src/tsdb/inc/tsdbMeta.h | 4 +- src/tsdb/src/tsdbCommit.c | 23 ++-- src/tsdb/src/tsdbMain.c | 16 +-- src/tsdb/src/tsdbMemTable.c | 119 +++++++++---------- src/tsdb/src/tsdbRead.c | 83 ++++++------- 12 files changed, 683 insertions(+), 192 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 2c4d711520..9d9eacb332 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -339,6 +339,58 @@ char* strdup_throw(const char* str); bool vgroupInfoIdentical(SNewVgroupInfo *pExisted, SVgroupMsg* src); SNewVgroupInfo createNewVgroupInfo(SVgroupMsg *pVgroupMsg); +typedef struct { + // for SDataRow + STSchema* pTSchema; + SSchema* pSchema; + int16_t sversion; + int32_t flen; + // for SKVRow + int16_t tCols; + int16_t nCols; + SColIdx* pColIdx; + uint16_t alloc; + uint16_t size; + void* buf; + + void* pDataBlock; + SSubmitBlk* pSubmitBlk; +} SMemRowBuilder; + +int tdInitMemRowBuilder(SMemRowBuilder* pBuilder); +void tdDestroyMemRowBuilder(SMemRowBuilder* pBuilder); +void tdResetMemRowBuilder(SMemRowBuilder* pBuilder); +SMemRow tdGetMemRowFromBuilder(SMemRowBuilder* pBuilder); + +static FORCE_INLINE int tdAddColToMemRow(SMemRowBuilder* pBuilder, int16_t colId, int8_t type, void* value) { + // TODO + + if (pBuilder->nCols >= pBuilder->tCols) { + pBuilder->tCols *= 2; + pBuilder->pColIdx = (SColIdx*)realloc((void*)(pBuilder->pColIdx), sizeof(SColIdx) * pBuilder->tCols); + if (pBuilder->pColIdx == NULL) return -1; + } + + pBuilder->pColIdx[pBuilder->nCols].colId = colId; + pBuilder->pColIdx[pBuilder->nCols].offset = pBuilder->size; + + pBuilder->nCols++; + + int tlen = IS_VAR_DATA_TYPE(type) ? varDataTLen(value) : TYPE_BYTES[type]; + if (tlen > pBuilder->alloc - pBuilder->size) { + while (tlen > pBuilder->alloc - pBuilder->size) { + pBuilder->alloc *= 2; + } + pBuilder->buf = realloc(pBuilder->buf, pBuilder->alloc); + if (pBuilder->buf == NULL) return -1; + } + + memcpy(POINTER_SHIFT(pBuilder->buf, pBuilder->size), value, tlen); + pBuilder->size += tlen; + + return 0; +} + #ifdef __cplusplus } #endif diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 0b9867971a..c7241860bc 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -14,7 +14,7 @@ */ #include "tscUtil.h" -#include "hash.h" +#include "hash.h" #include "os.h" #include "taosmsg.h" #include "texpr.h" @@ -1636,6 +1636,136 @@ int32_t tscGetDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, i return TSDB_CODE_SUCCESS; } +int tdInitMemRowBuilder(SMemRowBuilder* pBuilder) { + pBuilder->pSchema = NULL; + pBuilder->sversion = 0; + pBuilder->tCols = 128; + pBuilder->nCols = 0; + pBuilder->pColIdx = (SColIdx*)malloc(sizeof(SColIdx) * pBuilder->tCols); + if (pBuilder->pColIdx == NULL) return -1; + pBuilder->alloc = 1024; + pBuilder->size = 0; + pBuilder->buf = malloc(pBuilder->alloc); + if (pBuilder->buf == NULL) { + free(pBuilder->pColIdx); + return -1; + } + return 0; +} + +void tdDestroyMemRowBuilder(SMemRowBuilder* pBuilder) { + tfree(pBuilder->pColIdx); + tfree(pBuilder->buf); +} + +void tdResetMemRowBuilder(SMemRowBuilder* pBuilder) { + pBuilder->nCols = 0; + pBuilder->size = 0; +} + +#define KvRowNullColRatio 0.75 // If nullable column ratio larger than 0.75, utilize SKVRow, otherwise SDataRow. +#define KvRowNColsThresh 4096 // default value: 32 + +static FORCE_INLINE uint8_t tdRowTypeJudger(SSchema* pSchema, void* pData, int32_t nCols, int32_t flen, + uint16_t* nColsNotNull) { + ASSERT(pData != NULL); + if (nCols < KvRowNColsThresh) { + return SMEM_ROW_DATA; + } + int32_t dataRowLen = flen; + int32_t kvRowLen = 0; + + uint16_t nColsNull = 0; + char* p = (char*)pData; + for (int i = 0; i < nCols; ++i) { + if (IS_VAR_DATA_TYPE(pSchema[i].type)) { + dataRowLen += varDataTLen(p); + if (!isNull(p, pSchema[i].type)) { + kvRowLen += sizeof(SColIdx) + varDataTLen(p); + } else { + ++nColsNull; + } + } else { + if (!isNull(p, pSchema[i].type)) { + kvRowLen += sizeof(SColIdx) + varDataTLen(p); + } else { + ++nColsNull; + } + } + + // next column + p += pSchema[i].bytes; + } + + tscDebug("prop:nColsNull %d, nCols: %d, kvRowLen: %d, dataRowLen: %d", nColsNull, nCols, kvRowLen, dataRowLen); + + if (kvRowLen < dataRowLen) { + if (nColsNotNull) { + *nColsNotNull = nCols - nColsNull; + } + return SMEM_ROW_KV; + } + + return SMEM_ROW_DATA; +} + +SMemRow tdGetMemRowFromBuilder(SMemRowBuilder* pBuilder) { + SSchema* pSchema = pBuilder->pSchema; + char* p = (char*)pBuilder->buf; + + uint16_t nColsNotNull = 0; + uint8_t memRowType = tdRowTypeJudger(pSchema, p, pBuilder->nCols, pBuilder->flen, &nColsNotNull); + tscDebug("prop:memType is %d", memRowType); + + memRowType = SMEM_ROW_DATA; + SMemRow* memRow = (SMemRow)pBuilder->pDataBlock; + memRowSetType(memRow, memRowType); + + if (memRowType == SMEM_ROW_DATA) { + int toffset = 0; + SDataRow trow = (SDataRow)memRowBody(memRow); + dataRowSetLen(trow, (uint16_t)(TD_DATA_ROW_HEAD_SIZE + pBuilder->flen)); + dataRowSetVersion(trow, pBuilder->sversion); + + p = (char*)pBuilder->buf; + for (int32_t j = 0; j < pBuilder->nCols; ++j) { + tdAppendColVal(trow, p, pSchema[j].type, pSchema[j].bytes, toffset); + toffset += TYPE_BYTES[pSchema[j].type]; + p += pSchema[j].bytes; + } + pBuilder->buf = p; + } else { + uint16_t tlen = TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * nColsNotNull + pBuilder->size; + SKVRow row = (SKVRow)pBuilder->pDataBlock; + + kvRowSetNCols(row, nColsNotNull); + kvRowSetLen(row, tlen); + + memcpy(kvRowColIdx(row), pBuilder->pColIdx, sizeof(SColIdx) * pBuilder->nCols); + memcpy(kvRowValues(row), pBuilder->buf, pBuilder->size); + } + + pBuilder->pDataBlock = (char*)pBuilder->pDataBlock + memRowTLen(memRow); // next row + pBuilder->pSubmitBlk->dataLen += memRowTLen(memRow); + + // int tlen = sizeof(SColIdx) * pBuilder->nCols + pBuilder->size; + // if (tlen == 0) return NULL; + + // tlen += TD_KV_ROW_HEAD_SIZE; + + // SKVRow row = malloc(tlen); + // if (row == NULL) return NULL; + + // kvRowSetNCols(row, pBuilder->nCols); + // kvRowSetLen(row, tlen); + + // memcpy(kvRowColIdx(row), pBuilder->pColIdx, sizeof(SColIdx) * pBuilder->nCols); + // memcpy(kvRowValues(row), pBuilder->buf, pBuilder->size); + + return NULL; +} + +// Erase the empty space reserved for binary data static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, bool includeSchema) { // TODO: optimize this function, handle the case while binary is not presented STableMeta* pTableMeta = pTableDataBlock->pTableMeta; @@ -1675,12 +1805,24 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, bo char* p = pTableDataBlock->pData + sizeof(SSubmitBlk); pBlock->dataLen = 0; int32_t numOfRows = htons(pBlock->numOfRows); - + + SMemRowBuilder mRowBuilder; + mRowBuilder.pSchema = pSchema; + mRowBuilder.sversion = pTableMeta->sversion; + mRowBuilder.flen = flen; + mRowBuilder.nCols = tinfo.numOfColumns; + mRowBuilder.pDataBlock = pDataBlock; + mRowBuilder.pSubmitBlk = pBlock; + mRowBuilder.buf = p; + mRowBuilder.size = 0; + for (int32_t i = 0; i < numOfRows; ++i) { - SDataRow trow = (SDataRow) pDataBlock; +#if 0 + SDataRow trow = (SDataRow)pDataBlock; // generate each SDataRow one by one dataRowSetLen(trow, (uint16_t)(TD_DATA_ROW_HEAD_SIZE + flen)); dataRowSetVersion(trow, pTableMeta->sversion); + // scan each column data and generate the data row int toffset = 0; for (int32_t j = 0; j < tinfo.numOfColumns; j++) { tdAppendColVal(trow, p, pSchema[j].type, pSchema[j].bytes, toffset); @@ -1688,8 +1830,10 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, bo p += pSchema[j].bytes; } - pDataBlock = (char*)pDataBlock + dataRowLen(trow); - pBlock->dataLen += dataRowLen(trow); + pDataBlock = (char*)pDataBlock + dataRowLen(trow); // next SDataRow + pBlock->dataLen += dataRowLen(trow); // SSubmitBlk data length +#endif + tdGetMemRowFromBuilder(&mRowBuilder); } int32_t len = pBlock->dataLen + pBlock->schemaLen; @@ -1701,13 +1845,14 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, bo static int32_t getRowExpandSize(STableMeta* pTableMeta) { int32_t result = TD_DATA_ROW_HEAD_SIZE; - int32_t columns = tscGetNumOfColumns(pTableMeta); + int32_t columns = tscGetNumOfColumns(pTableMeta); SSchema* pSchema = tscGetTableSchema(pTableMeta); for(int32_t i = 0; i < columns; i++) { if (IS_VAR_DATA_TYPE((pSchema + i)->type)) { result += TYPE_BYTES[TSDB_DATA_TYPE_BINARY]; } } + result += TD_MEM_ROW_TYPE_SIZE; // add len of SMemRow flag return result; } diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index 8ee7329156..294d8b2cf8 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -24,6 +24,31 @@ extern "C" { #endif +typedef struct { + VarDataLenT len; + uint8_t data; +} SBinaryNullT; + +typedef struct { + VarDataLenT len; + uint32_t data; +} SNCharNullT; + +extern const uint8_t BoolNull; +extern const uint8_t TinyintNull; +extern const uint16_t SmallintNull; +extern const uint32_t IntNull; +extern const uint64_t BigintNull; +extern const uint64_t TimestampNull; +extern const uint8_t UTinyintNull; +extern const uint16_t USmallintNull; +extern const uint32_t UIntNull; +extern const uint64_t UBigintNull; +extern const uint32_t FloatNull; +extern const uint64_t DoubleNull; +extern const SBinaryNullT BinaryNull; +extern const SNCharNullT NcharNull; + #define STR_TO_VARSTR(x, str) \ do { \ VarDataLenT __len = (VarDataLenT)strlen(str); \ @@ -45,10 +70,10 @@ extern "C" { // ----------------- TSDB COLUMN DEFINITION typedef struct { - int8_t type; // Column type - int16_t colId; // column ID - int16_t bytes; // column bytes - int16_t offset; // point offset in SDataRow after the header part + int8_t type; // Column type + int16_t colId; // column ID + uint16_t bytes; // column bytes + uint16_t offset; // point offset in SDataRow/SKVRow after the header part. } STColumn; #define colType(col) ((col)->type) @@ -159,9 +184,9 @@ static FORCE_INLINE int tkeyComparFn(const void *tkey1, const void *tkey2) { return 0; } } -// ----------------- Data row structure +// ----------------- Sequential Data row structure -/* A data row, the format is like below: +/* A sequential data row, the format is like below: * |<--------------------+--------------------------- len ---------------------------------->| * |<-- Head -->|<--------- flen -------------->| | * +---------------------+---------------------------------+---------------------------------+ @@ -173,6 +198,18 @@ static FORCE_INLINE int tkeyComparFn(const void *tkey1, const void *tkey2) { * NOTE: timestamp in this row structure is TKEY instead of TSKEY */ typedef void *SDataRow; +/* A memory data row, the format is like below: + *|---------+---------------------+--------------------------- len ---------------------------------->| + *|<- type->|<-- Head -->|<--------- flen -------------->| | + *|---------+---------------------+---------------------------------+---------------------------------+ + *| uint8_t | uint16_t | int16_t | | | + *|---------+----------+----------+---------------------------------+---------------------------------+ + *| flag | len | sversion | First part | Second part | + *|---------+----------+----------+---------------------------------+---------------------------------+ + * + * NOTE: timestamp in this row structure is TKEY instead of TSKEY + */ +typedef void *SMemRow; #define TD_DATA_ROW_HEAD_SIZE (sizeof(uint16_t) + sizeof(int16_t)) @@ -187,13 +224,14 @@ typedef void *SDataRow; #define dataRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_DATA_ROW_HEAD_SIZE) #define dataRowDeleted(r) TKEY_IS_DELETED(dataRowTKey(r)) -SDataRow tdNewDataRowFromSchema(STSchema *pSchema); -void tdFreeDataRow(SDataRow row); +// SDataRow tdNewDataRowFromSchema(STSchema *pSchema); +// void tdFreeDataRow(SDataRow row); void tdInitDataRow(SDataRow row, STSchema *pSchema); -SDataRow tdDataRowDup(SDataRow row); +// SDataRow tdDataRowDup(SDataRow row); +SMemRow tdMemRowDup(SMemRow row); // offset here not include dataRow header length -static FORCE_INLINE int tdAppendColVal(SDataRow row, void *value, int8_t type, int32_t bytes, int32_t offset) { +static FORCE_INLINE int tdAppendColVal(SDataRow row, const void *value, int8_t type, int32_t bytes, int32_t offset) { ASSERT(value != NULL); int32_t toffset = offset + TD_DATA_ROW_HEAD_SIZE; char * ptr = (char *)POINTER_SHIFT(row, dataRowLen(row)); @@ -215,15 +253,6 @@ static FORCE_INLINE int tdAppendColVal(SDataRow row, void *value, int8_t type, i return 0; } -// NOTE: offset here including the header size -static FORCE_INLINE void *tdGetRowDataOfCol(SDataRow row, int8_t type, int32_t offset) { - if (IS_VAR_DATA_TYPE(type)) { - return POINTER_SHIFT(row, *(VarDataOffsetT *)POINTER_SHIFT(row, offset)); - } else { - return POINTER_SHIFT(row, offset); - } -} - // ----------------- Data column structure typedef struct SDataCol { int8_t type; // column type @@ -237,17 +266,56 @@ typedef struct SDataCol { TSKEY ts; // only used in last NULL column } SDataCol; +#define isAllRowOfColNull(pCol) ((pCol)->len == 0) static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; } void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints); -void dataColAppendVal(SDataCol *pCol, void *value, int numOfRows, int maxPoints); +void dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints); void dataColSetOffset(SDataCol *pCol, int nEle); bool isNEleNull(SDataCol *pCol, int nEle); void dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints); +static const void *tdGetNullVal(int8_t type) { + switch (type) { + case TSDB_DATA_TYPE_BOOL: + return &BoolNull; + case TSDB_DATA_TYPE_TINYINT: + return &TinyintNull; + case TSDB_DATA_TYPE_SMALLINT: + return &SmallintNull; + case TSDB_DATA_TYPE_INT: + return &IntNull; + case TSDB_DATA_TYPE_BIGINT: + return &BigintNull; + case TSDB_DATA_TYPE_FLOAT: + return &FloatNull; + case TSDB_DATA_TYPE_DOUBLE: + return &DoubleNull; + case TSDB_DATA_TYPE_BINARY: + return &BinaryNull; + case TSDB_DATA_TYPE_TIMESTAMP: + return &TimestampNull; + case TSDB_DATA_TYPE_NCHAR: + return &NcharNull; + case TSDB_DATA_TYPE_UTINYINT: + return &UTinyintNull; + case TSDB_DATA_TYPE_USMALLINT: + return &USmallintNull; + case TSDB_DATA_TYPE_UINT: + return &UIntNull; + case TSDB_DATA_TYPE_UBIGINT: + return &UBigintNull; + default: + ASSERT(0); + return NULL; + } +} // Get the data pointer from a column-wised data -static FORCE_INLINE void *tdGetColDataOfRow(SDataCol *pCol, int row) { +static FORCE_INLINE const void *tdGetColDataOfRow(SDataCol *pCol, int row) { + if (isAllRowOfColNull(pCol)) { + return tdGetNullVal(pCol->type); + } if (IS_VAR_DATA_TYPE(pCol->type)) { return POINTER_SHIFT(pCol->pData, pCol->dataOff[row]); } else { @@ -279,7 +347,7 @@ typedef struct { } SDataCols; #define keyCol(pCols) (&((pCols)->cols[0])) // Key column -#define dataColsTKeyAt(pCols, idx) ((TKEY *)(keyCol(pCols)->pData))[(idx)] +#define dataColsTKeyAt(pCols, idx) ((TKEY *)(keyCol(pCols)->pData))[(idx)] // the idx row of column-wised data #define dataColsKeyAt(pCols, idx) tdGetKey(dataColsTKeyAt(pCols, idx)) static FORCE_INLINE TKEY dataColsTKeyFirst(SDataCols *pCols) { if (pCols->numOfRows) { @@ -318,13 +386,13 @@ void tdResetDataCols(SDataCols *pCols); int tdInitDataCols(SDataCols *pCols, STSchema *pSchema); SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData); SDataCols *tdFreeDataCols(SDataCols *pCols); -void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols); +void tdAppendMemRowToDataCol(SMemRow row, STSchema *pSchema, SDataCols *pCols); int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int *pOffset); // ----------------- K-V data row structure /* * +----------+----------+---------------------------------+---------------------------------+ - * | int16_t | int16_t | | | + * | uint16_t | int16_t | | | * +----------+----------+---------------------------------+---------------------------------+ * | len | ncols | cols index | data part | * +----------+----------+---------------------------------+---------------------------------+ @@ -332,13 +400,13 @@ int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge typedef void *SKVRow; typedef struct { - int16_t colId; - int16_t offset; + int16_t colId; + uint16_t offset; } SColIdx; #define TD_KV_ROW_HEAD_SIZE (2 * sizeof(int16_t)) -#define kvRowLen(r) (*(int16_t *)(r)) +#define kvRowLen(r) (*(uint16_t *)(r)) #define kvRowNCols(r) (*(int16_t *)POINTER_SHIFT(r, sizeof(int16_t))) #define kvRowSetLen(r, len) kvRowLen(r) = (len) #define kvRowSetNCols(r, n) kvRowNCols(r) = (n) @@ -349,6 +417,11 @@ typedef struct { #define kvRowColIdxAt(r, i) (kvRowColIdx(r) + (i)) #define kvRowFree(r) tfree(r) #define kvRowEnd(r) POINTER_SHIFT(r, kvRowLen(r)) +#define kvRowVersion(r) (-1) + +#define kvRowTKey(r) (*(TKEY *)(kvRowValues(r))) +#define kvRowKey(r) tdGetKey(kvRowTKey(r)) +#define kvRowDeleted(r) TKEY_IS_DELETED(kvRowTKey(r)) SKVRow tdKVRowDup(SKVRow row); int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value); @@ -377,8 +450,8 @@ typedef struct { int16_t tCols; int16_t nCols; SColIdx *pColIdx; - int16_t alloc; - int16_t size; + uint16_t alloc; + uint16_t size; void * buf; } SKVRowBuilder; @@ -414,6 +487,94 @@ static FORCE_INLINE int tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t colId, return 0; } +// ----------------- Data row structure + +// ----------------- Sequential Data row structure +/* A sequential data row, the format is like below: + * |<--------------------+--------------------------- len ---------------------------------->| + * |<-- Head -->|<--------- flen -------------->| | + * +---------------------+---------------------------------+---------------------------------+ + * | uint16_t | int16_t | | | + * +----------+----------+---------------------------------+---------------------------------+ + * | len | sversion | First part | Second part | + * +----------+----------+---------------------------------+---------------------------------+ + * + * NOTE: timestamp in this row structure is TKEY instead of TSKEY + */ +// ----------------- K-V data row structure +/* + * +----------+----------+---------------------------------+---------------------------------+ + * | int16_t | int16_t | | | + * +----------+----------+---------------------------------+---------------------------------+ + * | len | ncols | cols index | data part | + * +----------+----------+---------------------------------+---------------------------------+ + */ + +#define TD_MEM_ROW_TYPE_SIZE sizeof(uint8_t) +#define TD_MEM_ROW_HEAD_SIZE (TD_MEM_ROW_TYPE_SIZE + sizeof(uint16_t) + sizeof(int16_t)) + +#define SMEM_ROW_DATA 0U // SDataRow +#define SMEM_ROW_KV 1U // SKVRow +#define TD_DO_NOTHING \ + do { \ + } while (0) + +#define memRowType(r) (*(uint8_t *)(r)) +#define memRowBody(r) POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE) +#define memRowLen(r) (*(uint16_t *)POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE)) +#define memRowTLen(r) (*(uint16_t *)POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE) + (uint16_t)TD_MEM_ROW_TYPE_SIZE) + +#define isDataRow(r) (SMEM_ROW_DATA == memRowType(r)) +#define isKvRow(r) (SMEM_ROW_KV == memRowType(r)) +#define memRowVersion(r) (isDataRow(r) ? dataRowVersion(memRowBody(r)) : kvRowVersion(r)) // schema version + +#define memRowTuple(r) (isDataRow(r) ? dataRowTuple(memRowBody(r)) : kvRowValues(memRowBody(r))) + +#define memRowTKey(r) (isDataRow(r) ? dataRowTKey(memRowBody(r)) : kvRowTKey(memRowBody(r))) +#define memRowKey(r) (isDataRow(r) ? dataRowKey(memRowBody(r)) : kvRowKey(memRowBody(r))) + +#define memRowSetType(r, t) (memRowType(r) = (t)) +#define memRowSetLen(r, l) (memRowLen(r) = (l)) +#define memRowSetVersion(r, v) (isDataRow(r) ? dataRowSetVersion(r, v) : TD_DO_NOTHING) +#define memRowCpy(dst, r) memcpy((dst), (r), memRowTLen(r)) +#define memRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_MEM_ROW_HEAD_SIZE) +#define memRowDeleted(r) TKEY_IS_DELETED(memRowTKey(r)) + +#define memRowNCols(r) (*(int16_t *)POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE + sizeof(int16_t))) // for SKVRow +#define memRowSetNCols(r, n) memRowNCols(r) = (n) // for SKVRow +#define memRowColIdx(r) (SColIdx *)POINTER_SHIFT(r, TD_MEM_ROW_HEAD_SIZE) // for SKVRow +#define memRowValues(r) POINTER_SHIFT(r, TD_MEM_ROW_HEAD_SIZE + sizeof(SColIdx) * memRowNCols(r)) // for SKVRow + +// NOTE: offset here including the header size +static FORCE_INLINE void *tdGetRowDataOfCol(void *row, int8_t type, int32_t offset) { + if (IS_VAR_DATA_TYPE(type)) { + return POINTER_SHIFT(row, *(VarDataOffsetT *)POINTER_SHIFT(row, offset)); + } else { + return POINTER_SHIFT(row, offset); + } + return NULL; +} +static FORCE_INLINE void *tdGetKvRowDataOfCol(void *row, int8_t type, int32_t offset) { + return POINTER_SHIFT(row, offset); +} + +static FORCE_INLINE void *tdGetMemRowDataOfCol(void *row, int8_t type, int32_t offset) { + if (isDataRow(row)) { + return tdGetRowDataOfCol(row, type, offset); + } else if (isKvRow(row)) { + return tdGetKvRowDataOfCol(row, type, offset); + } else { + ASSERT(0); + } + return NULL; +} + +// #define kvRowCpy(dst, r) memcpy((dst), (r), kvRowLen(r)) +// #define kvRowColVal(r, colIdx) POINTER_SHIFT(kvRowValues(r), (colIdx)->offset) +// #define kvRowColIdxAt(r, i) (kvRowColIdx(r) + (i)) +// #define kvRowFree(r) tfree(r) +// #define kvRowEnd(r) POINTER_SHIFT(r, kvRowLen(r)) + #ifdef __cplusplus } #endif diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index 7ae34d532c..a5819ac858 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -18,6 +18,21 @@ #include "tcoding.h" #include "wchar.h" +const uint8_t BoolNull = TSDB_DATA_BOOL_NULL; +const uint8_t TinyintNull = TSDB_DATA_TINYINT_NULL; +const uint16_t SmallintNull = TSDB_DATA_SMALLINT_NULL; +const uint32_t IntNull = TSDB_DATA_INT_NULL; +const uint64_t BigintNull = TSDB_DATA_BIGINT_NULL; +const uint64_t TimestampNull = TSDB_DATA_BIGINT_NULL; +const uint8_t UTinyintNull = TSDB_DATA_UTINYINT_NULL; +const uint16_t USmallintNull = TSDB_DATA_USMALLINT_NULL; +const uint32_t UIntNull = TSDB_DATA_UINT_NULL; +const uint64_t UBigintNull = TSDB_DATA_UBIGINT_NULL; +const uint32_t FloatNull = TSDB_DATA_FLOAT_NULL; +const uint64_t DoubleNull = TSDB_DATA_DOUBLE_NULL; +const SBinaryNullT BinaryNull = {1, TSDB_DATA_BINARY_NULL}; +const SNCharNullT NcharNull = {4, TSDB_DATA_NCHAR_NULL}; + static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, int limit2, int tRows); @@ -173,36 +188,43 @@ void tdInitDataRow(SDataRow row, STSchema *pSchema) { dataRowSetVersion(row, schemaVersion(pSchema)); } -SDataRow tdNewDataRowFromSchema(STSchema *pSchema) { - int32_t size = dataRowMaxBytesFromSchema(pSchema); +// SDataRow tdNewDataRowFromSchema(STSchema *pSchema) { +// int32_t size = dataRowMaxBytesFromSchema(pSchema); - SDataRow row = malloc(size); - if (row == NULL) return NULL; +// SDataRow row = malloc(size); +// if (row == NULL) return NULL; - tdInitDataRow(row, pSchema); - return row; -} +// tdInitDataRow(row, pSchema); +// return row; +// } /** * Free the SDataRow object */ -void tdFreeDataRow(SDataRow row) { - if (row) free(row); -} +// void tdFreeDataRow(SDataRow row) { +// if (row) free(row); +// } -SDataRow tdDataRowDup(SDataRow row) { - SDataRow trow = malloc(dataRowLen(row)); +// SDataRow tdDataRowDup(SDataRow row) { +// SDataRow trow = malloc(dataRowLen(row)); +// if (trow == NULL) return NULL; + +// dataRowCpy(trow, row); +// return trow; +// } + +SMemRow tdMemRowDup(SMemRow row) { + SMemRow trow = malloc(memRowTLen(row)); if (trow == NULL) return NULL; - dataRowCpy(trow, row); + memRowCpy(trow, row); return trow; } - void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints) { pDataCol->type = colType(pCol); pDataCol->colId = colColId(pCol); pDataCol->bytes = colBytes(pCol); - pDataCol->offset = colOffset(pCol) + TD_DATA_ROW_HEAD_SIZE; + pDataCol->offset = colOffset(pCol) + TD_MEM_ROW_HEAD_SIZE; pDataCol->len = 0; if (IS_VAR_DATA_TYPE(pDataCol->type)) { @@ -219,9 +241,21 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints) } // value from timestamp should be TKEY here instead of TSKEY -void dataColAppendVal(SDataCol *pCol, void *value, int numOfRows, int maxPoints) { +void dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints) { ASSERT(pCol != NULL && value != NULL); + if (pCol->len == 0) { + if (isNull(value, pCol->type)) { + // all null value yet, just return + return; + } + + if (numOfRows > 0) { + // Find the first not null value, fill all previous values as NULL + dataColSetNEleNull(pCol, numOfRows, maxPoints); + } + } + if (IS_VAR_DATA_TYPE(pCol->type)) { // set offset pCol->dataOff[numOfRows] = pCol->len; @@ -399,11 +433,10 @@ void tdResetDataCols(SDataCols *pCols) { } } } - -void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols) { +static void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols) { ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < dataRowKey(row)); - int rcol = 0; + int rcol = 0; // rowCol int dcol = 0; if (dataRowDeleted(row)) { @@ -419,7 +452,7 @@ void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols) while (dcol < pCols->numOfCols) { SDataCol *pDataCol = &(pCols->cols[dcol]); if (rcol >= schemaNCols(pSchema)) { - dataColSetNullAt(pDataCol, pCols->numOfRows); + dataColAppendVal(pDataCol, tdGetNullVal(pDataCol->type), pCols->numOfRows, pCols->maxPoints); dcol++; continue; } @@ -433,7 +466,7 @@ void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols) } else if (pRowCol->colId < pDataCol->colId) { rcol++; } else { - dataColSetNullAt(pDataCol, pCols->numOfRows); + dataColAppendVal(pDataCol, tdGetNullVal(pDataCol->type), pCols->numOfRows, pCols->maxPoints); dcol++; } } @@ -441,6 +474,98 @@ void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols) pCols->numOfRows++; } +static void tdGetKVRowColInfo(const STSchema *pSchema, SColIdx *pColIdx, int nRowCols, STColumn *pSTColumn, + int *nColMatched) { + int nSchema = schemaNCols(pSchema); + int iCol = 0; + int iSchema = 0; + int nColMatch = 0; + SColIdx * pIdx = pColIdx; + const STColumn *pColumn = NULL; + + while (iCol < nRowCols && iSchema < nSchema) { + pColumn = &pSchema->columns[iSchema]; + if (pIdx->colId == pColumn->colId) { + pSTColumn[nColMatch].colId = pIdx->colId; + pSTColumn[nColMatch].type = pColumn->type; + pSTColumn[nColMatch].bytes = pColumn->bytes; + pSTColumn[nColMatch].offset = pIdx->offset; + + pIdx += sizeof(SColIdx); + + ++iCol; + ++iSchema; + ++nColMatch; + } else if (pIdx->colId > pColumn->colId) { + ++iSchema; + } else { + pIdx += sizeof(SColIdx); + ++iCol; + } + } + *nColMatched = nColMatch; +} + +static void tdAppendKvRowToDataCol(SKVRow row, STSchema *pSchema, SDataCols *pCols) { + ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < kvRowKey(row)); + + int rcol = 0; + int dcol = 0; + + if (kvRowDeleted(row)) { + for (; dcol < pCols->numOfCols; dcol++) { + SDataCol *pDataCol = &(pCols->cols[dcol]); + if (dcol == 0) { + dataColAppendVal(pDataCol, kvRowValues(row), pCols->numOfRows, pCols->maxPoints); + } else { + dataColSetNullAt(pDataCol, pCols->numOfRows); + } + } + } else { + int nRowCols = kvRowNCols(row); + int nRowColsMatched = 0; + STColumn stColumn[nRowCols]; + tdGetKVRowColInfo(pSchema, kvRowColIdx(row), nRowCols, stColumn, &nRowColsMatched); + uDebug("kvRow: nRowCols=%d, nRowColsMatched=%d, nSchemaCols=%d", nRowCols, nRowColsMatched, schemaNCols(pSchema)); + + while (dcol < pCols->numOfCols) { + SDataCol *pDataCol = &(pCols->cols[dcol]); + if (rcol >= nRowColsMatched || rcol >= schemaNCols(pSchema)) { + dataColAppendVal(pDataCol, tdGetNullVal(pDataCol->type), pCols->numOfRows, pCols->maxPoints); + dcol++; + continue; + } + + SColIdx *colIdx = kvRowColIdxAt(row, rcol); + + if (colIdx->colId == pDataCol->colId) { + ASSERT(pDataCol->type == stColumn[rcol].type); + + void *value = tdGetKvRowDataOfCol(row, pDataCol->type, stColumn[rcol].offset + TD_KV_ROW_HEAD_SIZE); + dataColAppendVal(pDataCol, value, pCols->numOfRows, pCols->maxPoints); + dcol++; + rcol++; + } else if (colIdx->colId < pDataCol->colId) { + rcol++; + } else { + dataColAppendVal(pDataCol, tdGetNullVal(pDataCol->type), pCols->numOfRows, pCols->maxPoints); + dcol++; + } + } + } + pCols->numOfRows++; +} + +void tdAppendMemRowToDataCol(SMemRow row, STSchema *pSchema, SDataCols *pCols) { + if (isDataRow(row)) { + tdAppendDataRowToDataCol(memRowBody(row), pSchema, pCols); + } else if (isKvRow(row)) { + tdAppendKvRowToDataCol(memRowBody(row), pSchema, pCols); + } else { + ASSERT(0); + } +} + int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int *pOffset) { ASSERT(rowsToMerge > 0 && rowsToMerge <= source->numOfRows); ASSERT(target->numOfCols == source->numOfCols); @@ -559,11 +684,11 @@ int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value) { void * ptr = taosbsearch(&colId, kvRowColIdx(row), kvRowNCols(row), sizeof(SColIdx), comparTagId, TD_GE); if (ptr == NULL || ((SColIdx *)ptr)->colId > colId) { // need to add a column value to the row - int diff = IS_VAR_DATA_TYPE(type) ? varDataTLen(value) : TYPE_BYTES[type]; + uint16_t diff = IS_VAR_DATA_TYPE(type) ? varDataTLen(value) : TYPE_BYTES[type]; nrow = malloc(kvRowLen(row) + sizeof(SColIdx) + diff); if (nrow == NULL) return -1; - kvRowSetLen(nrow, kvRowLen(row) + (int16_t)sizeof(SColIdx) + diff); + kvRowSetLen(nrow, kvRowLen(row) + (uint16_t)sizeof(SColIdx) + diff); kvRowSetNCols(nrow, kvRowNCols(row) + 1); if (ptr == NULL) { @@ -605,8 +730,8 @@ int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value) { if (varDataTLen(value) == varDataTLen(pOldVal)) { // just update the column value in place memcpy(pOldVal, value, varDataTLen(value)); } else { // need to reallocate the memory - int16_t diff = varDataTLen(value) - varDataTLen(pOldVal); - int16_t nlen = kvRowLen(row) + diff; + uint16_t diff = varDataTLen(value) - varDataTLen(pOldVal); + uint16_t nlen = kvRowLen(row) + diff; ASSERT(nlen > 0); nrow = malloc(nlen); if (nrow == NULL) return -1; @@ -693,7 +818,7 @@ void tdResetKVRowBuilder(SKVRowBuilder *pBuilder) { } SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder) { - int tlen = sizeof(SColIdx) * pBuilder->nCols + pBuilder->size; + uint16_t tlen = sizeof(SColIdx) * pBuilder->nCols + pBuilder->size; if (tlen == 0) return NULL; tlen += TD_KV_ROW_HEAD_SIZE; @@ -709,3 +834,4 @@ SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder) { return row; } + diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c index f539e77253..a460b1d619 100644 --- a/src/cq/src/cqMain.c +++ b/src/cq/src/cqMain.c @@ -483,8 +483,8 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) { SSubmitMsg *pMsg = (SSubmitMsg *) (buffer + sizeof(SWalHead)); SSubmitBlk *pBlk = (SSubmitBlk *) (buffer + sizeof(SWalHead) + sizeof(SSubmitMsg)); - SDataRow trow = (SDataRow)pBlk->data; - tdInitDataRow(trow, pSchema); + SMemRow trow = (SMemRow)pBlk->data; + tdInitDataRow(POINTER_SHIFT(trow, TD_MEM_ROW_TYPE_SIZE), pSchema); for (int32_t i = 0; i < pSchema->numOfCols; i++) { STColumn *c = pSchema->columns + i; @@ -500,9 +500,9 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) { memcpy((char *)val + sizeof(VarDataLenT), buf, len); varDataLen(val) = len; } - tdAppendColVal(trow, val, c->type, c->bytes, c->offset); + tdAppendColVal(POINTER_SHIFT(trow, TD_MEM_ROW_TYPE_SIZE), val, c->type, c->bytes, c->offset); } - pBlk->dataLen = htonl(dataRowLen(trow)); + pBlk->dataLen = htonl(memRowTLen(trow)); pBlk->schemaLen = 0; pBlk->uid = htobe64(pObj->uid); @@ -511,7 +511,7 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) { pBlk->sversion = htonl(pSchema->version); pBlk->padding = 0; - pHead->len = sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + dataRowLen(trow); + pHead->len = sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + memRowTLen(trow); pMsg->header.vgId = htonl(pContext->vgId); pMsg->header.contLen = htonl(pHead->len); diff --git a/src/inc/ttype.h b/src/inc/ttype.h index 9949f31c59..0308379509 100644 --- a/src/inc/ttype.h +++ b/src/inc/ttype.h @@ -11,7 +11,7 @@ extern "C" { // ----------------- For variable data types such as TSDB_DATA_TYPE_BINARY and TSDB_DATA_TYPE_NCHAR typedef int32_t VarDataOffsetT; -typedef int16_t VarDataLenT; +typedef int16_t VarDataLenT; // maxDataLen: 32767 typedef struct tstr { VarDataLenT len; diff --git a/src/tsdb/inc/tsdbMemTable.h b/src/tsdb/inc/tsdbMemTable.h index babb7024b2..67e9976c70 100644 --- a/src/tsdb/inc/tsdbMemTable.h +++ b/src/tsdb/inc/tsdbMemTable.h @@ -71,27 +71,27 @@ int tsdbLoadDataFromCache(STable* pTable, SSkipListIterator* pIter, TSKEY maxK TKEY* filterKeys, int nFilterKeys, bool keepDup, SMergeInfo* pMergeInfo); void* tsdbCommitData(STsdbRepo* pRepo); -static FORCE_INLINE SDataRow tsdbNextIterRow(SSkipListIterator* pIter) { +static FORCE_INLINE SMemRow tsdbNextIterRow(SSkipListIterator* pIter) { if (pIter == NULL) return NULL; SSkipListNode* node = tSkipListIterGet(pIter); if (node == NULL) return NULL; - return (SDataRow)SL_GET_NODE_DATA(node); + return (SMemRow)SL_GET_NODE_DATA(node); } static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator* pIter) { - SDataRow row = tsdbNextIterRow(pIter); + SMemRow row = tsdbNextIterRow(pIter); if (row == NULL) return TSDB_DATA_TIMESTAMP_NULL; - return dataRowKey(row); + return memRowKey(row); } static FORCE_INLINE TKEY tsdbNextIterTKey(SSkipListIterator* pIter) { - SDataRow row = tsdbNextIterRow(pIter); + SMemRow row = tsdbNextIterRow(pIter); if (row == NULL) return TKEY_NULL; - return dataRowTKey(row); + return memRowTKey(row); } #endif /* _TD_TSDB_MEMTABLE_H_ */ \ No newline at end of file diff --git a/src/tsdb/inc/tsdbMeta.h b/src/tsdb/inc/tsdbMeta.h index 45bbd5a7c6..9a8de01f71 100644 --- a/src/tsdb/inc/tsdbMeta.h +++ b/src/tsdb/inc/tsdbMeta.h @@ -32,7 +32,7 @@ typedef struct STable { void* eventHandler; // TODO void* streamHandler; // TODO TSKEY lastKey; - SDataRow lastRow; + SMemRow lastRow; char* sql; void* cqhandle; SRWLatch latch; // TODO: implementa latch functions @@ -148,7 +148,7 @@ static FORCE_INLINE STSchema *tsdbGetTableTagSchema(STable *pTable) { } static FORCE_INLINE TSKEY tsdbGetTableLastKeyImpl(STable* pTable) { - ASSERT(pTable->lastRow == NULL || pTable->lastKey == dataRowKey(pTable->lastRow)); + ASSERT((pTable->lastRow == NULL) || (pTable->lastKey == memRowKey(pTable->lastRow))); return pTable->lastKey; } diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index 82cc6f07f7..d0bad840e0 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -920,7 +920,8 @@ int tsdbWriteBlockImpl(STsdbRepo *pRepo, STable *pTable, SDFile *pDFile, SDataCo SDataCol * pDataCol = pDataCols->cols + ncol; SBlockCol *pBlockCol = pBlockData->cols + nColsNotAllNull; - if (isNEleNull(pDataCol, rowsToWrite)) { // all data to commit are NULL, just ignore it + // if (isNEleNull(pDataCol, rowsToWrite)) { // all data to commit are NULL, just ignore it + if (isAllRowOfColNull(pDataCol)) { // all data to commit are NULL, just ignore it continue; } @@ -1264,12 +1265,12 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt while (true) { key1 = (*iter >= pDataCols->numOfRows) ? INT64_MAX : dataColsKeyAt(pDataCols, *iter); bool isRowDel = false; - SDataRow row = tsdbNextIterRow(pCommitIter->pIter); - if (row == NULL || dataRowKey(row) > maxKey) { + SMemRow row = tsdbNextIterRow(pCommitIter->pIter); + if (row == NULL || memRowKey(row) > maxKey) { key2 = INT64_MAX; } else { - key2 = dataRowKey(row); - isRowDel = dataRowDeleted(row); + key2 = memRowKey(row); + isRowDel = memRowDeleted(row); } if (key1 == INT64_MAX && key2 == INT64_MAX) break; @@ -1284,24 +1285,24 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt (*iter)++; } else if (key1 > key2) { if (!isRowDel) { - if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) { - pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, dataRowVersion(row)); + if (pSchema == NULL || schemaVersion(pSchema) != memRowVersion(row)) { + pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, memRowVersion(row)); ASSERT(pSchema != NULL); } - tdAppendDataRowToDataCol(row, pSchema, pTarget); + tdAppendMemRowToDataCol(row, pSchema, pTarget); } tSkipListIterNext(pCommitIter->pIter); } else { if (update) { if (!isRowDel) { - if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) { - pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, dataRowVersion(row)); + if (pSchema == NULL || schemaVersion(pSchema) != memRowVersion(row)) { + pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, memRowVersion(row)); ASSERT(pSchema != NULL); } - tdAppendDataRowToDataCol(row, pSchema, pTarget); + tdAppendMemRowToDataCol(row, pSchema, pTarget); } } else { ASSERT(!isRowDel); diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 0cbabb8909..c0474f3b07 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -639,7 +639,7 @@ static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pRea int numColumns; int32_t blockIdx; SDataStatis* pBlockStatis = NULL; - SDataRow row = NULL; + SMemRow row = NULL; // restore last column data with last schema int err = 0; @@ -655,13 +655,13 @@ static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pRea } } - row = taosTMalloc(dataRowMaxBytesFromSchema(pSchema)); + row = taosTMalloc(memRowMaxBytesFromSchema(pSchema)); if (row == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; err = -1; goto out; } - tdInitDataRow(row, pSchema); + tdInitDataRow(POINTER_SHIFT(row, TD_MEM_ROW_TYPE_SIZE), pSchema); // first load block index info if (tsdbLoadBlockInfo(pReadh, NULL) < 0) { @@ -718,9 +718,10 @@ static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pRea // OK,let's load row from backward to get not-null column for (int32_t rowId = pBlock->numOfRows - 1; rowId >= 0; rowId--) { SDataCol *pDataCol = pReadh->pDCols[0]->cols + i; - tdAppendColVal(row, tdGetColDataOfRow(pDataCol, rowId), pCol->type, pCol->bytes, pCol->offset); + tdAppendColVal(POINTER_SHIFT(row, TD_MEM_ROW_TYPE_SIZE), tdGetColDataOfRow(pDataCol, rowId), pCol->type, + pCol->bytes, pCol->offset); //SDataCol *pDataCol = readh.pDCols[0]->cols + j; - void* value = tdGetRowDataOfCol(row, (int8_t)pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset); + void *value = tdGetMemRowDataOfCol(row, (int8_t)pCol->type, TD_MEM_ROW_HEAD_SIZE + pCol->offset); if (isNull(value, pCol->type)) { continue; } @@ -740,8 +741,9 @@ static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pRea // save row ts(in column 0) pDataCol = pReadh->pDCols[0]->cols + 0; pCol = schemaColAt(pSchema, 0); - tdAppendColVal(row, tdGetColDataOfRow(pDataCol, rowId), pCol->type, pCol->bytes, pCol->offset); - pLastCol->ts = dataRowKey(row); + tdAppendColVal(POINTER_SHIFT(row, TD_MEM_ROW_TYPE_SIZE), tdGetColDataOfRow(pDataCol, rowId), pCol->type, + pCol->bytes, pCol->offset); + pLastCol->ts = memRowKey(row); pTable->restoreColumnNum += 1; diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 9d8b1ca7f2..94bac68296 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -21,7 +21,7 @@ typedef struct { int32_t totalLen; int32_t len; - SDataRow row; + SMemRow row; } SSubmitBlkIter; typedef struct { @@ -36,20 +36,19 @@ static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable); static void tsdbFreeTableData(STableData *pTableData); static char * tsdbGetTsTupleKey(const void *data); static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables); -static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, SDataRow row); +static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, SMemRow row); static int tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter); -static SDataRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter); +static SMemRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter); static int tsdbScanAndConvertSubmitMsg(STsdbRepo *pRepo, SSubmitMsg *pMsg); static int tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, int32_t *affectedrows); -static int tsdbCopyRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable, void **ppRow); +static int tsdbCopyRowToMem(STsdbRepo *pRepo, SMemRow row, STable *pTable, void **ppRow); static int tsdbInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter); static int tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock); static int tsdbCheckTableSchema(STsdbRepo *pRepo, SSubmitBlk *pBlock, STable *pTable); static int tsdbInsertDataToTableImpl(STsdbRepo *pRepo, STable *pTable, void **rows, int rowCounter); static void tsdbFreeRows(STsdbRepo *pRepo, void **rows, int rowCounter); -static int tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, SDataRow row); - -static FORCE_INLINE int tsdbCheckRowRange(STsdbRepo *pRepo, STable *pTable, SDataRow row, TSKEY minKey, TSKEY maxKey, +static int tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, SMemRow row); +static FORCE_INLINE int tsdbCheckRowRange(STsdbRepo *pRepo, STable *pTable, SMemRow row, TSKEY minKey, TSKEY maxKey, TSKEY now); int32_t tsdbInsertData(STsdbRepo *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg *pRsp) { @@ -354,7 +353,7 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey TSKEY fKey = 0; bool isRowDel = false; int filterIter = 0; - SDataRow row = NULL; + SMemRow row = NULL; SMergeInfo mInfo; if (pMergeInfo == NULL) pMergeInfo = &mInfo; @@ -365,12 +364,12 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey if (pCols) tdResetDataCols(pCols); row = tsdbNextIterRow(pIter); - if (row == NULL || dataRowKey(row) > maxKey) { + if (row == NULL || memRowKey(row) > maxKey) { rowKey = INT64_MAX; isRowDel = false; } else { - rowKey = dataRowKey(row); - isRowDel = dataRowDeleted(row); + rowKey = memRowKey(row); + isRowDel = memRowDeleted(row); } if (filterIter >= nFilterKeys) { @@ -407,12 +406,12 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey tSkipListIterNext(pIter); row = tsdbNextIterRow(pIter); - if (row == NULL || dataRowKey(row) > maxKey) { + if (row == NULL || memRowKey(row) > maxKey) { rowKey = INT64_MAX; isRowDel = false; } else { - rowKey = dataRowKey(row); - isRowDel = dataRowDeleted(row); + rowKey = memRowKey(row); + isRowDel = memRowDeleted(row); } } else { if (isRowDel) { @@ -437,12 +436,12 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey tSkipListIterNext(pIter); row = tsdbNextIterRow(pIter); - if (row == NULL || dataRowKey(row) > maxKey) { + if (row == NULL || memRowKey(row) > maxKey) { rowKey = INT64_MAX; isRowDel = false; } else { - rowKey = dataRowKey(row); - isRowDel = dataRowDeleted(row); + rowKey = memRowKey(row); + isRowDel = memRowDeleted(row); } filterIter++; @@ -548,7 +547,7 @@ static void tsdbFreeTableData(STableData *pTableData) { } } -static char *tsdbGetTsTupleKey(const void *data) { return dataRowTuple((SDataRow)data); } +static char *tsdbGetTsTupleKey(const void *data) { return memRowTuple((SMemRow)data); } static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables) { ASSERT(pMemTable->maxTables < maxTables); @@ -572,17 +571,17 @@ static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables) { return 0; } -static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, SDataRow row) { +static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, SMemRow row) { if (pCols) { - if (*ppSchema == NULL || schemaVersion(*ppSchema) != dataRowVersion(row)) { - *ppSchema = tsdbGetTableSchemaImpl(pTable, false, false, dataRowVersion(row)); + if (*ppSchema == NULL || schemaVersion(*ppSchema) != memRowVersion(row)) { + *ppSchema = tsdbGetTableSchemaImpl(pTable, false, false, memRowVersion(row)); if (*ppSchema == NULL) { ASSERT(false); return -1; } } - tdAppendDataRowToDataCol(row, *ppSchema, pCols); + tdAppendMemRowToDataCol(row, *ppSchema, pCols); } return 0; @@ -592,31 +591,32 @@ static int tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter) { if (pBlock->dataLen <= 0) return -1; pIter->totalLen = pBlock->dataLen; pIter->len = 0; - pIter->row = (SDataRow)(pBlock->data+pBlock->schemaLen); + pIter->row = (SMemRow)(pBlock->data + pBlock->schemaLen); return 0; } -static SDataRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter) { - SDataRow row = pIter->row; +static SMemRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter) { + SMemRow row = pIter->row; // firstly, get current row if (row == NULL) return NULL; - pIter->len += dataRowLen(row); - if (pIter->len >= pIter->totalLen) { + pIter->len += memRowTLen(row); + if (pIter->len >= pIter->totalLen) { // reach the end pIter->row = NULL; } else { - pIter->row = (char *)row + dataRowLen(row); + pIter->row = (char *)row + memRowTLen(row); // secondly, move to next row } return row; } -static FORCE_INLINE int tsdbCheckRowRange(STsdbRepo *pRepo, STable *pTable, SDataRow row, TSKEY minKey, TSKEY maxKey, +static FORCE_INLINE int tsdbCheckRowRange(STsdbRepo *pRepo, STable *pTable, SMemRow row, TSKEY minKey, TSKEY maxKey, TSKEY now) { - if (dataRowKey(row) < minKey || dataRowKey(row) > maxKey) { + TSKEY rowKey = memRowKey(row); + if (rowKey < minKey || rowKey > maxKey) { tsdbError("vgId:%d table %s tid %d uid %" PRIu64 " timestamp is out of range! now %" PRId64 " minKey %" PRId64 " maxKey %" PRId64 " row key %" PRId64, REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), now, minKey, maxKey, - dataRowKey(row)); + rowKey); terrno = TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE; return -1; } @@ -630,7 +630,7 @@ static int tsdbScanAndConvertSubmitMsg(STsdbRepo *pRepo, SSubmitMsg *pMsg) { SSubmitMsgIter msgIter = {0}; SSubmitBlk * pBlock = NULL; SSubmitBlkIter blkIter = {0}; - SDataRow row = NULL; + SMemRow row = NULL; TSKEY now = taosGetTimestamp(pRepo->config.precision); TSKEY minKey = now - tsMsPerDay[pRepo->config.precision] * pRepo->config.keep; TSKEY maxKey = now + tsMsPerDay[pRepo->config.precision] * pRepo->config.daysPerFile; @@ -698,7 +698,7 @@ static int tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, int32_t * int64_t points = 0; STable * pTable = NULL; SSubmitBlkIter blkIter = {0}; - SDataRow row = NULL; + SMemRow row = NULL; void * rows[TSDB_MAX_INSERT_BATCH] = {0}; int rowCounter = 0; @@ -744,10 +744,10 @@ _err: return -1; } -static int tsdbCopyRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable, void **ppRow) { +static int tsdbCopyRowToMem(STsdbRepo *pRepo, SMemRow row, STable *pTable, void **ppRow) { STsdbCfg * pCfg = &pRepo->config; - TKEY tkey = dataRowTKey(row); - TSKEY key = dataRowKey(row); + TKEY tkey = memRowTKey(row); + TSKEY key = memRowKey(row); bool isRowDelete = TKEY_IS_DELETED(tkey); if (isRowDelete) { @@ -765,15 +765,15 @@ static int tsdbCopyRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable, void } } - void *pRow = tsdbAllocBytes(pRepo, dataRowLen(row)); + void *pRow = tsdbAllocBytes(pRepo, memRowTLen(row)); if (pRow == NULL) { tsdbError("vgId:%d failed to insert row with key %" PRId64 " to table %s while allocate %d bytes since %s", - REPO_ID(pRepo), key, TABLE_CHAR_NAME(pTable), dataRowLen(row), tstrerror(terrno)); + REPO_ID(pRepo), key, TABLE_CHAR_NAME(pTable), memRowTLen(row), tstrerror(terrno)); return -1; } - dataRowCpy(pRow, row); - ppRow[0] = pRow; + memRowCpy(pRow, row); + ppRow[0] = pRow; // save the memory address of data rows tsdbTrace("vgId:%d a row is %s table %s tid %d uid %" PRIu64 " key %" PRIu64, REPO_ID(pRepo), isRowDelete ? "deleted from" : "updated in", TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), @@ -954,8 +954,8 @@ static void tsdbFreeRows(STsdbRepo *pRepo, void **rows, int rowCounter) { STsdbBufPool *pBufPool = pRepo->pPool; for (int i = rowCounter - 1; i >= 0; --i) { - SDataRow row = (SDataRow)rows[i]; - int bytes = (int)dataRowLen(row); + SMemRow row = (SMemRow)rows[i]; + int bytes = (int)memRowTLen(row); if (pRepo->mem->extraBuffList == NULL) { STsdbBufBlock *pBufBlock = tsdbGetCurrBufBlock(pRepo); @@ -988,15 +988,16 @@ static void tsdbFreeRows(STsdbRepo *pRepo, void **rows, int rowCounter) { } } -static void updateTableLatestColumn(STsdbRepo *pRepo, STable *pTable, SDataRow row) { - tsdbDebug("vgId:%d updateTableLatestColumn, %s row version:%d", REPO_ID(pRepo), pTable->name->data, dataRowVersion(row)); +static void updateTableLatestColumn(STsdbRepo *pRepo, STable *pTable, SMemRow row) { + tsdbDebug("vgId:%d updateTableLatestColumn, %s row version:%d", REPO_ID(pRepo), pTable->name->data, + memRowVersion(row)); STSchema* pSchema = tsdbGetTableLatestSchema(pTable); if (tsdbUpdateLastColSchema(pTable, pSchema) < 0) { return; } - pSchema = tsdbGetTableSchemaByVersion(pTable, dataRowVersion(row)); + pSchema = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row)); if (pSchema == NULL) { return; } @@ -1010,8 +1011,8 @@ static void updateTableLatestColumn(STsdbRepo *pRepo, STable *pTable, SDataRow r if (idx == -1) { continue; } - - void* value = tdGetRowDataOfCol(row, (int8_t)pTCol->type, TD_DATA_ROW_HEAD_SIZE + pSchema->columns[j].offset); + + void *value = tdGetMemRowDataOfCol(row, (int8_t)pTCol->type, TD_MEM_ROW_HEAD_SIZE + pSchema->columns[j].offset); if (isNull(value, pTCol->type)) { continue; } @@ -1027,11 +1028,11 @@ static void updateTableLatestColumn(STsdbRepo *pRepo, STable *pTable, SDataRow r memcpy(pDataCol->pData, value, pDataCol->bytes); //tsdbInfo("updateTableLatestColumn vgId:%d cache column %d for %d,%s", REPO_ID(pRepo), j, pDataCol->bytes, (char*)pDataCol->pData); - pDataCol->ts = dataRowKey(row); + pDataCol->ts = memRowKey(row); } } -static int tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, SDataRow row) { +static int tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, SMemRow row) { STsdbCfg *pCfg = &pRepo->config; // if cacheLastRow config has been reset, free the lastRow @@ -1042,31 +1043,31 @@ static int tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, SDataRow TSDB_WUNLOCK_TABLE(pTable); } - if (tsdbGetTableLastKeyImpl(pTable) < dataRowKey(row)) { + if (tsdbGetTableLastKeyImpl(pTable) < memRowKey(row)) { if (CACHE_LAST_ROW(pCfg) || pTable->lastRow != NULL) { - SDataRow nrow = pTable->lastRow; - if (taosTSizeof(nrow) < dataRowLen(row)) { - SDataRow orow = nrow; - nrow = taosTMalloc(dataRowLen(row)); + SMemRow nrow = pTable->lastRow; + if (taosTSizeof(nrow) < memRowTLen(row)) { + SMemRow orow = nrow; + nrow = taosTMalloc(memRowTLen(row)); if (nrow == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return -1; } - dataRowCpy(nrow, row); + memRowCpy(nrow, row); TSDB_WLOCK_TABLE(pTable); - pTable->lastKey = dataRowKey(row); + pTable->lastKey = memRowKey(row); pTable->lastRow = nrow; TSDB_WUNLOCK_TABLE(pTable); taosTZfree(orow); } else { TSDB_WLOCK_TABLE(pTable); - pTable->lastKey = dataRowKey(row); - dataRowCpy(nrow, row); + pTable->lastKey = memRowKey(row); + memRowCpy(nrow, row); TSDB_WUNLOCK_TABLE(pTable); } } else { - pTable->lastKey = dataRowKey(row); + pTable->lastKey = memRowKey(row); } if (CACHE_LAST_NULL_COLUMN(pCfg)) { diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 972c3c4e10..32232e6815 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -139,7 +139,7 @@ typedef struct STableGroupSupporter { static STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList); static int32_t checkForCachedLastRow(STsdbQueryHandle* pQueryHandle, STableGroupInfo *groupList); static int32_t checkForCachedLast(STsdbQueryHandle* pQueryHandle); -static int32_t tsdbGetCachedLastRow(STable* pTable, SDataRow* pRes, TSKEY* lastKey); +static int32_t tsdbGetCachedLastRow(STable* pTable, SMemRow* pRes, TSKEY* lastKey); static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle); static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SBlock* pBlock); @@ -669,8 +669,8 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter); assert(node != NULL); - SDataRow row = (SDataRow)SL_GET_NODE_DATA(node); - TSKEY key = dataRowKey(row); // first timestamp in buffer + SMemRow row = (SMemRow)SL_GET_NODE_DATA(node); + TSKEY key = memRowKey(row); // first timestamp in buffer tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 "-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", 0x%"PRIx64, pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pMem->keyFirst, pMem->keyLast, @@ -691,8 +691,8 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter); assert(node != NULL); - SDataRow row = (SDataRow)SL_GET_NODE_DATA(node); - TSKEY key = dataRowKey(row); // first timestamp in buffer + SMemRow row = (SMemRow)SL_GET_NODE_DATA(node); + TSKEY key = memRowKey(row); // first timestamp in buffer tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 "-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", 0x%"PRIx64, pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pIMem->keyFirst, pIMem->keyLast, @@ -716,19 +716,19 @@ static void destroyTableMemIterator(STableCheckInfo* pCheckInfo) { tSkipListDestroyIter(pCheckInfo->iiter); } -static SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int32_t update) { - SDataRow rmem = NULL, rimem = NULL; +static SMemRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int32_t update) { + SMemRow rmem = NULL, rimem = NULL; if (pCheckInfo->iter) { SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter); if (node != NULL) { - rmem = (SDataRow)SL_GET_NODE_DATA(node); + rmem = (SMemRow)SL_GET_NODE_DATA(node); } } if (pCheckInfo->iiter) { SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter); if (node != NULL) { - rimem = (SDataRow)SL_GET_NODE_DATA(node); + rimem = (SMemRow)SL_GET_NODE_DATA(node); } } @@ -746,8 +746,8 @@ static SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order return rimem; } - TSKEY r1 = dataRowKey(rmem); - TSKEY r2 = dataRowKey(rimem); + TSKEY r1 = memRowKey(rmem); + TSKEY r2 = memRowKey(rimem); if (r1 == r2) { // data ts are duplicated, ignore the data in mem if (!update) { @@ -826,12 +826,12 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { initTableMemIterator(pHandle, pCheckInfo); } - SDataRow row = getSDataRowInTableMem(pCheckInfo, pHandle->order, pCfg->update); + SMemRow row = getSDataRowInTableMem(pCheckInfo, pHandle->order, pCfg->update); if (row == NULL) { return false; } - pCheckInfo->lastKey = dataRowKey(row); // first timestamp in buffer + pCheckInfo->lastKey = memRowKey(row); // first timestamp in buffer tsdbDebug("%p uid:%" PRId64", tid:%d check data in buffer from skey:%" PRId64 ", order:%d, 0x%"PRIx64, pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, pCheckInfo->lastKey, pHandle->order, pHandle->qId); @@ -1082,11 +1082,11 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SBlock* p int32_t code = TSDB_CODE_SUCCESS; /*bool hasData = */ initTableMemIterator(pQueryHandle, pCheckInfo); - SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update); + SMemRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update); assert(cur->pos >= 0 && cur->pos <= binfo.rows); - TSKEY key = (row != NULL)? dataRowKey(row):TSKEY_INITIAL_VAL; + TSKEY key = (row != NULL) ? memRowKey(row) : TSKEY_INITIAL_VAL; if (key != TSKEY_INITIAL_VAL) { tsdbDebug("%p key in mem:%"PRId64", 0x%"PRIx64, pQueryHandle, key, pQueryHandle->qId); } else { @@ -1327,7 +1327,7 @@ int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity // todo refactor, only copy one-by-one for (int32_t k = start; k < num + start; ++k) { - char* p = tdGetColDataOfRow(src, k); + const char* p = tdGetColDataOfRow(src, k); memcpy(dst, p, varDataTLen(p)); dst += bytes; } @@ -1378,14 +1378,14 @@ int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity return numOfRows + num; } -static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, SDataRow row, +static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, SMemRow row, int32_t numOfCols, STable* pTable, STSchema* pSchema) { char* pData = NULL; // the schema version info is embeded in SDataRow int32_t numOfRowCols = 0; if (pSchema == NULL) { - pSchema = tsdbGetTableSchemaByVersion(pTable, dataRowVersion(row)); + pSchema = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row)); numOfRowCols = schemaNCols(pSchema); } else { numOfRowCols = schemaNCols(pSchema); @@ -1406,7 +1406,8 @@ static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, } if (pSchema->columns[j].colId == pColInfo->info.colId) { - void* value = tdGetRowDataOfCol(row, (int8_t)pColInfo->info.type, TD_DATA_ROW_HEAD_SIZE + pSchema->columns[j].offset); + void* value = + tdGetMemRowDataOfCol(row, (int8_t)pColInfo->info.type, TD_MEM_ROW_HEAD_SIZE + pSchema->columns[j].offset); switch (pColInfo->info.type) { case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: @@ -1656,12 +1657,12 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* } else if (pCheckInfo->iter != NULL || pCheckInfo->iiter != NULL) { SSkipListNode* node = NULL; do { - SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update); + SMemRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update); if (row == NULL) { break; } - TSKEY key = dataRowKey(row); + TSKEY key = memRowKey(row); if ((key > pQueryHandle->window.ekey && ASCENDING_TRAVERSE(pQueryHandle->order)) || (key < pQueryHandle->window.ekey && !ASCENDING_TRAVERSE(pQueryHandle->order))) { break; @@ -1674,11 +1675,11 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* if ((key < tsArray[pos] && ASCENDING_TRAVERSE(pQueryHandle->order)) || (key > tsArray[pos] && !ASCENDING_TRAVERSE(pQueryHandle->order))) { - if (rv != dataRowVersion(row)) { - pSchema = tsdbGetTableSchemaByVersion(pTable, dataRowVersion(row)); - rv = dataRowVersion(row); - } - + if (rv != memRowVersion(row)) { + pSchema = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row)); + rv = memRowVersion(row); + } + copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row, numOfCols, pTable, pSchema); numOfRows += 1; if (cur->win.skey == TSKEY_INITIAL_VAL) { @@ -1692,11 +1693,11 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* moveToNextRowInMem(pCheckInfo); } else if (key == tsArray[pos]) { // data in buffer has the same timestamp of data in file block, ignore it if (pCfg->update) { - if (rv != dataRowVersion(row)) { - pSchema = tsdbGetTableSchemaByVersion(pTable, dataRowVersion(row)); - rv = dataRowVersion(row); + if (rv != memRowVersion(row)) { + pSchema = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row)); + rv = memRowVersion(row); } - + copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row, numOfCols, pTable, pSchema); numOfRows += 1; if (cur->win.skey == TSKEY_INITIAL_VAL) { @@ -1746,8 +1747,10 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* * copy them all to result buffer, since it may be overlapped with file data block. */ if (node == NULL || - ((dataRowKey((SDataRow)SL_GET_NODE_DATA(node)) > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) || - ((dataRowKey((SDataRow)SL_GET_NODE_DATA(node)) < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order))) { + ((memRowKey((SMemRow)SL_GET_NODE_DATA(node)) > pQueryHandle->window.ekey) && + ASCENDING_TRAVERSE(pQueryHandle->order)) || + ((memRowKey((SMemRow)SL_GET_NODE_DATA(node)) < pQueryHandle->window.ekey) && + !ASCENDING_TRAVERSE(pQueryHandle->order))) { // no data in cache or data in cache is greater than the ekey of time window, load data from file block if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = tsArray[pos]; @@ -2333,12 +2336,12 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int STSchema* pSchema = NULL; do { - SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update); + SMemRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update); if (row == NULL) { break; } - TSKEY key = dataRowKey(row); + TSKEY key = memRowKey(row); if ((key > maxKey && ASCENDING_TRAVERSE(pQueryHandle->order)) || (key < maxKey && !ASCENDING_TRAVERSE(pQueryHandle->order))) { tsdbDebug("%p key:%"PRIu64" beyond qrange:%"PRId64" - %"PRId64", no more data in buffer", pQueryHandle, key, pQueryHandle->window.skey, pQueryHandle->window.ekey); @@ -2351,9 +2354,9 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int } win->ekey = key; - if (rv != dataRowVersion(row)) { - pSchema = tsdbGetTableSchemaByVersion(pTable, dataRowVersion(row)); - rv = dataRowVersion(row); + if (rv != memRowVersion(row)) { + pSchema = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row)); + rv = memRowVersion(row); } copyOneRowFromMem(pQueryHandle, maxRowsToRead, numOfRows, row, numOfCols, pTable, pSchema); @@ -2470,7 +2473,7 @@ static bool loadCachedLastRow(STsdbQueryHandle* pQueryHandle) { SQueryFilePos* cur = &pQueryHandle->cur; - SDataRow pRow = NULL; + SMemRow pRow = NULL; TSKEY key = TSKEY_INITIAL_VAL; int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1; @@ -2873,7 +2876,7 @@ bool tsdbGetExternalRow(TsdbQueryHandleT pHandle) { * if lastRow == NULL, return TSDB_CODE_TDB_NO_CACHE_LAST_ROW * else set pRes and return TSDB_CODE_SUCCESS and save lastKey */ -int32_t tsdbGetCachedLastRow(STable* pTable, SDataRow* pRes, TSKEY* lastKey) { +int32_t tsdbGetCachedLastRow(STable* pTable, SMemRow* pRes, TSKEY* lastKey) { int32_t code = TSDB_CODE_SUCCESS; TSDB_RLOCK_TABLE(pTable); @@ -2884,7 +2887,7 @@ int32_t tsdbGetCachedLastRow(STable* pTable, SDataRow* pRes, TSKEY* lastKey) { } if (pRes) { - *pRes = tdDataRowDup(pTable->lastRow); + *pRes = tdMemRowDup(pTable->lastRow); if (*pRes == NULL) { code = TSDB_CODE_TDB_OUT_OF_MEMORY; } From f3a1f84a7442f367a14135dfc560cc6d2d909dd4 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 30 Jun 2021 10:39:42 +0800 Subject: [PATCH 05/67] support SKVRow for 4096 columns --- src/client/inc/tscUtil.h | 4 +- src/client/src/tscUtil.c | 65 ++++++------ src/common/inc/tdataformat.h | 39 ++++++-- src/tsdb/src/tsdbRead.c | 187 ++++++++++++++++++++++++++++++++++- 4 files changed, 254 insertions(+), 41 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 9d9eacb332..ca00d03343 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -346,8 +346,8 @@ typedef struct { int16_t sversion; int32_t flen; // for SKVRow - int16_t tCols; - int16_t nCols; + uint16_t tCols; + uint16_t nCols; SColIdx* pColIdx; uint16_t alloc; uint16_t size; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index c7241860bc..e6444fb4a4 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1664,7 +1664,7 @@ void tdResetMemRowBuilder(SMemRowBuilder* pBuilder) { } #define KvRowNullColRatio 0.75 // If nullable column ratio larger than 0.75, utilize SKVRow, otherwise SDataRow. -#define KvRowNColsThresh 4096 // default value: 32 +#define KvRowNColsThresh 1 // default value: 32 static FORCE_INLINE uint8_t tdRowTypeJudger(SSchema* pSchema, void* pData, int32_t nCols, int32_t flen, uint16_t* nColsNotNull) { @@ -1672,22 +1672,22 @@ static FORCE_INLINE uint8_t tdRowTypeJudger(SSchema* pSchema, void* pData, int32 if (nCols < KvRowNColsThresh) { return SMEM_ROW_DATA; } - int32_t dataRowLen = flen; - int32_t kvRowLen = 0; + int32_t dataRowLength = flen; + int32_t kvRowLength = 0; uint16_t nColsNull = 0; char* p = (char*)pData; for (int i = 0; i < nCols; ++i) { if (IS_VAR_DATA_TYPE(pSchema[i].type)) { - dataRowLen += varDataTLen(p); + dataRowLength += varDataTLen(p); if (!isNull(p, pSchema[i].type)) { - kvRowLen += sizeof(SColIdx) + varDataTLen(p); + kvRowLength += (sizeof(SColIdx) + varDataTLen(p)); } else { ++nColsNull; } } else { if (!isNull(p, pSchema[i].type)) { - kvRowLen += sizeof(SColIdx) + varDataTLen(p); + kvRowLength += (sizeof(SColIdx) + TYPE_BYTES[pSchema[i].type]); } else { ++nColsNull; } @@ -1697,9 +1697,9 @@ static FORCE_INLINE uint8_t tdRowTypeJudger(SSchema* pSchema, void* pData, int32 p += pSchema[i].bytes; } - tscDebug("prop:nColsNull %d, nCols: %d, kvRowLen: %d, dataRowLen: %d", nColsNull, nCols, kvRowLen, dataRowLen); + tscInfo("prop:nColsNull %d, nCols: %d, kvRowLen: %d, dataRowLen: %d", nColsNull, nCols, kvRowLength, dataRowLength); - if (kvRowLen < dataRowLen) { + if (kvRowLength < dataRowLength) { if (nColsNotNull) { *nColsNotNull = nCols - nColsNull; } @@ -1713,20 +1713,24 @@ SMemRow tdGetMemRowFromBuilder(SMemRowBuilder* pBuilder) { SSchema* pSchema = pBuilder->pSchema; char* p = (char*)pBuilder->buf; + if(pBuilder->nCols <= 0){ + return NULL; + } + uint16_t nColsNotNull = 0; uint8_t memRowType = tdRowTypeJudger(pSchema, p, pBuilder->nCols, pBuilder->flen, &nColsNotNull); tscDebug("prop:memType is %d", memRowType); - memRowType = SMEM_ROW_DATA; + SMemRow* memRow = (SMemRow)pBuilder->pDataBlock; memRowSetType(memRow, memRowType); if (memRowType == SMEM_ROW_DATA) { - int toffset = 0; SDataRow trow = (SDataRow)memRowBody(memRow); dataRowSetLen(trow, (uint16_t)(TD_DATA_ROW_HEAD_SIZE + pBuilder->flen)); dataRowSetVersion(trow, pBuilder->sversion); + int toffset = 0; p = (char*)pBuilder->buf; for (int32_t j = 0; j < pBuilder->nCols; ++j) { tdAppendColVal(trow, p, pSchema[j].type, pSchema[j].bytes, toffset); @@ -1734,35 +1738,34 @@ SMemRow tdGetMemRowFromBuilder(SMemRowBuilder* pBuilder) { p += pSchema[j].bytes; } pBuilder->buf = p; - } else { + } else if (memRowType == SMEM_ROW_KV) { + ASSERT(nColsNotNull < pBuilder->nCols); + uint16_t tlen = TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * nColsNotNull + pBuilder->size; - SKVRow row = (SKVRow)pBuilder->pDataBlock; + SKVRow kvRow = (SKVRow)memRowBody(memRow); - kvRowSetNCols(row, nColsNotNull); - kvRowSetLen(row, tlen); + kvRowSetNCols(kvRow, nColsNotNull); + kvRowSetLen(kvRow, tlen); - memcpy(kvRowColIdx(row), pBuilder->pColIdx, sizeof(SColIdx) * pBuilder->nCols); - memcpy(kvRowValues(row), pBuilder->buf, pBuilder->size); + int toffset = 0; + p = (char*)pBuilder->buf; + for (int32_t j = 0; j < pBuilder->nCols; ++j) { + if(!isNull(p, pSchema[j].type)) { + tdAppendKvColVal(kvRow, p, pSchema[j].colId, pSchema[j].type, toffset); + toffset += sizeof(SColIdx); + } + p += pSchema[j].bytes; + } + pBuilder->buf = p; + + } else { + ASSERT(0); } pBuilder->pDataBlock = (char*)pBuilder->pDataBlock + memRowTLen(memRow); // next row pBuilder->pSubmitBlk->dataLen += memRowTLen(memRow); - // int tlen = sizeof(SColIdx) * pBuilder->nCols + pBuilder->size; - // if (tlen == 0) return NULL; - - // tlen += TD_KV_ROW_HEAD_SIZE; - - // SKVRow row = malloc(tlen); - // if (row == NULL) return NULL; - - // kvRowSetNCols(row, pBuilder->nCols); - // kvRowSetLen(row, tlen); - - // memcpy(kvRowColIdx(row), pBuilder->pColIdx, sizeof(SColIdx) * pBuilder->nCols); - // memcpy(kvRowValues(row), pBuilder->buf, pBuilder->size); - - return NULL; + return memRow; } // Erase the empty space reserved for binary data diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index 294d8b2cf8..bdb1d78cdb 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -404,10 +404,10 @@ typedef struct { uint16_t offset; } SColIdx; -#define TD_KV_ROW_HEAD_SIZE (2 * sizeof(int16_t)) +#define TD_KV_ROW_HEAD_SIZE (sizeof(uint16_t) + sizeof(int16_t)) #define kvRowLen(r) (*(uint16_t *)(r)) -#define kvRowNCols(r) (*(int16_t *)POINTER_SHIFT(r, sizeof(int16_t))) +#define kvRowNCols(r) (*(int16_t *)POINTER_SHIFT(r, sizeof(uint16_t))) #define kvRowSetLen(r, len) kvRowLen(r) = (len) #define kvRowSetNCols(r, n) kvRowNCols(r) = (n) #define kvRowColIdx(r) (SColIdx *)POINTER_SHIFT(r, TD_KV_ROW_HEAD_SIZE) @@ -445,6 +445,33 @@ static FORCE_INLINE void *tdGetKVRowValOfCol(SKVRow row, int16_t colId) { return kvRowColVal(row, (SColIdx *)ret); } +// offset here not include kvRow header length +static FORCE_INLINE int tdAppendKvColVal(SKVRow row, const void *value, int16_t colId, int8_t type, int32_t offset) { + ASSERT(value != NULL); + int32_t toffset = offset + TD_KV_ROW_HEAD_SIZE; + SColIdx *pColIdx = (SColIdx *)POINTER_SHIFT(row, toffset); + char * ptr = (char *)POINTER_SHIFT(row, kvRowLen(row)); + + pColIdx->colId = colId; + pColIdx->offset = kvRowLen(row); + + if (IS_VAR_DATA_TYPE(type)) { + memcpy(ptr, value, varDataTLen(value)); + kvRowLen(row) += varDataTLen(value); + } else { + if (offset == 0) { + ASSERT(type == TSDB_DATA_TYPE_TIMESTAMP); + TKEY tvalue = tdGetTKEY(*(TSKEY *)value); + memcpy(ptr, (void *)(&tvalue), TYPE_BYTES[type]); + } else { + memcpy(ptr, value, TYPE_BYTES[type]); + } + kvRowLen(row) += TYPE_BYTES[type]; + } + + return 0; +} + // ----------------- K-V data row builder typedef struct { int16_t tCols; @@ -540,10 +567,10 @@ static FORCE_INLINE int tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t colId, #define memRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_MEM_ROW_HEAD_SIZE) #define memRowDeleted(r) TKEY_IS_DELETED(memRowTKey(r)) -#define memRowNCols(r) (*(int16_t *)POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE + sizeof(int16_t))) // for SKVRow -#define memRowSetNCols(r, n) memRowNCols(r) = (n) // for SKVRow -#define memRowColIdx(r) (SColIdx *)POINTER_SHIFT(r, TD_MEM_ROW_HEAD_SIZE) // for SKVRow -#define memRowValues(r) POINTER_SHIFT(r, TD_MEM_ROW_HEAD_SIZE + sizeof(SColIdx) * memRowNCols(r)) // for SKVRow +// #define memRowNCols(r) (*(int16_t *)POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE + sizeof(int16_t))) // for SKVRow +// #define memRowSetNCols(r, n) memRowNCols(r) = (n) // for SKVRow +// #define memRowColIdx(r) (SColIdx *)POINTER_SHIFT(r, TD_MEM_ROW_HEAD_SIZE) // for SKVRow +// #define memRowValues(r) POINTER_SHIFT(r, TD_MEM_ROW_HEAD_SIZE + sizeof(SColIdx) * memRowNCols(r)) // for SKVRow // NOTE: offset here including the header size static FORCE_INLINE void *tdGetRowDataOfCol(void *row, int8_t type, int32_t offset) { diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 32232e6815..fffe5b28ef 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -1377,12 +1377,12 @@ int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity return numOfRows + num; } - +#if 0 static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, SMemRow row, int32_t numOfCols, STable* pTable, STSchema* pSchema) { char* pData = NULL; - // the schema version info is embeded in SDataRow + // the schema version info is embedded in SDataRow, and use latest schema version for SKVRow int32_t numOfRowCols = 0; if (pSchema == NULL) { pSchema = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row)); @@ -1477,7 +1477,190 @@ static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, i++; } } +#endif +static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, SMemRow row, + int32_t numOfCols, STable* pTable, STSchema* pSchema) { + char* pData = NULL; + + // the schema version info is embedded in SDataRow, and use latest schema version for SKVRow + int32_t numOfRowCols = 0; + if (pSchema == NULL) { + pSchema = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row)); + numOfRowCols = schemaNCols(pSchema); + } else { + numOfRowCols = schemaNCols(pSchema); + } + + int32_t i = 0; + + if (isDataRow(row)) { + int32_t j = 0; + while (i < numOfCols && j < numOfRowCols) { + SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); + if (pSchema->columns[j].colId < pColInfo->info.colId) { + j++; + continue; + } + + if (ASCENDING_TRAVERSE(pQueryHandle->order)) { + pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; + } else { + pData = (char*)pColInfo->pData + (capacity - numOfRows - 1) * pColInfo->info.bytes; + } + + if (pSchema->columns[j].colId == pColInfo->info.colId) { + void* value = + tdGetRowDataOfCol(row, (int8_t)pColInfo->info.type, TD_MEM_ROW_HEAD_SIZE + pSchema->columns[j].offset); + switch (pColInfo->info.type) { + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + memcpy(pData, value, varDataTLen(value)); + break; + case TSDB_DATA_TYPE_NULL: + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: + *(uint8_t*)pData = *(uint8_t*)value; + break; + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: + *(uint16_t*)pData = *(uint16_t*)value; + break; + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: + *(uint32_t*)pData = *(uint32_t*)value; + break; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: + *(uint64_t*)pData = *(uint64_t*)value; + break; + case TSDB_DATA_TYPE_FLOAT: + SET_FLOAT_PTR(pData, value); + break; + case TSDB_DATA_TYPE_DOUBLE: + SET_DOUBLE_PTR(pData, value); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + *(TSKEY*)pData = tdGetKey(*(TKEY*)value); + } else { + *(TSKEY*)pData = *(TSKEY*)value; + } + break; + default: + memcpy(pData, value, pColInfo->info.bytes); + } + + j++; + i++; + } else { // pColInfo->info.colId < pSchema->columns[j].colId, it is a NULL data + if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { + setVardataNull(pData, pColInfo->info.type); + } else { + setNull(pData, pColInfo->info.type, pColInfo->info.bytes); + } + i++; + } + } + } else if (isKvRow(row)) { + SKVRow kvRow = memRowBody(row); + int32_t k = 0; + int32_t nKvRowCols = kvRowNCols(kvRow); + + while (i < numOfCols && k < nKvRowCols) { + SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); + SColIdx* pColIdx = kvRowColIdxAt(kvRow, k); + + if (pColIdx->colId < pColInfo->info.colId) { + ++k; + continue; + } + + if (ASCENDING_TRAVERSE(pQueryHandle->order)) { + pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; + } else { + pData = (char*)pColInfo->pData + (capacity - numOfRows - 1) * pColInfo->info.bytes; + } + + if (pColIdx->colId == pColInfo->info.colId) { + STColumn* pSTColumn = tdGetColOfID(pSchema, pColIdx->colId); + if (pSTColumn != NULL) { + void* value = tdGetKvRowDataOfCol(row, pSTColumn->type, TD_MEM_ROW_HEAD_SIZE + pColIdx->offset); + switch (pColInfo->info.type) { + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + memcpy(pData, value, varDataTLen(value)); + break; + case TSDB_DATA_TYPE_NULL: + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: + *(uint8_t*)pData = *(uint8_t*)value; + break; + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: + *(uint16_t*)pData = *(uint16_t*)value; + break; + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: + *(uint32_t*)pData = *(uint32_t*)value; + break; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: + *(uint64_t*)pData = *(uint64_t*)value; + break; + case TSDB_DATA_TYPE_FLOAT: + SET_FLOAT_PTR(pData, value); + break; + case TSDB_DATA_TYPE_DOUBLE: + SET_DOUBLE_PTR(pData, value); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + *(TSKEY*)pData = tdGetKey(*(TKEY*)value); + } else { + *(TSKEY*)pData = *(TSKEY*)value; + } + break; + default: + memcpy(pData, value, pColInfo->info.bytes); + } + ++k; + ++i; + continue; + } + ++k; // pSTColumn is NULL + } + // If (pColInfo->info.colId < pColIdx->colId) or pSTColumn is NULL, it is a NULL data + if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { + setVardataNull(pData, pColInfo->info.type); + } else { + setNull(pData, pColInfo->info.type, pColInfo->info.bytes); + } + ++i; + } + } else { + ASSERT(0); + } + + while (i < numOfCols) { // the remain columns are all null data + SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); + if (ASCENDING_TRAVERSE(pQueryHandle->order)) { + pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; + } else { + pData = (char*)pColInfo->pData + (capacity - numOfRows - 1) * pColInfo->info.bytes; + } + + if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { + setVardataNull(pData, pColInfo->info.type); + } else { + setNull(pData, pColInfo->info.type, pColInfo->info.bytes); + } + + i++; + } +} static void moveDataToFront(STsdbQueryHandle* pQueryHandle, int32_t numOfRows, int32_t numOfCols) { if (numOfRows == 0 || ASCENDING_TRAVERSE(pQueryHandle->order)) { return; From 8f58cb6cb26cb44713200c006df172e03ff0244e Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 30 Jun 2021 13:41:35 +0800 Subject: [PATCH 06/67] code optimization --- src/client/inc/tscUtil.h | 52 +++++++++--------- src/client/src/tscUtil.c | 60 ++++++++++----------- src/common/inc/tdataformat.h | 66 ++++++++++------------- src/common/src/tdataformat.c | 35 ++++++------ src/inc/ttype.h | 2 +- src/tsdb/src/tsdbRead.c | 101 ----------------------------------- 6 files changed, 103 insertions(+), 213 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 6065feedea..3c5cb4f23a 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -359,39 +359,39 @@ typedef struct { SSubmitBlk* pSubmitBlk; } SMemRowBuilder; -int tdInitMemRowBuilder(SMemRowBuilder* pBuilder); -void tdDestroyMemRowBuilder(SMemRowBuilder* pBuilder); -void tdResetMemRowBuilder(SMemRowBuilder* pBuilder); -SMemRow tdGetMemRowFromBuilder(SMemRowBuilder* pBuilder); +// int tdInitMemRowBuilder(SMemRowBuilder* pBuilder); +// void tdDestroyMemRowBuilder(SMemRowBuilder* pBuilder); +// void tdResetMemRowBuilder(SMemRowBuilder* pBuilder); +SMemRow tdGenMemRowFromBuilder(SMemRowBuilder* pBuilder); -static FORCE_INLINE int tdAddColToMemRow(SMemRowBuilder* pBuilder, int16_t colId, int8_t type, void* value) { - // TODO +// static FORCE_INLINE int tdAddColToMemRow(SMemRowBuilder* pBuilder, int16_t colId, int8_t type, void* value) { +// // TODO - if (pBuilder->nCols >= pBuilder->tCols) { - pBuilder->tCols *= 2; - pBuilder->pColIdx = (SColIdx*)realloc((void*)(pBuilder->pColIdx), sizeof(SColIdx) * pBuilder->tCols); - if (pBuilder->pColIdx == NULL) return -1; - } +// if (pBuilder->nCols >= pBuilder->tCols) { +// pBuilder->tCols *= 2; +// pBuilder->pColIdx = (SColIdx*)realloc((void*)(pBuilder->pColIdx), sizeof(SColIdx) * pBuilder->tCols); +// if (pBuilder->pColIdx == NULL) return -1; +// } - pBuilder->pColIdx[pBuilder->nCols].colId = colId; - pBuilder->pColIdx[pBuilder->nCols].offset = pBuilder->size; +// pBuilder->pColIdx[pBuilder->nCols].colId = colId; +// pBuilder->pColIdx[pBuilder->nCols].offset = pBuilder->size; - pBuilder->nCols++; +// pBuilder->nCols++; - int tlen = IS_VAR_DATA_TYPE(type) ? varDataTLen(value) : TYPE_BYTES[type]; - if (tlen > pBuilder->alloc - pBuilder->size) { - while (tlen > pBuilder->alloc - pBuilder->size) { - pBuilder->alloc *= 2; - } - pBuilder->buf = realloc(pBuilder->buf, pBuilder->alloc); - if (pBuilder->buf == NULL) return -1; - } +// int tlen = IS_VAR_DATA_TYPE(type) ? varDataTLen(value) : TYPE_BYTES[type]; +// if (tlen > pBuilder->alloc - pBuilder->size) { +// while (tlen > pBuilder->alloc - pBuilder->size) { +// pBuilder->alloc *= 2; +// } +// pBuilder->buf = realloc(pBuilder->buf, pBuilder->alloc); +// if (pBuilder->buf == NULL) return -1; +// } - memcpy(POINTER_SHIFT(pBuilder->buf, pBuilder->size), value, tlen); - pBuilder->size += tlen; +// memcpy(POINTER_SHIFT(pBuilder->buf, pBuilder->size), value, tlen); +// pBuilder->size += tlen; - return 0; -} +// return 0; +// } #ifdef __cplusplus } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index bda309a159..2c53d90cea 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -14,7 +14,7 @@ */ #include "tscUtil.h" -#include "hash.h" +#include "hash.h" #include "os.h" #include "taosmsg.h" #include "texpr.h" @@ -1640,35 +1640,34 @@ int32_t tscGetDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, i return TSDB_CODE_SUCCESS; } -int tdInitMemRowBuilder(SMemRowBuilder* pBuilder) { - pBuilder->pSchema = NULL; - pBuilder->sversion = 0; - pBuilder->tCols = 128; - pBuilder->nCols = 0; - pBuilder->pColIdx = (SColIdx*)malloc(sizeof(SColIdx) * pBuilder->tCols); - if (pBuilder->pColIdx == NULL) return -1; - pBuilder->alloc = 1024; - pBuilder->size = 0; - pBuilder->buf = malloc(pBuilder->alloc); - if (pBuilder->buf == NULL) { - free(pBuilder->pColIdx); - return -1; - } - return 0; -} +// int tdInitMemRowBuilder(SMemRowBuilder* pBuilder) { +// pBuilder->pSchema = NULL; +// pBuilder->sversion = 0; +// pBuilder->tCols = 128; +// pBuilder->nCols = 0; +// pBuilder->pColIdx = (SColIdx*)malloc(sizeof(SColIdx) * pBuilder->tCols); +// if (pBuilder->pColIdx == NULL) return -1; +// pBuilder->alloc = 1024; +// pBuilder->size = 0; +// pBuilder->buf = malloc(pBuilder->alloc); +// if (pBuilder->buf == NULL) { +// free(pBuilder->pColIdx); +// return -1; +// } +// return 0; +// } -void tdDestroyMemRowBuilder(SMemRowBuilder* pBuilder) { - tfree(pBuilder->pColIdx); - tfree(pBuilder->buf); -} +// void tdDestroyMemRowBuilder(SMemRowBuilder* pBuilder) { +// tfree(pBuilder->pColIdx); +// tfree(pBuilder->buf); +// } -void tdResetMemRowBuilder(SMemRowBuilder* pBuilder) { - pBuilder->nCols = 0; - pBuilder->size = 0; -} +// void tdResetMemRowBuilder(SMemRowBuilder* pBuilder) { +// pBuilder->nCols = 0; +// pBuilder->size = 0; +// } -#define KvRowNullColRatio 0.75 // If nullable column ratio larger than 0.75, utilize SKVRow, otherwise SDataRow. -#define KvRowNColsThresh 1 // default value: 32 +#define KvRowNColsThresh 1 // default value: 32 TODO: for test, restore to 32 after test finished static FORCE_INLINE uint8_t tdRowTypeJudger(SSchema* pSchema, void* pData, int32_t nCols, int32_t flen, uint16_t* nColsNotNull) { @@ -1701,7 +1700,8 @@ static FORCE_INLINE uint8_t tdRowTypeJudger(SSchema* pSchema, void* pData, int32 p += pSchema[i].bytes; } - tscInfo("prop:nColsNull %d, nCols: %d, kvRowLen: %d, dataRowLen: %d", nColsNull, nCols, kvRowLength, dataRowLength); + tscDebug("prop:nColsNull %d, nCols: %d, kvRowLen: %d, dataRowLen: %d", (int32_t)nColsNull, nCols, kvRowLength, + dataRowLength); if (kvRowLength < dataRowLength) { if (nColsNotNull) { @@ -1713,7 +1713,7 @@ static FORCE_INLINE uint8_t tdRowTypeJudger(SSchema* pSchema, void* pData, int32 return SMEM_ROW_DATA; } -SMemRow tdGetMemRowFromBuilder(SMemRowBuilder* pBuilder) { +SMemRow tdGenMemRowFromBuilder(SMemRowBuilder* pBuilder) { SSchema* pSchema = pBuilder->pSchema; char* p = (char*)pBuilder->buf; @@ -1840,7 +1840,7 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, bo pDataBlock = (char*)pDataBlock + dataRowLen(trow); // next SDataRow pBlock->dataLen += dataRowLen(trow); // SSubmitBlk data length #endif - tdGetMemRowFromBuilder(&mRowBuilder); + tdGenMemRowFromBuilder(&mRowBuilder); } int32_t len = pBlock->dataLen + pBlock->schemaLen; diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index 4b69192b7b..5c3ccdd781 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -224,10 +224,10 @@ typedef void *SMemRow; #define dataRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_DATA_ROW_HEAD_SIZE) #define dataRowDeleted(r) TKEY_IS_DELETED(dataRowTKey(r)) -// SDataRow tdNewDataRowFromSchema(STSchema *pSchema); -// void tdFreeDataRow(SDataRow row); +SDataRow tdNewDataRowFromSchema(STSchema *pSchema); +void tdFreeDataRow(SDataRow row); void tdInitDataRow(SDataRow row, STSchema *pSchema); -// SDataRow tdDataRowDup(SDataRow row); +SDataRow tdDataRowDup(SDataRow row); SMemRow tdMemRowDup(SMemRow row); // offset here not include dataRow header length @@ -253,6 +253,16 @@ static FORCE_INLINE int tdAppendColVal(SDataRow row, const void *value, int8_t t return 0; } +// NOTE: offset here including the header size +static FORCE_INLINE void *tdGetRowDataOfCol(void *row, int8_t type, int32_t offset) { + if (IS_VAR_DATA_TYPE(type)) { + return POINTER_SHIFT(row, *(VarDataOffsetT *)POINTER_SHIFT(row, offset)); + } else { + return POINTER_SHIFT(row, offset); + } + return NULL; +} + // ----------------- Data column structure typedef struct SDataCol { int8_t type; // column type @@ -519,27 +529,26 @@ static FORCE_INLINE int tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t colId, return 0; } -// ----------------- Data row structure - // ----------------- Sequential Data row structure -/* A sequential data row, the format is like below: - * |<--------------------+--------------------------- len ---------------------------------->| - * |<-- Head -->|<--------- flen -------------->| | - * +---------------------+---------------------------------+---------------------------------+ - * | uint16_t | int16_t | | | - * +----------+----------+---------------------------------+---------------------------------+ - * | len | sversion | First part | Second part | - * +----------+----------+---------------------------------+---------------------------------+ +/* + * |-------------------------------+--------------------------- len ---------------------------------->| + * |<-------- Head ------>|<--------- flen -------------->| | + * |---------+---------------------+---------------------------------+---------------------------------+ + * | uint8_t | uint16_t | int16_t | | | + * |---------+----------+----------+---------------------------------+---------------------------------+ + * | flag | len | sversion | First part | Second part | + * +---------+----------+----------+---------------------------------+---------------------------------+ * * NOTE: timestamp in this row structure is TKEY instead of TSKEY */ + // ----------------- K-V data row structure /* - * +----------+----------+---------------------------------+---------------------------------+ - * | int16_t | int16_t | | | - * +----------+----------+---------------------------------+---------------------------------+ - * | len | ncols | cols index | data part | - * +----------+----------+---------------------------------+---------------------------------+ + * |--------------------+----------+---------------------------------+---------------------------------+ + * | uint8_t | uint16_t | int16_t | | | + * |---------+----------+----------+---------------------------------+---------------------------------+ + * | flag | len | ncols | cols index | data part | + * |---------+----------+----------+---------------------------------+---------------------------------+ */ #define TD_MEM_ROW_TYPE_SIZE sizeof(uint8_t) @@ -572,24 +581,11 @@ static FORCE_INLINE int tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t colId, #define memRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_MEM_ROW_HEAD_SIZE) #define memRowDeleted(r) TKEY_IS_DELETED(memRowTKey(r)) -// #define memRowNCols(r) (*(int16_t *)POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE + sizeof(int16_t))) // for SKVRow -// #define memRowSetNCols(r, n) memRowNCols(r) = (n) // for SKVRow -// #define memRowColIdx(r) (SColIdx *)POINTER_SHIFT(r, TD_MEM_ROW_HEAD_SIZE) // for SKVRow -// #define memRowValues(r) POINTER_SHIFT(r, TD_MEM_ROW_HEAD_SIZE + sizeof(SColIdx) * memRowNCols(r)) // for SKVRow - // NOTE: offset here including the header size -static FORCE_INLINE void *tdGetRowDataOfCol(void *row, int8_t type, int32_t offset) { - if (IS_VAR_DATA_TYPE(type)) { - return POINTER_SHIFT(row, *(VarDataOffsetT *)POINTER_SHIFT(row, offset)); - } else { - return POINTER_SHIFT(row, offset); - } - return NULL; -} static FORCE_INLINE void *tdGetKvRowDataOfCol(void *row, int8_t type, int32_t offset) { return POINTER_SHIFT(row, offset); } - +// NOTE: offset here including the header size static FORCE_INLINE void *tdGetMemRowDataOfCol(void *row, int8_t type, int32_t offset) { if (isDataRow(row)) { return tdGetRowDataOfCol(row, type, offset); @@ -601,12 +597,6 @@ static FORCE_INLINE void *tdGetMemRowDataOfCol(void *row, int8_t type, int32_t o return NULL; } -// #define kvRowCpy(dst, r) memcpy((dst), (r), kvRowLen(r)) -// #define kvRowColVal(r, colIdx) POINTER_SHIFT(kvRowValues(r), (colIdx)->offset) -// #define kvRowColIdxAt(r, i) (kvRowColIdx(r) + (i)) -// #define kvRowFree(r) tfree(r) -// #define kvRowEnd(r) POINTER_SHIFT(r, kvRowLen(r)) - #ifdef __cplusplus } #endif diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index f962e5255b..ac7b15ea02 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -188,30 +188,30 @@ void tdInitDataRow(SDataRow row, STSchema *pSchema) { dataRowSetVersion(row, schemaVersion(pSchema)); } -// SDataRow tdNewDataRowFromSchema(STSchema *pSchema) { -// int32_t size = dataRowMaxBytesFromSchema(pSchema); +SDataRow tdNewDataRowFromSchema(STSchema *pSchema) { + int32_t size = dataRowMaxBytesFromSchema(pSchema); -// SDataRow row = malloc(size); -// if (row == NULL) return NULL; + SDataRow row = malloc(size); + if (row == NULL) return NULL; -// tdInitDataRow(row, pSchema); -// return row; -// } + tdInitDataRow(row, pSchema); + return row; +} /** * Free the SDataRow object */ -// void tdFreeDataRow(SDataRow row) { -// if (row) free(row); -// } +void tdFreeDataRow(SDataRow row) { + if (row) free(row); +} -// SDataRow tdDataRowDup(SDataRow row) { -// SDataRow trow = malloc(dataRowLen(row)); -// if (trow == NULL) return NULL; +SDataRow tdDataRowDup(SDataRow row) { + SDataRow trow = malloc(dataRowLen(row)); + if (trow == NULL) return NULL; -// dataRowCpy(trow, row); -// return trow; -// } + dataRowCpy(trow, row); + return trow; +} SMemRow tdMemRowDup(SMemRow row) { SMemRow trow = malloc(memRowTLen(row)); @@ -220,6 +220,7 @@ SMemRow tdMemRowDup(SMemRow row) { memRowCpy(trow, row); return trow; } + void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints) { pDataCol->type = colType(pCol); pDataCol->colId = colColId(pCol); @@ -436,7 +437,7 @@ void tdResetDataCols(SDataCols *pCols) { static void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols) { ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < dataRowKey(row)); - int rcol = 0; // rowCol + int rcol = 0; int dcol = 0; if (dataRowDeleted(row)) { diff --git a/src/inc/ttype.h b/src/inc/ttype.h index 0308379509..80b8ddcd4e 100644 --- a/src/inc/ttype.h +++ b/src/inc/ttype.h @@ -11,7 +11,7 @@ extern "C" { // ----------------- For variable data types such as TSDB_DATA_TYPE_BINARY and TSDB_DATA_TYPE_NCHAR typedef int32_t VarDataOffsetT; -typedef int16_t VarDataLenT; // maxDataLen: 32767 +typedef int16_t VarDataLenT; // maxVarDataLen: 32767 typedef struct tstr { VarDataLenT len; diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 0c87a7b735..faf099cdd2 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -1442,107 +1442,6 @@ int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity return numOfRows + num; } -#if 0 -static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, SMemRow row, - int32_t numOfCols, STable* pTable, STSchema* pSchema) { - char* pData = NULL; - - // the schema version info is embedded in SDataRow, and use latest schema version for SKVRow - int32_t numOfRowCols = 0; - if (pSchema == NULL) { - pSchema = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row)); - numOfRowCols = schemaNCols(pSchema); - } else { - numOfRowCols = schemaNCols(pSchema); - } - - int32_t i = 0, j = 0; - while(i < numOfCols && j < numOfRowCols) { - SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); - if (pSchema->columns[j].colId < pColInfo->info.colId) { - j++; - continue; - } - - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { - pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; - } else { - pData = (char*)pColInfo->pData + (capacity - numOfRows - 1) * pColInfo->info.bytes; - } - - if (pSchema->columns[j].colId == pColInfo->info.colId) { - void* value = - tdGetMemRowDataOfCol(row, (int8_t)pColInfo->info.type, TD_MEM_ROW_HEAD_SIZE + pSchema->columns[j].offset); - switch (pColInfo->info.type) { - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - memcpy(pData, value, varDataTLen(value)); - break; - case TSDB_DATA_TYPE_NULL: - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_UTINYINT: - *(uint8_t *)pData = *(uint8_t *)value; - break; - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_USMALLINT: - *(uint16_t *)pData = *(uint16_t *)value; - break; - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_UINT: - *(uint32_t *)pData = *(uint32_t *)value; - break; - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_UBIGINT: - *(uint64_t *)pData = *(uint64_t *)value; - break; - case TSDB_DATA_TYPE_FLOAT: - SET_FLOAT_PTR(pData, value); - break; - case TSDB_DATA_TYPE_DOUBLE: - SET_DOUBLE_PTR(pData, value); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - *(TSKEY *)pData = tdGetKey(*(TKEY *)value); - } else { - *(TSKEY *)pData = *(TSKEY *)value; - } - break; - default: - memcpy(pData, value, pColInfo->info.bytes); - } - - j++; - i++; - } else { // pColInfo->info.colId < pSchema->columns[j].colId, it is a NULL data - if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { - setVardataNull(pData, pColInfo->info.type); - } else { - setNull(pData, pColInfo->info.type, pColInfo->info.bytes); - } - i++; - } - } - - while (i < numOfCols) { // the remain columns are all null data - SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { - pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; - } else { - pData = (char*)pColInfo->pData + (capacity - numOfRows - 1) * pColInfo->info.bytes; - } - - if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { - setVardataNull(pData, pColInfo->info.type); - } else { - setNull(pData, pColInfo->info.type, pColInfo->info.bytes); - } - - i++; - } -} -#endif static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, SMemRow row, int32_t numOfCols, STable* pTable, STSchema* pSchema) { From 0d9cf34882fc380978fb605e477bb4f87115e47e Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 30 Jun 2021 18:52:26 +0800 Subject: [PATCH 07/67] SKVRow offset bug fix --- src/client/src/tscUtil.c | 9 +++------ src/common/inc/tdataformat.h | 6 +++--- src/common/src/tdataformat.c | 3 ++- src/tsdb/src/tsdbRead.c | 3 ++- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 2c53d90cea..c05c5e3d3b 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1667,7 +1667,7 @@ int32_t tscGetDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, i // pBuilder->size = 0; // } -#define KvRowNColsThresh 1 // default value: 32 TODO: for test, restore to 32 after test finished +#define KvRowNColsThresh 1 // default value: 128 TODO: for test, restore to default value after test finished static FORCE_INLINE uint8_t tdRowTypeJudger(SSchema* pSchema, void* pData, int32_t nCols, int32_t flen, uint16_t* nColsNotNull) { @@ -1724,7 +1724,6 @@ SMemRow tdGenMemRowFromBuilder(SMemRowBuilder* pBuilder) { uint16_t nColsNotNull = 0; uint8_t memRowType = tdRowTypeJudger(pSchema, p, pBuilder->nCols, pBuilder->flen, &nColsNotNull); tscDebug("prop:memType is %d", memRowType); - memRowType = SMEM_ROW_DATA; SMemRow* memRow = (SMemRow)pBuilder->pDataBlock; memRowSetType(memRow, memRowType); @@ -1744,12 +1743,10 @@ SMemRow tdGenMemRowFromBuilder(SMemRowBuilder* pBuilder) { pBuilder->buf = p; } else if (memRowType == SMEM_ROW_KV) { ASSERT(nColsNotNull < pBuilder->nCols); - - uint16_t tlen = TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * nColsNotNull + pBuilder->size; SKVRow kvRow = (SKVRow)memRowBody(memRow); - - kvRowSetNCols(kvRow, nColsNotNull); + uint16_t tlen = TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * nColsNotNull; kvRowSetLen(kvRow, tlen); + kvRowSetNCols(kvRow, nColsNotNull); int toffset = 0; p = (char*)pBuilder->buf; diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index 5c3ccdd781..ca8b4d33da 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -214,7 +214,7 @@ typedef void *SMemRow; #define TD_DATA_ROW_HEAD_SIZE (sizeof(uint16_t) + sizeof(int16_t)) #define dataRowLen(r) (*(uint16_t *)(r)) -#define dataRowVersion(r) *(int16_t *)POINTER_SHIFT(r, sizeof(int16_t)) +#define dataRowVersion(r) (*(int16_t *)POINTER_SHIFT(r, sizeof(int16_t))) #define dataRowTuple(r) POINTER_SHIFT(r, TD_DATA_ROW_HEAD_SIZE) #define dataRowTKey(r) (*(TKEY *)(dataRowTuple(r))) #define dataRowKey(r) tdGetKey(dataRowTKey(r)) @@ -425,7 +425,7 @@ typedef struct { #define kvRowNCols(r) (*(int16_t *)POINTER_SHIFT(r, sizeof(uint16_t))) #define kvRowSetLen(r, len) kvRowLen(r) = (len) #define kvRowSetNCols(r, n) kvRowNCols(r) = (n) -#define kvRowColIdx(r) (SColIdx *)POINTER_SHIFT(r, TD_KV_ROW_HEAD_SIZE) +#define kvRowColIdx(r) ((SColIdx *)POINTER_SHIFT(r, TD_KV_ROW_HEAD_SIZE)) #define kvRowValues(r) POINTER_SHIFT(r, TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * kvRowNCols(r)) #define kvRowCpy(dst, r) memcpy((dst), (r), kvRowLen(r)) #define kvRowColVal(r, colIdx) POINTER_SHIFT(kvRowValues(r), (colIdx)->offset) @@ -468,7 +468,7 @@ static FORCE_INLINE int tdAppendKvColVal(SKVRow row, const void *value, int16_t char * ptr = (char *)POINTER_SHIFT(row, kvRowLen(row)); pColIdx->colId = colId; - pColIdx->offset = kvRowLen(row); + pColIdx->offset = kvRowLen(row); // offset of pColIdx including the TD_KV_ROW_HEAD_SIZE if (IS_VAR_DATA_TYPE(type)) { memcpy(ptr, value, varDataTLen(value)); diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index ac7b15ea02..2a19d504dd 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -527,7 +527,8 @@ static void tdAppendKvRowToDataCol(SKVRow row, STSchema *pSchema, SDataCols *pCo int nRowColsMatched = 0; STColumn stColumn[nRowCols]; tdGetKVRowColInfo(pSchema, kvRowColIdx(row), nRowCols, stColumn, &nRowColsMatched); - uDebug("kvRow: nRowCols=%d, nRowColsMatched=%d, nSchemaCols=%d", nRowCols, nRowColsMatched, schemaNCols(pSchema)); + uDebug("prop:kvRow: nRowCols=%d, nRowColsMatched=%d, nSchemaCols=%d", nRowCols, nRowColsMatched, + schemaNCols(pSchema)); while (dcol < pCols->numOfCols) { SDataCol *pDataCol = &(pCols->cols[dcol]); diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index faf099cdd2..27502f67f7 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -1550,7 +1550,8 @@ static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, if (pColIdx->colId == pColInfo->info.colId) { STColumn* pSTColumn = tdGetColOfID(pSchema, pColIdx->colId); if (pSTColumn != NULL) { - void* value = tdGetKvRowDataOfCol(row, pSTColumn->type, TD_MEM_ROW_HEAD_SIZE + pColIdx->offset); + // offset of pColIdx including the TD_KV_ROW_HEAD_SIZE + void* value = tdGetKvRowDataOfCol(kvRow, pSTColumn->type, pColIdx->offset); switch (pColInfo->info.type) { case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: From 879f9f7526fdc40208711b21ecd1eb0997455dbe Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 1 Jul 2021 09:47:07 +0800 Subject: [PATCH 08/67] code optimization --- src/common/src/tdataformat.c | 72 ++++++++++++++++++------------------ 1 file changed, 35 insertions(+), 37 deletions(-) diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index 2a19d504dd..74f9893608 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -475,37 +475,37 @@ static void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols pCols->numOfRows++; } -static void tdGetKVRowColInfo(const STSchema *pSchema, SColIdx *pColIdx, int nRowCols, STColumn *pSTColumn, - int *nColMatched) { - int nSchema = schemaNCols(pSchema); - int iCol = 0; - int iSchema = 0; - int nColMatch = 0; - SColIdx * pIdx = pColIdx; - const STColumn *pColumn = NULL; +// static void tdGetKVRowColInfo(const STSchema *pSchema, SColIdx *pColIdx, int nRowCols, STColumn *pSTColumn, +// int *nColMatched) { +// int nSchema = schemaNCols(pSchema); +// int iCol = 0; +// int iSchema = 0; +// int nColMatch = 0; +// SColIdx * pIdx = pColIdx; +// const STColumn *pColumn = NULL; - while (iCol < nRowCols && iSchema < nSchema) { - pColumn = &pSchema->columns[iSchema]; - if (pIdx->colId == pColumn->colId) { - pSTColumn[nColMatch].colId = pIdx->colId; - pSTColumn[nColMatch].type = pColumn->type; - pSTColumn[nColMatch].bytes = pColumn->bytes; - pSTColumn[nColMatch].offset = pIdx->offset; +// while (iCol < nRowCols && iSchema < nSchema) { +// pColumn = &pSchema->columns[iSchema]; +// if (pIdx->colId == pColumn->colId) { +// pSTColumn[nColMatch].colId = pIdx->colId; +// pSTColumn[nColMatch].type = pColumn->type; +// pSTColumn[nColMatch].bytes = pColumn->bytes; +// pSTColumn[nColMatch].offset = pIdx->offset; - pIdx += sizeof(SColIdx); +// pIdx += sizeof(SColIdx); - ++iCol; - ++iSchema; - ++nColMatch; - } else if (pIdx->colId > pColumn->colId) { - ++iSchema; - } else { - pIdx += sizeof(SColIdx); - ++iCol; - } - } - *nColMatched = nColMatch; -} +// ++iCol; +// ++iSchema; +// ++nColMatch; +// } else if (pIdx->colId > pColumn->colId) { +// ++iSchema; +// } else { +// pIdx += sizeof(SColIdx); +// ++iCol; +// } +// } +// *nColMatched = nColMatch; +// } static void tdAppendKvRowToDataCol(SKVRow row, STSchema *pSchema, SDataCols *pCols) { ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < kvRowKey(row)); @@ -524,15 +524,15 @@ static void tdAppendKvRowToDataCol(SKVRow row, STSchema *pSchema, SDataCols *pCo } } else { int nRowCols = kvRowNCols(row); - int nRowColsMatched = 0; - STColumn stColumn[nRowCols]; - tdGetKVRowColInfo(pSchema, kvRowColIdx(row), nRowCols, stColumn, &nRowColsMatched); - uDebug("prop:kvRow: nRowCols=%d, nRowColsMatched=%d, nSchemaCols=%d", nRowCols, nRowColsMatched, - schemaNCols(pSchema)); + // int nRowColsMatched = 0; + // STColumn stColumn[nRowCols]; + // tdGetKVRowColInfo(pSchema, kvRowColIdx(row), nRowCols, stColumn, &nRowColsMatched); + // uInfo("prop:kvRow: nRowCols=%d, nRowColsMatched=%d, nSchemaCols=%d", nRowCols, nRowColsMatched, + // schemaNCols(pSchema)); while (dcol < pCols->numOfCols) { SDataCol *pDataCol = &(pCols->cols[dcol]); - if (rcol >= nRowColsMatched || rcol >= schemaNCols(pSchema)) { + if (rcol >= nRowCols || rcol >= schemaNCols(pSchema)) { dataColAppendVal(pDataCol, tdGetNullVal(pDataCol->type), pCols->numOfRows, pCols->maxPoints); dcol++; continue; @@ -541,9 +541,7 @@ static void tdAppendKvRowToDataCol(SKVRow row, STSchema *pSchema, SDataCols *pCo SColIdx *colIdx = kvRowColIdxAt(row, rcol); if (colIdx->colId == pDataCol->colId) { - ASSERT(pDataCol->type == stColumn[rcol].type); - - void *value = tdGetKvRowDataOfCol(row, pDataCol->type, stColumn[rcol].offset + TD_KV_ROW_HEAD_SIZE); + void *value = tdGetKvRowDataOfCol(row, pDataCol->type, colIdx->offset); dataColAppendVal(pDataCol, value, pCols->numOfRows, pCols->maxPoints); dcol++; rcol++; From 2e5ce3ebfe53d07703c10de033666c46ea6a464c Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Thu, 1 Jul 2021 06:28:22 +0000 Subject: [PATCH 09/67] adaption for cacheLast --- src/client/src/tscUtil.c | 2 +- src/common/inc/tdataformat.h | 44 ++++++++++++++--- src/common/src/tdataformat.c | 56 ++++------------------ src/inc/ttype.h | 2 +- src/tsdb/src/tsdbMain.c | 22 ++++----- src/tsdb/src/tsdbMemTable.c | 26 +++++++--- src/tsdb/src/tsdbRead.c | 92 +++++++++++++++++------------------- 7 files changed, 125 insertions(+), 119 deletions(-) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index c05c5e3d3b..a688d34d41 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1667,7 +1667,7 @@ int32_t tscGetDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, i // pBuilder->size = 0; // } -#define KvRowNColsThresh 1 // default value: 128 TODO: for test, restore to default value after test finished +#define KvRowNColsThresh 1 // default value: 1200 TODO: for test, restore to default value after test finished static FORCE_INLINE uint8_t tdRowTypeJudger(SSchema* pSchema, void* pData, int32_t nCols, int32_t flen, uint16_t* nColsNotNull) { diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index ca8b4d33da..c2460ec7da 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -280,13 +280,43 @@ typedef struct SDataCol { static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; } void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints); -void dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints); void dataColSetOffset(SDataCol *pCol, int nEle); bool isNEleNull(SDataCol *pCol, int nEle); void dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints); -static const void *tdGetNullVal(int8_t type) { +FORCE_INLINE void dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints); +// value from timestamp should be TKEY here instead of TSKEY +FORCE_INLINE void dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints) { + ASSERT(pCol != NULL && value != NULL); + + if (pCol->len == 0) { + if (isNull(value, pCol->type)) { + // all null value yet, just return + return; + } + + if (numOfRows > 0) { + // Find the first not null value, fill all previous values as NULL + dataColSetNEleNull(pCol, numOfRows, maxPoints); + } + } + + if (IS_VAR_DATA_TYPE(pCol->type)) { + // set offset + pCol->dataOff[numOfRows] = pCol->len; + // Copy data + memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, varDataTLen(value)); + // Update the length + pCol->len += varDataTLen(value); + } else { + ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfRows); + memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, pCol->bytes); + pCol->len += pCol->bytes; + } +} + +static FORCE_INLINE const void *tdGetNullVal(int8_t type) { switch (type) { case TSDB_DATA_TYPE_BOOL: return &BoolNull; @@ -460,6 +490,10 @@ static FORCE_INLINE void *tdGetKVRowValOfCol(SKVRow row, int16_t colId) { return kvRowColVal(row, (SColIdx *)ret); } +static FORCE_INLINE void *tdGetKVRowIdxOfCol(SKVRow row, int16_t colId) { + return taosbsearch(&colId, kvRowColIdx(row), kvRowNCols(row), sizeof(SColIdx), comparTagId, TD_EQ); +} + // offset here not include kvRow header length static FORCE_INLINE int tdAppendKvColVal(SKVRow row, const void *value, int16_t colId, int8_t type, int32_t offset) { ASSERT(value != NULL); @@ -582,15 +616,13 @@ static FORCE_INLINE int tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t colId, #define memRowDeleted(r) TKEY_IS_DELETED(memRowTKey(r)) // NOTE: offset here including the header size -static FORCE_INLINE void *tdGetKvRowDataOfCol(void *row, int8_t type, int32_t offset) { - return POINTER_SHIFT(row, offset); -} +static FORCE_INLINE void *tdGetKvRowDataOfCol(void *row, int32_t offset) { return POINTER_SHIFT(row, offset); } // NOTE: offset here including the header size static FORCE_INLINE void *tdGetMemRowDataOfCol(void *row, int8_t type, int32_t offset) { if (isDataRow(row)) { return tdGetRowDataOfCol(row, type, offset); } else if (isKvRow(row)) { - return tdGetKvRowDataOfCol(row, type, offset); + return tdGetKvRowDataOfCol(row, offset); } else { ASSERT(0); } diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index 74f9893608..d346b5c463 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -240,7 +240,7 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints) *pBuf = POINTER_SHIFT(*pBuf, pDataCol->spaceSize); } } - +#if 0 // value from timestamp should be TKEY here instead of TSKEY void dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints) { ASSERT(pCol != NULL && value != NULL); @@ -270,6 +270,7 @@ void dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxP pCol->len += pCol->bytes; } } +#endif bool isNEleNull(SDataCol *pCol, int nEle) { for (int i = 0; i < nEle; i++) { @@ -278,7 +279,7 @@ bool isNEleNull(SDataCol *pCol, int nEle) { return true; } -void dataColSetNullAt(SDataCol *pCol, int index) { +FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index) { if (IS_VAR_DATA_TYPE(pCol->type)) { pCol->dataOff[index] = pCol->len; char *ptr = POINTER_SHIFT(pCol->pData, pCol->len); @@ -475,38 +476,6 @@ static void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols pCols->numOfRows++; } -// static void tdGetKVRowColInfo(const STSchema *pSchema, SColIdx *pColIdx, int nRowCols, STColumn *pSTColumn, -// int *nColMatched) { -// int nSchema = schemaNCols(pSchema); -// int iCol = 0; -// int iSchema = 0; -// int nColMatch = 0; -// SColIdx * pIdx = pColIdx; -// const STColumn *pColumn = NULL; - -// while (iCol < nRowCols && iSchema < nSchema) { -// pColumn = &pSchema->columns[iSchema]; -// if (pIdx->colId == pColumn->colId) { -// pSTColumn[nColMatch].colId = pIdx->colId; -// pSTColumn[nColMatch].type = pColumn->type; -// pSTColumn[nColMatch].bytes = pColumn->bytes; -// pSTColumn[nColMatch].offset = pIdx->offset; - -// pIdx += sizeof(SColIdx); - -// ++iCol; -// ++iSchema; -// ++nColMatch; -// } else if (pIdx->colId > pColumn->colId) { -// ++iSchema; -// } else { -// pIdx += sizeof(SColIdx); -// ++iCol; -// } -// } -// *nColMatched = nColMatch; -// } - static void tdAppendKvRowToDataCol(SKVRow row, STSchema *pSchema, SDataCols *pCols) { ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < kvRowKey(row)); @@ -523,33 +492,28 @@ static void tdAppendKvRowToDataCol(SKVRow row, STSchema *pSchema, SDataCols *pCo } } } else { - int nRowCols = kvRowNCols(row); - // int nRowColsMatched = 0; - // STColumn stColumn[nRowCols]; - // tdGetKVRowColInfo(pSchema, kvRowColIdx(row), nRowCols, stColumn, &nRowColsMatched); - // uInfo("prop:kvRow: nRowCols=%d, nRowColsMatched=%d, nSchemaCols=%d", nRowCols, nRowColsMatched, - // schemaNCols(pSchema)); + int nRowCols = kvRowNCols(row); while (dcol < pCols->numOfCols) { SDataCol *pDataCol = &(pCols->cols[dcol]); if (rcol >= nRowCols || rcol >= schemaNCols(pSchema)) { dataColAppendVal(pDataCol, tdGetNullVal(pDataCol->type), pCols->numOfRows, pCols->maxPoints); - dcol++; + ++dcol; continue; } SColIdx *colIdx = kvRowColIdxAt(row, rcol); if (colIdx->colId == pDataCol->colId) { - void *value = tdGetKvRowDataOfCol(row, pDataCol->type, colIdx->offset); + void *value = tdGetKvRowDataOfCol(row, colIdx->offset); dataColAppendVal(pDataCol, value, pCols->numOfRows, pCols->maxPoints); - dcol++; - rcol++; + ++dcol; + ++rcol; } else if (colIdx->colId < pDataCol->colId) { - rcol++; + ++rcol; } else { dataColAppendVal(pDataCol, tdGetNullVal(pDataCol->type), pCols->numOfRows, pCols->maxPoints); - dcol++; + ++dcol; } } } diff --git a/src/inc/ttype.h b/src/inc/ttype.h index 80b8ddcd4e..b858d2a7f5 100644 --- a/src/inc/ttype.h +++ b/src/inc/ttype.h @@ -121,7 +121,7 @@ typedef struct tstr { #define IS_VALID_UINT(_t) ((_t) >= 0 && (_t) < UINT32_MAX) #define IS_VALID_UBIGINT(_t) ((_t) >= 0 && (_t) < UINT64_MAX) -static FORCE_INLINE bool isNull(const char *val, int32_t type) { +FORCE_INLINE bool isNull(const char *val, int32_t type) { switch (type) { case TSDB_DATA_TYPE_BOOL: return *(uint8_t *)val == TSDB_DATA_BOOL_NULL; diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index c0c65cecd5..b34308fd76 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -661,7 +661,9 @@ static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pRea err = -1; goto out; } - tdInitDataRow(POINTER_SHIFT(row, TD_MEM_ROW_TYPE_SIZE), pSchema); + + memRowSetType(row, SMEM_ROW_DATA); + tdInitDataRow(memRowBody(row), pSchema); // first load block index info if (tsdbLoadBlockInfo(pReadh, NULL) < 0) { @@ -718,10 +720,9 @@ static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pRea // OK,let's load row from backward to get not-null column for (int32_t rowId = pBlock->numOfRows - 1; rowId >= 0; rowId--) { SDataCol *pDataCol = pReadh->pDCols[0]->cols + i; - tdAppendColVal(POINTER_SHIFT(row, TD_MEM_ROW_TYPE_SIZE), tdGetColDataOfRow(pDataCol, rowId), pCol->type, - pCol->bytes, pCol->offset); + tdAppendColVal(memRowBody(row), tdGetColDataOfRow(pDataCol, rowId), pCol->type, pCol->bytes, pCol->offset); //SDataCol *pDataCol = readh.pDCols[0]->cols + j; - void *value = tdGetMemRowDataOfCol(row, (int8_t)pCol->type, TD_MEM_ROW_HEAD_SIZE + pCol->offset); + void *value = tdGetRowDataOfCol(memRowBody(row), (int8_t)pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset); if (isNull(value, pCol->type)) { continue; } @@ -741,8 +742,7 @@ static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pRea // save row ts(in column 0) pDataCol = pReadh->pDCols[0]->cols + 0; pCol = schemaColAt(pSchema, 0); - tdAppendColVal(POINTER_SHIFT(row, TD_MEM_ROW_TYPE_SIZE), tdGetColDataOfRow(pDataCol, rowId), pCol->type, - pCol->bytes, pCol->offset); + tdAppendColVal(memRowBody(row), tdGetColDataOfRow(pDataCol, rowId), pCol->type, pCol->bytes, pCol->offset); pLastCol->ts = memRowKey(row); pTable->restoreColumnNum += 1; @@ -779,18 +779,18 @@ static int tsdbRestoreLastRow(STsdbRepo *pRepo, STable *pTable, SReadH* pReadh, // Get the data in row STSchema *pSchema = tsdbGetTableSchema(pTable); - pTable->lastRow = taosTMalloc(dataRowMaxBytesFromSchema(pSchema)); + pTable->lastRow = taosTMalloc(memRowMaxBytesFromSchema(pSchema)); if (pTable->lastRow == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return -1; } - - tdInitDataRow(pTable->lastRow, pSchema); + memRowSetType(pTable->lastRow, SMEM_ROW_DATA); + tdInitDataRow(memRowBody(pTable->lastRow), pSchema); for (int icol = 0; icol < schemaNCols(pSchema); icol++) { STColumn *pCol = schemaColAt(pSchema, icol); SDataCol *pDataCol = pReadh->pDCols[0]->cols + icol; - tdAppendColVal(pTable->lastRow, tdGetColDataOfRow(pDataCol, pBlock->numOfRows - 1), pCol->type, pCol->bytes, - pCol->offset); + tdAppendColVal(memRowBody(pTable->lastRow), tdGetColDataOfRow(pDataCol, pBlock->numOfRows - 1), pCol->type, + pCol->bytes, pCol->offset); } return 0; diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 7967462ffe..41e28ac909 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -933,12 +933,12 @@ static int tsdbInsertDataToTableImpl(STsdbRepo *pRepo, STable *pTable, void **ro tSkipListPutBatch(pTableData->pData, rows, rowCounter); int64_t dsize = SL_SIZE(pTableData->pData) - osize; - if (pMemTable->keyFirst > dataRowKey(rows[0])) pMemTable->keyFirst = dataRowKey(rows[0]); - if (pMemTable->keyLast < dataRowKey(rows[rowCounter - 1])) pMemTable->keyLast = dataRowKey(rows[rowCounter - 1]); + if (pMemTable->keyFirst > memRowKey(rows[0])) pMemTable->keyFirst = memRowKey(rows[0]); + if (pMemTable->keyLast < memRowKey(rows[rowCounter - 1])) pMemTable->keyLast = memRowKey(rows[rowCounter - 1]); pMemTable->numOfRows += dsize; - if (pTableData->keyFirst > dataRowKey(rows[0])) pTableData->keyFirst = dataRowKey(rows[0]); - if (pTableData->keyLast < dataRowKey(rows[rowCounter - 1])) pTableData->keyLast = dataRowKey(rows[rowCounter - 1]); + if (pTableData->keyFirst > memRowKey(rows[0])) pTableData->keyFirst = memRowKey(rows[0]); + if (pTableData->keyLast < memRowKey(rows[rowCounter - 1])) pTableData->keyLast = memRowKey(rows[rowCounter - 1]); pTableData->numOfRows += dsize; // update table latest info @@ -1004,6 +1004,8 @@ static void updateTableLatestColumn(STsdbRepo *pRepo, STable *pTable, SMemRow ro SDataCol *pLatestCols = pTable->lastCols; + bool isDataRow = isDataRow(row); + void *rowBody = memRowBody(row); for (int16_t j = 0; j < schemaNCols(pSchema); j++) { STColumn *pTCol = schemaColAt(pSchema, j); // ignore not exist colId @@ -1012,8 +1014,20 @@ static void updateTableLatestColumn(STsdbRepo *pRepo, STable *pTable, SMemRow ro continue; } - void *value = tdGetMemRowDataOfCol(row, (int8_t)pTCol->type, TD_MEM_ROW_HEAD_SIZE + pSchema->columns[j].offset); - if (isNull(value, pTCol->type)) { + void *value = NULL; + + if (isDataRow) { + value = tdGetRowDataOfCol(rowBody, (int8_t)pTCol->type, TD_DATA_ROW_HEAD_SIZE + pSchema->columns[j].offset); + } else { + // SKVRow + SColIdx *pColIdx = tdGetKVRowIdxOfCol(rowBody, pTCol->colId); + if(pColIdx) { +value = tdGetKvRowDataOfCol(rowBody, pColIdx->offset); + } + + } + + if ((value == NULL) || isNull(value, pTCol->type)) { continue; } diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 27502f67f7..395598f527 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -1548,56 +1548,52 @@ static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, } if (pColIdx->colId == pColInfo->info.colId) { - STColumn* pSTColumn = tdGetColOfID(pSchema, pColIdx->colId); - if (pSTColumn != NULL) { - // offset of pColIdx including the TD_KV_ROW_HEAD_SIZE - void* value = tdGetKvRowDataOfCol(kvRow, pSTColumn->type, pColIdx->offset); - switch (pColInfo->info.type) { - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - memcpy(pData, value, varDataTLen(value)); - break; - case TSDB_DATA_TYPE_NULL: - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_UTINYINT: - *(uint8_t*)pData = *(uint8_t*)value; - break; - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_USMALLINT: - *(uint16_t*)pData = *(uint16_t*)value; - break; - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_UINT: - *(uint32_t*)pData = *(uint32_t*)value; - break; - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_UBIGINT: - *(uint64_t*)pData = *(uint64_t*)value; - break; - case TSDB_DATA_TYPE_FLOAT: - SET_FLOAT_PTR(pData, value); - break; - case TSDB_DATA_TYPE_DOUBLE: - SET_DOUBLE_PTR(pData, value); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - *(TSKEY*)pData = tdGetKey(*(TKEY*)value); - } else { - *(TSKEY*)pData = *(TSKEY*)value; - } - break; - default: - memcpy(pData, value, pColInfo->info.bytes); - } - ++k; - ++i; - continue; + // offset of pColIdx for SKVRow including the TD_KV_ROW_HEAD_SIZE + void* value = tdGetKvRowDataOfCol(kvRow, pColIdx->offset); + switch (pColInfo->info.type) { + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + memcpy(pData, value, varDataTLen(value)); + break; + case TSDB_DATA_TYPE_NULL: + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: + *(uint8_t*)pData = *(uint8_t*)value; + break; + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: + *(uint16_t*)pData = *(uint16_t*)value; + break; + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: + *(uint32_t*)pData = *(uint32_t*)value; + break; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: + *(uint64_t*)pData = *(uint64_t*)value; + break; + case TSDB_DATA_TYPE_FLOAT: + SET_FLOAT_PTR(pData, value); + break; + case TSDB_DATA_TYPE_DOUBLE: + SET_DOUBLE_PTR(pData, value); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + *(TSKEY*)pData = tdGetKey(*(TKEY*)value); + } else { + *(TSKEY*)pData = *(TSKEY*)value; + } + break; + default: + memcpy(pData, value, pColInfo->info.bytes); } - ++k; // pSTColumn is NULL + ++k; + ++i; + continue; } - // If (pColInfo->info.colId < pColIdx->colId) or pSTColumn is NULL, it is a NULL data + // If (pColInfo->info.colId < pColIdx->colId), it is NULL data if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { setVardataNull(pData, pColInfo->info.type); } else { From fe77479a565cb415118aea681893ed7181bfc870 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 1 Jul 2021 16:01:33 +0800 Subject: [PATCH 10/67] CQ adaption to SMemRow --- src/client/inc/tscUtil.h | 35 ----------------------------------- src/client/src/tscUtil.c | 22 ++-------------------- src/common/inc/tdataformat.h | 35 ++++++++++++++++++----------------- src/cq/src/cqMain.c | 8 +++++--- src/query/src/qAggMain.c | 4 ++-- src/tsdb/src/tsdbCommit.c | 1 - src/tsdb/src/tsdbMemTable.c | 5 ++--- src/tsdb/src/tsdbRead.c | 2 +- 8 files changed, 30 insertions(+), 82 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 3c5cb4f23a..93684fa02c 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -348,10 +348,7 @@ typedef struct { int16_t sversion; int32_t flen; // for SKVRow - uint16_t tCols; uint16_t nCols; - SColIdx* pColIdx; - uint16_t alloc; uint16_t size; void* buf; @@ -359,40 +356,8 @@ typedef struct { SSubmitBlk* pSubmitBlk; } SMemRowBuilder; -// int tdInitMemRowBuilder(SMemRowBuilder* pBuilder); -// void tdDestroyMemRowBuilder(SMemRowBuilder* pBuilder); -// void tdResetMemRowBuilder(SMemRowBuilder* pBuilder); SMemRow tdGenMemRowFromBuilder(SMemRowBuilder* pBuilder); -// static FORCE_INLINE int tdAddColToMemRow(SMemRowBuilder* pBuilder, int16_t colId, int8_t type, void* value) { -// // TODO - -// if (pBuilder->nCols >= pBuilder->tCols) { -// pBuilder->tCols *= 2; -// pBuilder->pColIdx = (SColIdx*)realloc((void*)(pBuilder->pColIdx), sizeof(SColIdx) * pBuilder->tCols); -// if (pBuilder->pColIdx == NULL) return -1; -// } - -// pBuilder->pColIdx[pBuilder->nCols].colId = colId; -// pBuilder->pColIdx[pBuilder->nCols].offset = pBuilder->size; - -// pBuilder->nCols++; - -// int tlen = IS_VAR_DATA_TYPE(type) ? varDataTLen(value) : TYPE_BYTES[type]; -// if (tlen > pBuilder->alloc - pBuilder->size) { -// while (tlen > pBuilder->alloc - pBuilder->size) { -// pBuilder->alloc *= 2; -// } -// pBuilder->buf = realloc(pBuilder->buf, pBuilder->alloc); -// if (pBuilder->buf == NULL) return -1; -// } - -// memcpy(POINTER_SHIFT(pBuilder->buf, pBuilder->size), value, tlen); -// pBuilder->size += tlen; - -// return 0; -// } - #ifdef __cplusplus } #endif diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index a688d34d41..d51ff3d3ed 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1716,6 +1716,7 @@ static FORCE_INLINE uint8_t tdRowTypeJudger(SSchema* pSchema, void* pData, int32 SMemRow tdGenMemRowFromBuilder(SMemRowBuilder* pBuilder) { SSchema* pSchema = pBuilder->pSchema; char* p = (char*)pBuilder->buf; + int toffset = 0; if(pBuilder->nCols <= 0){ return NULL; @@ -1723,7 +1724,6 @@ SMemRow tdGenMemRowFromBuilder(SMemRowBuilder* pBuilder) { uint16_t nColsNotNull = 0; uint8_t memRowType = tdRowTypeJudger(pSchema, p, pBuilder->nCols, pBuilder->flen, &nColsNotNull); - tscDebug("prop:memType is %d", memRowType); SMemRow* memRow = (SMemRow)pBuilder->pDataBlock; memRowSetType(memRow, memRowType); @@ -1733,7 +1733,6 @@ SMemRow tdGenMemRowFromBuilder(SMemRowBuilder* pBuilder) { dataRowSetLen(trow, (uint16_t)(TD_DATA_ROW_HEAD_SIZE + pBuilder->flen)); dataRowSetVersion(trow, pBuilder->sversion); - int toffset = 0; p = (char*)pBuilder->buf; for (int32_t j = 0; j < pBuilder->nCols; ++j) { tdAppendColVal(trow, p, pSchema[j].type, pSchema[j].bytes, toffset); @@ -1748,7 +1747,6 @@ SMemRow tdGenMemRowFromBuilder(SMemRowBuilder* pBuilder) { kvRowSetLen(kvRow, tlen); kvRowSetNCols(kvRow, nColsNotNull); - int toffset = 0; p = (char*)pBuilder->buf; for (int32_t j = 0; j < pBuilder->nCols; ++j) { if(!isNull(p, pSchema[j].type)) { @@ -1821,22 +1819,6 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, bo mRowBuilder.size = 0; for (int32_t i = 0; i < numOfRows; ++i) { -#if 0 - SDataRow trow = (SDataRow)pDataBlock; // generate each SDataRow one by one - dataRowSetLen(trow, (uint16_t)(TD_DATA_ROW_HEAD_SIZE + flen)); - dataRowSetVersion(trow, pTableMeta->sversion); - - // scan each column data and generate the data row - int toffset = 0; - for (int32_t j = 0; j < tinfo.numOfColumns; j++) { - tdAppendColVal(trow, p, pSchema[j].type, pSchema[j].bytes, toffset); - toffset += TYPE_BYTES[pSchema[j].type]; - p += pSchema[j].bytes; - } - - pDataBlock = (char*)pDataBlock + dataRowLen(trow); // next SDataRow - pBlock->dataLen += dataRowLen(trow); // SSubmitBlk data length -#endif tdGenMemRowFromBuilder(&mRowBuilder); } @@ -1849,7 +1831,7 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, bo static int32_t getRowExpandSize(STableMeta* pTableMeta) { int32_t result = TD_DATA_ROW_HEAD_SIZE; - int32_t columns = tscGetNumOfColumns(pTableMeta); + int32_t columns = tscGetNumOfColumns(pTableMeta); SSchema* pSchema = tscGetTableSchema(pTableMeta); for(int32_t i = 0; i < columns; i++) { if (IS_VAR_DATA_TYPE((pSchema + i)->type)) { diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index c2460ec7da..4ab44a7918 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -73,7 +73,7 @@ typedef struct { int8_t type; // Column type int16_t colId; // column ID uint16_t bytes; // column bytes - uint16_t offset; // point offset in SDataRow/SKVRow after the header part. + uint16_t offset; // point offset in SDataRow after the header part. } STColumn; #define colType(col) ((col)->type) @@ -184,20 +184,6 @@ static FORCE_INLINE int tkeyComparFn(const void *tkey1, const void *tkey2) { return 0; } } -// ----------------- Sequential Data row structure - -/* A sequential data row, the format is like below: - * |<--------------------+--------------------------- len ---------------------------------->| - * |<-- Head -->|<--------- flen -------------->| | - * +---------------------+---------------------------------+---------------------------------+ - * | uint16_t | int16_t | | | - * +----------+----------+---------------------------------+---------------------------------+ - * | len | sversion | First part | Second part | - * +----------+----------+---------------------------------+---------------------------------+ - * - * NOTE: timestamp in this row structure is TKEY instead of TSKEY - */ -typedef void *SDataRow; /* A memory data row, the format is like below: *|---------+---------------------+--------------------------- len ---------------------------------->| *|<- type->|<-- Head -->|<--------- flen -------------->| | @@ -211,10 +197,25 @@ typedef void *SDataRow; */ typedef void *SMemRow; +// ----------------- Data row structure + +/* A data row, the format is like below: + * |<--------------------+--------------------------- len ---------------------------------->| + * |<-- Head -->|<--------- flen -------------->| | + * +---------------------+---------------------------------+---------------------------------+ + * | uint16_t | int16_t | | | + * +----------+----------+---------------------------------+---------------------------------+ + * | len | sversion | First part | Second part | + * +----------+----------+---------------------------------+---------------------------------+ + * + * NOTE: timestamp in this row structure is TKEY instead of TSKEY + */ +typedef void *SDataRow; + #define TD_DATA_ROW_HEAD_SIZE (sizeof(uint16_t) + sizeof(int16_t)) #define dataRowLen(r) (*(uint16_t *)(r)) -#define dataRowVersion(r) (*(int16_t *)POINTER_SHIFT(r, sizeof(int16_t))) +#define dataRowVersion(r) *(int16_t *)POINTER_SHIFT(r, sizeof(int16_t)) #define dataRowTuple(r) POINTER_SHIFT(r, TD_DATA_ROW_HEAD_SIZE) #define dataRowTKey(r) (*(TKEY *)(dataRowTuple(r))) #define dataRowKey(r) tdGetKey(dataRowTKey(r)) @@ -254,7 +255,7 @@ static FORCE_INLINE int tdAppendColVal(SDataRow row, const void *value, int8_t t } // NOTE: offset here including the header size -static FORCE_INLINE void *tdGetRowDataOfCol(void *row, int8_t type, int32_t offset) { +static FORCE_INLINE void *tdGetRowDataOfCol(SDataRow *row, int8_t type, int32_t offset) { if (IS_VAR_DATA_TYPE(type)) { return POINTER_SHIFT(row, *(VarDataOffsetT *)POINTER_SHIFT(row, offset)); } else { diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c index a460b1d619..6fe530dce5 100644 --- a/src/cq/src/cqMain.c +++ b/src/cq/src/cqMain.c @@ -476,7 +476,7 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) { cDebug("vgId:%d, id:%d CQ:%s stream result is ready", pContext->vgId, pObj->tid, pObj->sqlStr); - int32_t size = sizeof(SWalHead) + sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + TD_DATA_ROW_HEAD_SIZE + pObj->rowSize; + int32_t size = sizeof(SWalHead) + sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + TD_MEM_ROW_HEAD_SIZE + pObj->rowSize; char *buffer = calloc(size, 1); SWalHead *pHead = (SWalHead *)buffer; @@ -484,7 +484,9 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) { SSubmitBlk *pBlk = (SSubmitBlk *) (buffer + sizeof(SWalHead) + sizeof(SSubmitMsg)); SMemRow trow = (SMemRow)pBlk->data; - tdInitDataRow(POINTER_SHIFT(trow, TD_MEM_ROW_TYPE_SIZE), pSchema); + SDataRow dataRow = (SDataRow)memRowBody(trow); + memRowSetType(trow, SMEM_ROW_DATA); + tdInitDataRow(dataRow, pSchema); for (int32_t i = 0; i < pSchema->numOfCols; i++) { STColumn *c = pSchema->columns + i; @@ -500,7 +502,7 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) { memcpy((char *)val + sizeof(VarDataLenT), buf, len); varDataLen(val) = len; } - tdAppendColVal(POINTER_SHIFT(trow, TD_MEM_ROW_TYPE_SIZE), val, c->type, c->bytes, c->offset); + tdAppendColVal(dataRow, val, c->type, c->bytes, c->offset); } pBlk->dataLen = htonl(memRowTLen(trow)); pBlk->schemaLen = 0; diff --git a/src/query/src/qAggMain.c b/src/query/src/qAggMain.c index 676e5b6ce6..ef1408ab28 100644 --- a/src/query/src/qAggMain.c +++ b/src/query/src/qAggMain.c @@ -4002,7 +4002,7 @@ void blockInfo_func(SQLFunctionCtx* pCtx) { int32_t len = *(int32_t*) pCtx->pInput; blockDistInfoFromBinary((char*)pCtx->pInput + sizeof(int32_t), len, pDist); - pDist->rowSize = (int16_t) pCtx->param[0].i64; + pDist->rowSize = (uint16_t)pCtx->param[0].i64; memcpy(pCtx->pOutput, pCtx->pInput, sizeof(int32_t) + len); @@ -4149,7 +4149,7 @@ void blockinfo_func_finalizer(SQLFunctionCtx* pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo); - pDist->rowSize = (int16_t)pCtx->param[0].i64; + pDist->rowSize = (uint16_t)pCtx->param[0].i64; generateBlockDistResult(pDist, pCtx->pOutput); // cannot set the numOfIteratedElems again since it is set during previous iteration diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index b566f2095b..5700b87d5e 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -920,7 +920,6 @@ int tsdbWriteBlockImpl(STsdbRepo *pRepo, STable *pTable, SDFile *pDFile, SDataCo SDataCol * pDataCol = pDataCols->cols + ncol; SBlockCol *pBlockCol = pBlockData->cols + nColsNotAllNull; - // if (isNEleNull(pDataCol, rowsToWrite)) { // all data to commit are NULL, just ignore it if (isAllRowOfColNull(pDataCol)) { // all data to commit are NULL, just ignore it continue; } diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 41e28ac909..4314514105 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -1021,10 +1021,9 @@ static void updateTableLatestColumn(STsdbRepo *pRepo, STable *pTable, SMemRow ro } else { // SKVRow SColIdx *pColIdx = tdGetKVRowIdxOfCol(rowBody, pTCol->colId); - if(pColIdx) { -value = tdGetKvRowDataOfCol(rowBody, pColIdx->offset); + if (pColIdx) { + value = tdGetKvRowDataOfCol(rowBody, pColIdx->offset); } - } if ((value == NULL) || isNull(value, pTCol->type)) { diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 395598f527..195e9f8584 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -145,7 +145,7 @@ typedef struct STableGroupSupporter { static STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList); static int32_t checkForCachedLastRow(STsdbQueryHandle* pQueryHandle, STableGroupInfo *groupList); static int32_t checkForCachedLast(STsdbQueryHandle* pQueryHandle); -static int32_t tsdbGetCachedLastRow(STable* pTable, SMemRow* pRes, TSKEY* lastKey); +static int32_t tsdbGetCachedLastRow(STable* pTable, SMemRow* pRes, TSKEY* lastKey); static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle); static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SBlock* pBlock); From c28ac1665ee51b6d8394f64d94a1df0342b45c12 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 1 Jul 2021 18:14:52 +0800 Subject: [PATCH 11/67] fix compile issue on Windows --- src/client/src/tscUtil.c | 29 +-------------------------- src/common/inc/tdataformat.h | 38 +++--------------------------------- src/common/src/tdataformat.c | 6 ++---- src/inc/ttype.h | 2 +- 4 files changed, 7 insertions(+), 68 deletions(-) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 64ae79f73d..36d4ef7096 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1640,33 +1640,6 @@ int32_t tscGetDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, i return TSDB_CODE_SUCCESS; } -// int tdInitMemRowBuilder(SMemRowBuilder* pBuilder) { -// pBuilder->pSchema = NULL; -// pBuilder->sversion = 0; -// pBuilder->tCols = 128; -// pBuilder->nCols = 0; -// pBuilder->pColIdx = (SColIdx*)malloc(sizeof(SColIdx) * pBuilder->tCols); -// if (pBuilder->pColIdx == NULL) return -1; -// pBuilder->alloc = 1024; -// pBuilder->size = 0; -// pBuilder->buf = malloc(pBuilder->alloc); -// if (pBuilder->buf == NULL) { -// free(pBuilder->pColIdx); -// return -1; -// } -// return 0; -// } - -// void tdDestroyMemRowBuilder(SMemRowBuilder* pBuilder) { -// tfree(pBuilder->pColIdx); -// tfree(pBuilder->buf); -// } - -// void tdResetMemRowBuilder(SMemRowBuilder* pBuilder) { -// pBuilder->nCols = 0; -// pBuilder->size = 0; -// } - #define KvRowNColsThresh 1 // default value: 1200 TODO: for test, restore to default value after test finished static FORCE_INLINE uint8_t tdRowTypeJudger(SSchema* pSchema, void* pData, int32_t nCols, int32_t flen, @@ -1700,7 +1673,7 @@ static FORCE_INLINE uint8_t tdRowTypeJudger(SSchema* pSchema, void* pData, int32 p += pSchema[i].bytes; } - tscDebug("prop:nColsNull %d, nCols: %d, kvRowLen: %d, dataRowLen: %d", (int32_t)nColsNull, nCols, kvRowLength, + tscDebug("nColsNull %d, nCols: %d, kvRowLen: %d, dataRowLen: %d", (int32_t)nColsNull, nCols, kvRowLength, dataRowLength); if (kvRowLength < dataRowLength) { diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index 4ab44a7918..3901f3cec0 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -255,13 +255,12 @@ static FORCE_INLINE int tdAppendColVal(SDataRow row, const void *value, int8_t t } // NOTE: offset here including the header size -static FORCE_INLINE void *tdGetRowDataOfCol(SDataRow *row, int8_t type, int32_t offset) { +static FORCE_INLINE void *tdGetRowDataOfCol(SDataRow row, int8_t type, int32_t offset) { if (IS_VAR_DATA_TYPE(type)) { return POINTER_SHIFT(row, *(VarDataOffsetT *)POINTER_SHIFT(row, offset)); } else { return POINTER_SHIFT(row, offset); } - return NULL; } // ----------------- Data column structure @@ -281,42 +280,12 @@ typedef struct SDataCol { static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; } void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints); +void dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints); void dataColSetOffset(SDataCol *pCol, int nEle); bool isNEleNull(SDataCol *pCol, int nEle); void dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints); -FORCE_INLINE void dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints); -// value from timestamp should be TKEY here instead of TSKEY -FORCE_INLINE void dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints) { - ASSERT(pCol != NULL && value != NULL); - - if (pCol->len == 0) { - if (isNull(value, pCol->type)) { - // all null value yet, just return - return; - } - - if (numOfRows > 0) { - // Find the first not null value, fill all previous values as NULL - dataColSetNEleNull(pCol, numOfRows, maxPoints); - } - } - - if (IS_VAR_DATA_TYPE(pCol->type)) { - // set offset - pCol->dataOff[numOfRows] = pCol->len; - // Copy data - memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, varDataTLen(value)); - // Update the length - pCol->len += varDataTLen(value); - } else { - ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfRows); - memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, pCol->bytes); - pCol->len += pCol->bytes; - } -} - static FORCE_INLINE const void *tdGetNullVal(int8_t type) { switch (type) { case TSDB_DATA_TYPE_BOOL: @@ -456,7 +425,7 @@ typedef struct { #define kvRowNCols(r) (*(int16_t *)POINTER_SHIFT(r, sizeof(uint16_t))) #define kvRowSetLen(r, len) kvRowLen(r) = (len) #define kvRowSetNCols(r, n) kvRowNCols(r) = (n) -#define kvRowColIdx(r) ((SColIdx *)POINTER_SHIFT(r, TD_KV_ROW_HEAD_SIZE)) +#define kvRowColIdx(r) (SColIdx *)POINTER_SHIFT(r, TD_KV_ROW_HEAD_SIZE) #define kvRowValues(r) POINTER_SHIFT(r, TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * kvRowNCols(r)) #define kvRowCpy(dst, r) memcpy((dst), (r), kvRowLen(r)) #define kvRowColVal(r, colIdx) POINTER_SHIFT(kvRowValues(r), (colIdx)->offset) @@ -464,7 +433,6 @@ typedef struct { #define kvRowFree(r) tfree(r) #define kvRowEnd(r) POINTER_SHIFT(r, kvRowLen(r)) #define kvRowVersion(r) (-1) - #define kvRowTKey(r) (*(TKEY *)(kvRowValues(r))) #define kvRowKey(r) tdGetKey(kvRowTKey(r)) #define kvRowDeleted(r) TKEY_IS_DELETED(kvRowTKey(r)) diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index d346b5c463..5d1052d4fd 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -225,7 +225,7 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints) pDataCol->type = colType(pCol); pDataCol->colId = colColId(pCol); pDataCol->bytes = colBytes(pCol); - pDataCol->offset = colOffset(pCol) + TD_MEM_ROW_HEAD_SIZE; + pDataCol->offset = colOffset(pCol) + TD_DATA_ROW_HEAD_SIZE; pDataCol->len = 0; if (IS_VAR_DATA_TYPE(pDataCol->type)) { @@ -240,7 +240,6 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints) *pBuf = POINTER_SHIFT(*pBuf, pDataCol->spaceSize); } } -#if 0 // value from timestamp should be TKEY here instead of TSKEY void dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints) { ASSERT(pCol != NULL && value != NULL); @@ -270,7 +269,6 @@ void dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxP pCol->len += pCol->bytes; } } -#endif bool isNEleNull(SDataCol *pCol, int nEle) { for (int i = 0; i < nEle; i++) { @@ -648,7 +646,7 @@ int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value) { void * ptr = taosbsearch(&colId, kvRowColIdx(row), kvRowNCols(row), sizeof(SColIdx), comparTagId, TD_GE); if (ptr == NULL || ((SColIdx *)ptr)->colId > colId) { // need to add a column value to the row - uint16_t diff = IS_VAR_DATA_TYPE(type) ? varDataTLen(value) : TYPE_BYTES[type]; + int diff = IS_VAR_DATA_TYPE(type) ? varDataTLen(value) : TYPE_BYTES[type]; nrow = malloc(kvRowLen(row) + sizeof(SColIdx) + diff); if (nrow == NULL) return -1; diff --git a/src/inc/ttype.h b/src/inc/ttype.h index b858d2a7f5..80b8ddcd4e 100644 --- a/src/inc/ttype.h +++ b/src/inc/ttype.h @@ -121,7 +121,7 @@ typedef struct tstr { #define IS_VALID_UINT(_t) ((_t) >= 0 && (_t) < UINT32_MAX) #define IS_VALID_UBIGINT(_t) ((_t) >= 0 && (_t) < UINT64_MAX) -FORCE_INLINE bool isNull(const char *val, int32_t type) { +static FORCE_INLINE bool isNull(const char *val, int32_t type) { switch (type) { case TSDB_DATA_TYPE_BOOL: return *(uint8_t *)val == TSDB_DATA_BOOL_NULL; From c02c5d8bfd2783edbfe286f310f58409be4c7c08 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 1 Jul 2021 18:47:20 +0800 Subject: [PATCH 12/67] tiny change to trigger PR --- src/common/src/tdataformat.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index 5d1052d4fd..6061b84209 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -780,7 +780,7 @@ void tdResetKVRowBuilder(SKVRowBuilder *pBuilder) { } SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder) { - uint16_t tlen = sizeof(SColIdx) * pBuilder->nCols + pBuilder->size; + int tlen = sizeof(SColIdx) * pBuilder->nCols + pBuilder->size; if (tlen == 0) return NULL; tlen += TD_KV_ROW_HEAD_SIZE; From dd8aaaf366a6cc670e1e1f4cc1981708e6e09eb2 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Fri, 2 Jul 2021 13:04:32 +0800 Subject: [PATCH 13/67] bug fix for reading SDataRow VarT Values --- src/client/inc/tscUtil.h | 1 - src/client/src/tscUtil.c | 7 ++++--- src/tsdb/src/tsdbRead.c | 3 ++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 2291feb08a..6515c7c4e5 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -343,7 +343,6 @@ SNewVgroupInfo createNewVgroupInfo(SVgroupMsg *pVgroupMsg); typedef struct { // for SDataRow - STSchema* pTSchema; SSchema* pSchema; int16_t sversion; int32_t flen; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 36d4ef7096..7e4a656622 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1674,7 +1674,7 @@ static FORCE_INLINE uint8_t tdRowTypeJudger(SSchema* pSchema, void* pData, int32 } tscDebug("nColsNull %d, nCols: %d, kvRowLen: %d, dataRowLen: %d", (int32_t)nColsNull, nCols, kvRowLength, - dataRowLength); + dataRowLength); if (kvRowLength < dataRowLength) { if (nColsNotNull) { @@ -1696,7 +1696,7 @@ SMemRow tdGenMemRowFromBuilder(SMemRowBuilder* pBuilder) { } uint16_t nColsNotNull = 0; - uint8_t memRowType = tdRowTypeJudger(pSchema, p, pBuilder->nCols, pBuilder->flen, &nColsNotNull); + uint8_t memRowType = tdRowTypeJudger(pSchema, p, pBuilder->nCols, pBuilder->flen, &nColsNotNull); SMemRow* memRow = (SMemRow)pBuilder->pDataBlock; memRowSetType(memRow, memRowType); @@ -1714,7 +1714,7 @@ SMemRow tdGenMemRowFromBuilder(SMemRowBuilder* pBuilder) { } pBuilder->buf = p; } else if (memRowType == SMEM_ROW_KV) { - ASSERT(nColsNotNull < pBuilder->nCols); + ASSERT(nColsNotNull <= pBuilder->nCols); SKVRow kvRow = (SKVRow)memRowBody(memRow); uint16_t tlen = TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * nColsNotNull; kvRowSetLen(kvRow, tlen); @@ -1810,6 +1810,7 @@ static int32_t getRowExpandSize(STableMeta* pTableMeta) { if (IS_VAR_DATA_TYPE((pSchema + i)->type)) { result += TYPE_BYTES[TSDB_DATA_TYPE_BINARY]; } + result += sizeof(SColIdx); } result += TD_MEM_ROW_TYPE_SIZE; // add len of SMemRow flag return result; diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 195e9f8584..7f344dc646 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -1459,6 +1459,7 @@ static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t i = 0; if (isDataRow(row)) { + SDataRow dataRow = memRowBody(row); int32_t j = 0; while (i < numOfCols && j < numOfRowCols) { SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); @@ -1475,7 +1476,7 @@ static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, if (pSchema->columns[j].colId == pColInfo->info.colId) { void* value = - tdGetRowDataOfCol(row, (int8_t)pColInfo->info.type, TD_MEM_ROW_HEAD_SIZE + pSchema->columns[j].offset); + tdGetRowDataOfCol(dataRow, (int8_t)pColInfo->info.type, TD_DATA_ROW_HEAD_SIZE + pSchema->columns[j].offset); switch (pColInfo->info.type) { case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: From e94bb935089988dea49389d53d3d9a0335a6b7d9 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sat, 3 Jul 2021 14:59:04 +0800 Subject: [PATCH 14/67] add sversion for KV type of SMemRow --- src/client/inc/tscUtil.h | 2 + src/client/src/tscUtil.c | 19 +++-- src/common/inc/tdataformat.h | 131 +++++++++++------------------------ src/common/src/tdataformat.c | 41 ++--------- src/cq/src/cqMain.c | 8 +-- src/inc/taosdef.h | 4 +- src/inc/ttype.h | 1 + src/tsdb/src/tsdbCommit.c | 20 +++--- src/tsdb/src/tsdbMain.c | 12 ++-- src/tsdb/src/tsdbMemTable.c | 20 +++--- src/tsdb/src/tsdbRead.c | 6 +- 11 files changed, 93 insertions(+), 171 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 6515c7c4e5..1a9fb4c665 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -40,6 +40,8 @@ extern "C" { #define UTIL_TABLE_IS_TMP_TABLE(metaInfo) \ (((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_TEMP_TABLE)) +#define KvRowNColsThresh 1 // default 1200. TODO: only for test, restore to default value after test finished + #pragma pack(push,1) // this struct is transfered as binary, padding two bytes to avoid // an 'uid' whose low bytes is 0xff being recoginized as NULL, diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 7e4a656622..fb09ceb5cb 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1639,17 +1639,14 @@ int32_t tscGetDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, i return TSDB_CODE_SUCCESS; } - -#define KvRowNColsThresh 1 // default value: 1200 TODO: for test, restore to default value after test finished - -static FORCE_INLINE uint8_t tdRowTypeJudger(SSchema* pSchema, void* pData, int32_t nCols, int32_t flen, - uint16_t* nColsNotNull) { +static FORCE_INLINE uint8_t checkTdRowType(SSchema* pSchema, void* pData, int32_t nCols, int32_t flen, + uint16_t* nColsNotNull) { ASSERT(pData != NULL); if (nCols < KvRowNColsThresh) { return SMEM_ROW_DATA; } int32_t dataRowLength = flen; - int32_t kvRowLength = 0; + int32_t kvRowLength = TD_MEM_ROW_KV_VER_SIZE; uint16_t nColsNull = 0; char* p = (char*)pData; @@ -1685,7 +1682,6 @@ static FORCE_INLINE uint8_t tdRowTypeJudger(SSchema* pSchema, void* pData, int32 return SMEM_ROW_DATA; } - SMemRow tdGenMemRowFromBuilder(SMemRowBuilder* pBuilder) { SSchema* pSchema = pBuilder->pSchema; char* p = (char*)pBuilder->buf; @@ -1696,13 +1692,13 @@ SMemRow tdGenMemRowFromBuilder(SMemRowBuilder* pBuilder) { } uint16_t nColsNotNull = 0; - uint8_t memRowType = tdRowTypeJudger(pSchema, p, pBuilder->nCols, pBuilder->flen, &nColsNotNull); - + uint8_t memRowType = checkTdRowType(pSchema, p, pBuilder->nCols, pBuilder->flen, &nColsNotNull); + // nColsNotNull = pBuilder->nCols; SMemRow* memRow = (SMemRow)pBuilder->pDataBlock; memRowSetType(memRow, memRowType); if (memRowType == SMEM_ROW_DATA) { - SDataRow trow = (SDataRow)memRowBody(memRow); + SDataRow trow = (SDataRow)memRowDataBody(memRow); dataRowSetLen(trow, (uint16_t)(TD_DATA_ROW_HEAD_SIZE + pBuilder->flen)); dataRowSetVersion(trow, pBuilder->sversion); @@ -1715,10 +1711,11 @@ SMemRow tdGenMemRowFromBuilder(SMemRowBuilder* pBuilder) { pBuilder->buf = p; } else if (memRowType == SMEM_ROW_KV) { ASSERT(nColsNotNull <= pBuilder->nCols); - SKVRow kvRow = (SKVRow)memRowBody(memRow); + SKVRow kvRow = (SKVRow)memRowKvBody(memRow); uint16_t tlen = TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * nColsNotNull; kvRowSetLen(kvRow, tlen); kvRowSetNCols(kvRow, nColsNotNull); + memRowKvSetVersion(memRow, pBuilder->sversion); p = (char*)pBuilder->buf; for (int32_t j = 0; j < pBuilder->nCols; ++j) { diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index 3901f3cec0..b0402d4674 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -24,31 +24,6 @@ extern "C" { #endif -typedef struct { - VarDataLenT len; - uint8_t data; -} SBinaryNullT; - -typedef struct { - VarDataLenT len; - uint32_t data; -} SNCharNullT; - -extern const uint8_t BoolNull; -extern const uint8_t TinyintNull; -extern const uint16_t SmallintNull; -extern const uint32_t IntNull; -extern const uint64_t BigintNull; -extern const uint64_t TimestampNull; -extern const uint8_t UTinyintNull; -extern const uint16_t USmallintNull; -extern const uint32_t UIntNull; -extern const uint64_t UBigintNull; -extern const uint32_t FloatNull; -extern const uint64_t DoubleNull; -extern const SBinaryNullT BinaryNull; -extern const SNCharNullT NcharNull; - #define STR_TO_VARSTR(x, str) \ do { \ VarDataLenT __len = (VarDataLenT)strlen(str); \ @@ -215,7 +190,7 @@ typedef void *SDataRow; #define TD_DATA_ROW_HEAD_SIZE (sizeof(uint16_t) + sizeof(int16_t)) #define dataRowLen(r) (*(uint16_t *)(r)) -#define dataRowVersion(r) *(int16_t *)POINTER_SHIFT(r, sizeof(int16_t)) +#define dataRowVersion(r) (*(int16_t *)POINTER_SHIFT(r, sizeof(int16_t))) #define dataRowTuple(r) POINTER_SHIFT(r, TD_DATA_ROW_HEAD_SIZE) #define dataRowTKey(r) (*(TKEY *)(dataRowTuple(r))) #define dataRowKey(r) tdGetKey(dataRowTKey(r)) @@ -232,7 +207,7 @@ SDataRow tdDataRowDup(SDataRow row); SMemRow tdMemRowDup(SMemRow row); // offset here not include dataRow header length -static FORCE_INLINE int tdAppendColVal(SDataRow row, const void *value, int8_t type, int32_t bytes, int32_t offset) { +static FORCE_INLINE int tdAppendColVal(SDataRow row, void *value, int8_t type, int32_t bytes, int32_t offset) { ASSERT(value != NULL); int32_t toffset = offset + TD_DATA_ROW_HEAD_SIZE; char * ptr = (char *)POINTER_SHIFT(row, dataRowLen(row)); @@ -276,56 +251,17 @@ typedef struct SDataCol { TSKEY ts; // only used in last NULL column } SDataCol; -#define isAllRowOfColNull(pCol) ((pCol)->len == 0) static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; } void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints); -void dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints); +void dataColAppendVal(SDataCol *pCol, void *value, int numOfRows, int maxPoints); void dataColSetOffset(SDataCol *pCol, int nEle); bool isNEleNull(SDataCol *pCol, int nEle); void dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints); -static FORCE_INLINE const void *tdGetNullVal(int8_t type) { - switch (type) { - case TSDB_DATA_TYPE_BOOL: - return &BoolNull; - case TSDB_DATA_TYPE_TINYINT: - return &TinyintNull; - case TSDB_DATA_TYPE_SMALLINT: - return &SmallintNull; - case TSDB_DATA_TYPE_INT: - return &IntNull; - case TSDB_DATA_TYPE_BIGINT: - return &BigintNull; - case TSDB_DATA_TYPE_FLOAT: - return &FloatNull; - case TSDB_DATA_TYPE_DOUBLE: - return &DoubleNull; - case TSDB_DATA_TYPE_BINARY: - return &BinaryNull; - case TSDB_DATA_TYPE_TIMESTAMP: - return &TimestampNull; - case TSDB_DATA_TYPE_NCHAR: - return &NcharNull; - case TSDB_DATA_TYPE_UTINYINT: - return &UTinyintNull; - case TSDB_DATA_TYPE_USMALLINT: - return &USmallintNull; - case TSDB_DATA_TYPE_UINT: - return &UIntNull; - case TSDB_DATA_TYPE_UBIGINT: - return &UBigintNull; - default: - ASSERT(0); - return NULL; - } -} // Get the data pointer from a column-wised data -static FORCE_INLINE const void *tdGetColDataOfRow(SDataCol *pCol, int row) { - if (isAllRowOfColNull(pCol)) { - return tdGetNullVal(pCol->type); - } +static FORCE_INLINE void *tdGetColDataOfRow(SDataCol *pCol, int row) { if (IS_VAR_DATA_TYPE(pCol->type)) { return POINTER_SHIFT(pCol->pData, pCol->dataOff[row]); } else { @@ -432,7 +368,6 @@ typedef struct { #define kvRowColIdxAt(r, i) (kvRowColIdx(r) + (i)) #define kvRowFree(r) tfree(r) #define kvRowEnd(r) POINTER_SHIFT(r, kvRowLen(r)) -#define kvRowVersion(r) (-1) #define kvRowTKey(r) (*(TKEY *)(kvRowValues(r))) #define kvRowKey(r) tdGetKey(kvRowTKey(r)) #define kvRowDeleted(r) TKEY_IS_DELETED(kvRowTKey(r)) @@ -532,7 +467,7 @@ static FORCE_INLINE int tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t colId, return 0; } -// ----------------- Sequential Data row structure +// ----------------- SMemRow appended with sequential data row structure /* * |-------------------------------+--------------------------- len ---------------------------------->| * |<-------- Head ------>|<--------- flen -------------->| | @@ -545,43 +480,55 @@ static FORCE_INLINE int tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t colId, * NOTE: timestamp in this row structure is TKEY instead of TSKEY */ -// ----------------- K-V data row structure -/* - * |--------------------+----------+---------------------------------+---------------------------------+ - * | uint8_t | uint16_t | int16_t | | | - * |---------+----------+----------+---------------------------------+---------------------------------+ - * | flag | len | ncols | cols index | data part | - * |---------+----------+----------+---------------------------------+---------------------------------+ +// ----------------- SMemRow appended with extended K-V data row structure +/* | + * |--------------------+----------+--------------------------------------------+---------------------------------+ + * | uint8_t | int16_t | uint16_t | int16_t | | | + * |---------+----------+----------+----------+---------------------------------+---------------------------------+ + * | flag | sversion | len | ncols | cols index | data part | + * |---------+----------+----------+----------+---------------------------------+---------------------------------+ */ #define TD_MEM_ROW_TYPE_SIZE sizeof(uint8_t) -#define TD_MEM_ROW_HEAD_SIZE (TD_MEM_ROW_TYPE_SIZE + sizeof(uint16_t) + sizeof(int16_t)) +#define TD_MEM_ROW_KV_VER_SIZE sizeof(int16_t) +#define TD_MEM_ROW_KV_TYPE_VER_SIZE (TD_MEM_ROW_TYPE_SIZE + TD_MEM_ROW_KV_VER_SIZE) +#define TD_MEM_ROW_DATA_HEAD_SIZE (TD_MEM_ROW_TYPE_SIZE + TD_DATA_ROW_HEAD_SIZE) +#define TD_MEM_ROW_KV_HEAD_SIZE (TD_MEM_ROW_TYPE_SIZE + TD_MEM_ROW_KV_VER_SIZE + TD_KV_ROW_HEAD_SIZE) #define SMEM_ROW_DATA 0U // SDataRow #define SMEM_ROW_KV 1U // SKVRow -#define TD_DO_NOTHING \ - do { \ - } while (0) #define memRowType(r) (*(uint8_t *)(r)) -#define memRowBody(r) POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE) -#define memRowLen(r) (*(uint16_t *)POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE)) -#define memRowTLen(r) (*(uint16_t *)POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE) + (uint16_t)TD_MEM_ROW_TYPE_SIZE) - #define isDataRow(r) (SMEM_ROW_DATA == memRowType(r)) #define isKvRow(r) (SMEM_ROW_KV == memRowType(r)) -#define memRowVersion(r) (isDataRow(r) ? dataRowVersion(memRowBody(r)) : kvRowVersion(r)) // schema version -#define memRowTuple(r) (isDataRow(r) ? dataRowTuple(memRowBody(r)) : kvRowValues(memRowBody(r))) +#define memRowDataBody(r) POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE) // section after flag +#define memRowKvBody(r) \ + POINTER_SHIFT(r, TD_MEM_ROW_KV_TYPE_VER_SIZE) // section after flag + sversion as to reuse of SKVRow +// #define memRowBody(r) (isDataRow(r) ? memRowDataBody(r) : memRowKvBody(r)) -#define memRowTKey(r) (isDataRow(r) ? dataRowTKey(memRowBody(r)) : kvRowTKey(memRowBody(r))) -#define memRowKey(r) (isDataRow(r) ? dataRowKey(memRowBody(r)) : kvRowKey(memRowBody(r))) +#define memRowDataLen(r) (*(TDRowLenT *)memRowDataBody(r)) +#define memRowKvLen(r) (*(TDRowLenT *)memRowKvBody(r)) +#define memRowDataTLen(r) (memRowDataLen(r) + (TDRowLenT)TD_MEM_ROW_TYPE_SIZE) +#define memRowKvTLen(r) (memRowKvLen(r) + (TDRowLenT)TD_MEM_ROW_KV_TYPE_VER_SIZE) + +#define memRowLen(r) (isDataRow(r) ? memRowDataLen(r) : memRowKvLen(r)) +#define memRowTLen(r) (isDataRow(r) ? memRowDataTLen(r) : memRowKvTLen(r)) + +#define memRowDataVersion(r) dataRowVersion(memRowDataBody(r)) +#define memRowKvVersion(r) (*(int16_t *)POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE)) +#define memRowVersion(r) (isDataRow(r) ? memRowDataVersion(r) : memRowKvVersion(r)) // schema version +#define memRowKvSetVersion(r, v) (memRowKvVersion(r) = (v)) +#define memRowTuple(r) (isDataRow(r) ? dataRowTuple(memRowDataBody(r)) : kvRowValues(memRowKvBody(r))) + +#define memRowTKey(r) (isDataRow(r) ? dataRowTKey(memRowDataBody(r)) : kvRowTKey(memRowKvBody(r))) +#define memRowKey(r) (isDataRow(r) ? dataRowKey(memRowDataBody(r)) : kvRowKey(memRowKvBody(r))) #define memRowSetType(r, t) (memRowType(r) = (t)) -#define memRowSetLen(r, l) (memRowLen(r) = (l)) -#define memRowSetVersion(r, v) (isDataRow(r) ? dataRowSetVersion(r, v) : TD_DO_NOTHING) +#define memRowSetLen(r, l) (isDataRow(r) ? memRowDataLen(r) = (l) : memRowKvLen(r) = (l)) +#define memRowSetVersion(r, v) (isDataRow(r) ? dataRowSetVersion(memRowDataBody(r), v) : memRowKvSetVersion(r, v)) #define memRowCpy(dst, r) memcpy((dst), (r), memRowTLen(r)) -#define memRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_MEM_ROW_HEAD_SIZE) +#define memRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_MEM_ROW_DATA_HEAD_SIZE) #define memRowDeleted(r) TKEY_IS_DELETED(memRowTKey(r)) // NOTE: offset here including the header size diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index 6061b84209..a46d2e84b0 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -18,21 +18,6 @@ #include "tcoding.h" #include "wchar.h" -const uint8_t BoolNull = TSDB_DATA_BOOL_NULL; -const uint8_t TinyintNull = TSDB_DATA_TINYINT_NULL; -const uint16_t SmallintNull = TSDB_DATA_SMALLINT_NULL; -const uint32_t IntNull = TSDB_DATA_INT_NULL; -const uint64_t BigintNull = TSDB_DATA_BIGINT_NULL; -const uint64_t TimestampNull = TSDB_DATA_BIGINT_NULL; -const uint8_t UTinyintNull = TSDB_DATA_UTINYINT_NULL; -const uint16_t USmallintNull = TSDB_DATA_USMALLINT_NULL; -const uint32_t UIntNull = TSDB_DATA_UINT_NULL; -const uint64_t UBigintNull = TSDB_DATA_UBIGINT_NULL; -const uint32_t FloatNull = TSDB_DATA_FLOAT_NULL; -const uint64_t DoubleNull = TSDB_DATA_DOUBLE_NULL; -const SBinaryNullT BinaryNull = {1, TSDB_DATA_BINARY_NULL}; -const SNCharNullT NcharNull = {4, TSDB_DATA_NCHAR_NULL}; - static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, int limit2, int tRows); @@ -241,21 +226,9 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints) } } // value from timestamp should be TKEY here instead of TSKEY -void dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints) { +void dataColAppendVal(SDataCol *pCol, void *value, int numOfRows, int maxPoints) { ASSERT(pCol != NULL && value != NULL); - if (pCol->len == 0) { - if (isNull(value, pCol->type)) { - // all null value yet, just return - return; - } - - if (numOfRows > 0) { - // Find the first not null value, fill all previous values as NULL - dataColSetNEleNull(pCol, numOfRows, maxPoints); - } - } - if (IS_VAR_DATA_TYPE(pCol->type)) { // set offset pCol->dataOff[numOfRows] = pCol->len; @@ -452,7 +425,7 @@ static void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols while (dcol < pCols->numOfCols) { SDataCol *pDataCol = &(pCols->cols[dcol]); if (rcol >= schemaNCols(pSchema)) { - dataColAppendVal(pDataCol, tdGetNullVal(pDataCol->type), pCols->numOfRows, pCols->maxPoints); + dataColSetNullAt(pDataCol, pCols->numOfRows); dcol++; continue; } @@ -466,7 +439,7 @@ static void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols } else if (pRowCol->colId < pDataCol->colId) { rcol++; } else { - dataColAppendVal(pDataCol, tdGetNullVal(pDataCol->type), pCols->numOfRows, pCols->maxPoints); + dataColSetNullAt(pDataCol, pCols->numOfRows); dcol++; } } @@ -495,7 +468,7 @@ static void tdAppendKvRowToDataCol(SKVRow row, STSchema *pSchema, SDataCols *pCo while (dcol < pCols->numOfCols) { SDataCol *pDataCol = &(pCols->cols[dcol]); if (rcol >= nRowCols || rcol >= schemaNCols(pSchema)) { - dataColAppendVal(pDataCol, tdGetNullVal(pDataCol->type), pCols->numOfRows, pCols->maxPoints); + dataColSetNullAt(pDataCol, pCols->numOfRows); ++dcol; continue; } @@ -510,7 +483,7 @@ static void tdAppendKvRowToDataCol(SKVRow row, STSchema *pSchema, SDataCols *pCo } else if (colIdx->colId < pDataCol->colId) { ++rcol; } else { - dataColAppendVal(pDataCol, tdGetNullVal(pDataCol->type), pCols->numOfRows, pCols->maxPoints); + dataColSetNullAt(pDataCol, pCols->numOfRows); ++dcol; } } @@ -520,9 +493,9 @@ static void tdAppendKvRowToDataCol(SKVRow row, STSchema *pSchema, SDataCols *pCo void tdAppendMemRowToDataCol(SMemRow row, STSchema *pSchema, SDataCols *pCols) { if (isDataRow(row)) { - tdAppendDataRowToDataCol(memRowBody(row), pSchema, pCols); + tdAppendDataRowToDataCol(memRowDataBody(row), pSchema, pCols); } else if (isKvRow(row)) { - tdAppendKvRowToDataCol(memRowBody(row), pSchema, pCols); + tdAppendKvRowToDataCol(memRowKvBody(row), pSchema, pCols); } else { ASSERT(0); } diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c index 6fe530dce5..ee1022ea0c 100644 --- a/src/cq/src/cqMain.c +++ b/src/cq/src/cqMain.c @@ -476,7 +476,7 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) { cDebug("vgId:%d, id:%d CQ:%s stream result is ready", pContext->vgId, pObj->tid, pObj->sqlStr); - int32_t size = sizeof(SWalHead) + sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + TD_MEM_ROW_HEAD_SIZE + pObj->rowSize; + int32_t size = sizeof(SWalHead) + sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + TD_MEM_ROW_DATA_HEAD_SIZE + pObj->rowSize; char *buffer = calloc(size, 1); SWalHead *pHead = (SWalHead *)buffer; @@ -484,7 +484,7 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) { SSubmitBlk *pBlk = (SSubmitBlk *) (buffer + sizeof(SWalHead) + sizeof(SSubmitMsg)); SMemRow trow = (SMemRow)pBlk->data; - SDataRow dataRow = (SDataRow)memRowBody(trow); + SDataRow dataRow = (SDataRow)memRowDataBody(trow); memRowSetType(trow, SMEM_ROW_DATA); tdInitDataRow(dataRow, pSchema); @@ -504,7 +504,7 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) { } tdAppendColVal(dataRow, val, c->type, c->bytes, c->offset); } - pBlk->dataLen = htonl(memRowTLen(trow)); + pBlk->dataLen = htonl(memRowDataTLen(trow)); pBlk->schemaLen = 0; pBlk->uid = htobe64(pObj->uid); @@ -513,7 +513,7 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) { pBlk->sversion = htonl(pSchema->version); pBlk->padding = 0; - pHead->len = sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + memRowTLen(trow); + pHead->len = sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + memRowDataTLen(trow); pMsg->header.vgId = htonl(pContext->vgId); pMsg->header.contLen = htonl(pHead->len); diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 85183666fe..7b53fcc638 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -321,8 +321,8 @@ do { \ #define TSDB_MAX_JOIN_TABLE_NUM 10 #define TSDB_MAX_UNION_CLAUSE 5 -#define TSDB_MAX_BINARY_LEN (16384-TSDB_KEYSIZE) // TD-4666: TODO use TSDB_MAX_BYTES_PER_ROW later. -#define TSDB_MAX_NCHAR_LEN (16384-TSDB_KEYSIZE) // TD-4666: TODO use TSDB_MAX_BYTES_PER_ROW later. +#define TSDB_MAX_BINARY_LEN (16384-TSDB_KEYSIZE) // keep 16384 +#define TSDB_MAX_NCHAR_LEN (16384-TSDB_KEYSIZE) // keep 16384 #define PRIMARYKEY_TIMESTAMP_COL_INDEX 0 #define TSDB_MAX_RPC_THREADS 5 diff --git a/src/inc/ttype.h b/src/inc/ttype.h index 80b8ddcd4e..0900f45350 100644 --- a/src/inc/ttype.h +++ b/src/inc/ttype.h @@ -12,6 +12,7 @@ extern "C" { // ----------------- For variable data types such as TSDB_DATA_TYPE_BINARY and TSDB_DATA_TYPE_NCHAR typedef int32_t VarDataOffsetT; typedef int16_t VarDataLenT; // maxVarDataLen: 32767 +typedef uint16_t TDRowLenT; typedef struct tstr { VarDataLenT len; diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index 5700b87d5e..69d855e57c 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -920,21 +920,21 @@ int tsdbWriteBlockImpl(STsdbRepo *pRepo, STable *pTable, SDFile *pDFile, SDataCo SDataCol * pDataCol = pDataCols->cols + ncol; SBlockCol *pBlockCol = pBlockData->cols + nColsNotAllNull; - if (isAllRowOfColNull(pDataCol)) { // all data to commit are NULL, just ignore it + if (isNEleNull(pDataCol, rowsToWrite)) { // all data to commit are NULL, just ignore it continue; } - memset(pBlockCol, 0, sizeof(*pBlockCol)); + memset(pBlockCol, 0, sizeof(*pBlockCol)); - pBlockCol->colId = pDataCol->colId; - pBlockCol->type = pDataCol->type; - if (tDataTypes[pDataCol->type].statisFunc) { - (*tDataTypes[pDataCol->type].statisFunc)(pDataCol->pData, rowsToWrite, &(pBlockCol->min), &(pBlockCol->max), - &(pBlockCol->sum), &(pBlockCol->minIndex), &(pBlockCol->maxIndex), - &(pBlockCol->numOfNull)); + pBlockCol->colId = pDataCol->colId; + pBlockCol->type = pDataCol->type; + if (tDataTypes[pDataCol->type].statisFunc) { + (*tDataTypes[pDataCol->type].statisFunc)(pDataCol->pData, rowsToWrite, &(pBlockCol->min), &(pBlockCol->max), + &(pBlockCol->sum), &(pBlockCol->minIndex), &(pBlockCol->maxIndex), + &(pBlockCol->numOfNull)); + } + nColsNotAllNull++; } - nColsNotAllNull++; - } ASSERT(nColsNotAllNull >= 0 && nColsNotAllNull <= pDataCols->numOfCols); diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index b34308fd76..272a915d61 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -663,7 +663,7 @@ static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pRea } memRowSetType(row, SMEM_ROW_DATA); - tdInitDataRow(memRowBody(row), pSchema); + tdInitDataRow(memRowDataBody(row), pSchema); // first load block index info if (tsdbLoadBlockInfo(pReadh, NULL) < 0) { @@ -720,9 +720,9 @@ static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pRea // OK,let's load row from backward to get not-null column for (int32_t rowId = pBlock->numOfRows - 1; rowId >= 0; rowId--) { SDataCol *pDataCol = pReadh->pDCols[0]->cols + i; - tdAppendColVal(memRowBody(row), tdGetColDataOfRow(pDataCol, rowId), pCol->type, pCol->bytes, pCol->offset); + tdAppendColVal(memRowDataBody(row), tdGetColDataOfRow(pDataCol, rowId), pCol->type, pCol->bytes, pCol->offset); //SDataCol *pDataCol = readh.pDCols[0]->cols + j; - void *value = tdGetRowDataOfCol(memRowBody(row), (int8_t)pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset); + void *value = tdGetRowDataOfCol(memRowDataBody(row), (int8_t)pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset); if (isNull(value, pCol->type)) { continue; } @@ -742,7 +742,7 @@ static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pRea // save row ts(in column 0) pDataCol = pReadh->pDCols[0]->cols + 0; pCol = schemaColAt(pSchema, 0); - tdAppendColVal(memRowBody(row), tdGetColDataOfRow(pDataCol, rowId), pCol->type, pCol->bytes, pCol->offset); + tdAppendColVal(memRowDataBody(row), tdGetColDataOfRow(pDataCol, rowId), pCol->type, pCol->bytes, pCol->offset); pLastCol->ts = memRowKey(row); pTable->restoreColumnNum += 1; @@ -785,11 +785,11 @@ static int tsdbRestoreLastRow(STsdbRepo *pRepo, STable *pTable, SReadH* pReadh, return -1; } memRowSetType(pTable->lastRow, SMEM_ROW_DATA); - tdInitDataRow(memRowBody(pTable->lastRow), pSchema); + tdInitDataRow(memRowDataBody(pTable->lastRow), pSchema); for (int icol = 0; icol < schemaNCols(pSchema); icol++) { STColumn *pCol = schemaColAt(pSchema, icol); SDataCol *pDataCol = pReadh->pDCols[0]->cols + icol; - tdAppendColVal(memRowBody(pTable->lastRow), tdGetColDataOfRow(pDataCol, pBlock->numOfRows - 1), pCol->type, + tdAppendColVal(memRowDataBody(pTable->lastRow), tdGetColDataOfRow(pDataCol, pBlock->numOfRows - 1), pCol->type, pCol->bytes, pCol->offset); } diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 4314514105..ad097ec0f5 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -932,13 +932,15 @@ static int tsdbInsertDataToTableImpl(STsdbRepo *pRepo, STable *pTable, void **ro int64_t osize = SL_SIZE(pTableData->pData); tSkipListPutBatch(pTableData->pData, rows, rowCounter); int64_t dsize = SL_SIZE(pTableData->pData) - osize; + TSKEY keyFirstRow = memRowKey(rows[0]); + TSKEY keyLastRow = memRowKey(rows[rowCounter - 1]); - if (pMemTable->keyFirst > memRowKey(rows[0])) pMemTable->keyFirst = memRowKey(rows[0]); - if (pMemTable->keyLast < memRowKey(rows[rowCounter - 1])) pMemTable->keyLast = memRowKey(rows[rowCounter - 1]); + if (pMemTable->keyFirst > keyFirstRow) pMemTable->keyFirst = keyFirstRow; + if (pMemTable->keyLast < keyLastRow) pMemTable->keyLast = keyLastRow; pMemTable->numOfRows += dsize; - if (pTableData->keyFirst > memRowKey(rows[0])) pTableData->keyFirst = memRowKey(rows[0]); - if (pTableData->keyLast < memRowKey(rows[rowCounter - 1])) pTableData->keyLast = memRowKey(rows[rowCounter - 1]); + if (pTableData->keyFirst > keyFirstRow) pTableData->keyFirst = keyFirstRow; + if (pTableData->keyLast < keyLastRow) pTableData->keyLast = keyLastRow; pTableData->numOfRows += dsize; // update table latest info @@ -1004,8 +1006,7 @@ static void updateTableLatestColumn(STsdbRepo *pRepo, STable *pTable, SMemRow ro SDataCol *pLatestCols = pTable->lastCols; - bool isDataRow = isDataRow(row); - void *rowBody = memRowBody(row); + bool isDataRow = isDataRow(row); for (int16_t j = 0; j < schemaNCols(pSchema); j++) { STColumn *pTCol = schemaColAt(pSchema, j); // ignore not exist colId @@ -1017,12 +1018,13 @@ static void updateTableLatestColumn(STsdbRepo *pRepo, STable *pTable, SMemRow ro void *value = NULL; if (isDataRow) { - value = tdGetRowDataOfCol(rowBody, (int8_t)pTCol->type, TD_DATA_ROW_HEAD_SIZE + pSchema->columns[j].offset); + value = tdGetRowDataOfCol(memRowDataBody(row), (int8_t)pTCol->type, + TD_DATA_ROW_HEAD_SIZE + pSchema->columns[j].offset); } else { // SKVRow - SColIdx *pColIdx = tdGetKVRowIdxOfCol(rowBody, pTCol->colId); + SColIdx *pColIdx = tdGetKVRowIdxOfCol(memRowKvBody(row), pTCol->colId); if (pColIdx) { - value = tdGetKvRowDataOfCol(rowBody, pColIdx->offset); + value = tdGetKvRowDataOfCol(memRowKvBody(row), pColIdx->offset); } } diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 7f344dc646..2d3e7b9e8b 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -1392,7 +1392,7 @@ int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity // todo refactor, only copy one-by-one for (int32_t k = start; k < num + start; ++k) { - const char* p = tdGetColDataOfRow(src, k); + char* p = tdGetColDataOfRow(src, k); memcpy(dst, p, varDataTLen(p)); dst += bytes; } @@ -1459,7 +1459,7 @@ static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t i = 0; if (isDataRow(row)) { - SDataRow dataRow = memRowBody(row); + SDataRow dataRow = memRowDataBody(row); int32_t j = 0; while (i < numOfCols && j < numOfRowCols) { SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); @@ -1529,7 +1529,7 @@ static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, } } } else if (isKvRow(row)) { - SKVRow kvRow = memRowBody(row); + SKVRow kvRow = memRowKvBody(row); int32_t k = 0; int32_t nKvRowCols = kvRowNCols(kvRow); From 9ed5a42cf60b4eadb7fc7f8d6aca0a2c9ed6838a Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 7 Jul 2021 07:14:08 +0800 Subject: [PATCH 15/67] tsc raw data combination restructure --- src/client/inc/tscUtil.h | 31 +- src/client/inc/tsclient.h | 46 ++- src/client/src/tscParseInsert.c | 539 +++++++++++++++++++++++++++++--- src/client/src/tscUtil.c | 175 ++++++----- src/common/inc/tdataformat.h | 106 ++++++- src/common/src/tdataformat.c | 15 + src/cq/src/cqMain.c | 2 +- src/tsdb/src/tsdbCommit.c | 18 +- src/tsdb/src/tsdbMain.c | 6 +- src/tsdb/tests/tsdbTests.cpp | 4 +- 10 files changed, 774 insertions(+), 168 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 1a9fb4c665..e62661e6f6 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -40,7 +40,8 @@ extern "C" { #define UTIL_TABLE_IS_TMP_TABLE(metaInfo) \ (((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_TEMP_TABLE)) -#define KvRowNColsThresh 1 // default 1200. TODO: only for test, restore to default value after test finished +#define KvRowNColsThresh 1 // default 1200 +#define KVRowRatio 0.85 // for NonVarType, we get value from SDataRow directly, while needs readdressing for SKVRow #pragma pack(push,1) // this struct is transfered as binary, padding two bytes to avoid @@ -96,11 +97,21 @@ typedef struct SVgroupTableInfo { SArray *itemList; // SArray } SVgroupTableInfo; +typedef struct SBlockKeyTuple { + TSKEY skey; + void* payloadAddr; +} SBlockKeyTuple; + +typedef struct SBlockKeyInfo { + int32_t nBytesAlloc; + SBlockKeyTuple* pKeyTuple; +} SBlockKeyInfo; + int32_t converToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *len); int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOffset, SName* name, STableMeta* pTableMeta, STableDataBlocks** dataBlocks); void tscDestroyDataBlock(STableDataBlocks* pDataBlock, bool removeMeta); -void tscSortRemoveDataBlockDupRows(STableDataBlocks* dataBuf); +int tscSortRemoveDataBlockDupRows(STableDataBlocks* dataBuf, SBlockKeyInfo* pBlkKeyInfo); void tscDestroyBoundColumnInfo(SParsedDataColInfo* pColInfo); void doRetrieveSubqueryData(SSchedMsg *pMsg); @@ -343,22 +354,6 @@ char* strdup_throw(const char* str); bool vgroupInfoIdentical(SNewVgroupInfo *pExisted, SVgroupMsg* src); SNewVgroupInfo createNewVgroupInfo(SVgroupMsg *pVgroupMsg); -typedef struct { - // for SDataRow - SSchema* pSchema; - int16_t sversion; - int32_t flen; - // for SKVRow - uint16_t nCols; - uint16_t size; - void* buf; - - void* pDataBlock; - SSubmitBlk* pSubmitBlk; -} SMemRowBuilder; - -SMemRow tdGenMemRowFromBuilder(SMemRowBuilder* pBuilder); - #ifdef __cplusplus } #endif diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index c1243535b1..428cbf3391 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -95,6 +95,43 @@ typedef struct SParsedDataColInfo { SBoundColumn *cols; } SParsedDataColInfo; +typedef struct { + // for SDataRow + SSchema *pSchema; + int16_t sversion; + int32_t flen; + // for SKVRow + uint16_t nCols; + uint16_t size; + void * buf; + + void * pDataBlock; + SSubmitBlk *pSubmitBlk; + uint16_t allNullLen; +} SMemRowBuilder; + +int FORCE_INLINE initSMemRowBuilder(SMemRowBuilder *pBuilder, SSchema *pSSchema, uint16_t nCols, + uint16_t allNullColsLen) { + ASSERT(nCols > 0); + pBuilder->pSchema = pSSchema; + pBuilder->allNullLen = allNullColsLen; // TODO: get allNullColsLen when creating or altering table meta + if (pBuilder->allNullLen == 0) { + for (uint16_t i = 0; i < nCols; ++i) { + uint8_t type = pSSchema[i].type; + int32_t typeLen = TYPE_BYTES[type]; + ASSERT(typeLen > 0); + pBuilder->allNullLen += typeLen; + if (TSDB_DATA_TYPE_BINARY == type) { + pBuilder->allNullLen += (sizeof(VarDataLenT) + CHAR_BYTES); + } else if (TSDB_DATA_TYPE_NCHAR == type) { + int len = sizeof(VarDataLenT) + TSDB_NCHAR_SIZE; + pBuilder->allNullLen += len; + } + } + } + return 0; +} + typedef struct STableDataBlocks { SName tableName; int8_t tsSource; // where does the UNIX timestamp come from, server or client @@ -109,12 +146,13 @@ typedef struct STableDataBlocks { STableMeta *pTableMeta; // the tableMeta of current table, the table meta will be used during submit, keep a ref to avoid to be removed from cache char *pData; - SParsedDataColInfo boundColumnInfo; + SParsedDataColInfo boundColumnInfo; // for parameter ('?') binding - uint32_t numOfAllocedParams; - uint32_t numOfParams; - SParamInfo *params; + uint32_t numOfAllocedParams; + uint32_t numOfParams; + SParamInfo * params; + SMemRowBuilder rowBuilder; } STableDataBlocks; typedef struct { diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 137a7be7c7..e8da633217 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -38,10 +38,16 @@ enum { TSDB_USE_CLI_TS = 1, }; +static uint8_t TRUE_VALUE = (uint8_t)TSDB_TRUE; +static uint8_t FALSE_VALUE = (uint8_t)TSDB_FALSE; + static int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t *numOfRows); static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDataColInfo *pColInfo, SSchema *pSchema, char *str, char **end); +static FORCE_INLINE int32_t getPaddingRowSize(STableComInfo *tinfo) { + return tinfo->rowSize + PAYLOAD_HEADER_LEN + PAYLOAD_ID_TYPE_LEN * tinfo->numOfColumns; +} static int32_t tscToDouble(SStrToken *pToken, double *value, char **endPtr) { errno = 0; *value = strtold(pToken->z, endPtr); @@ -378,6 +384,349 @@ int32_t tsParseOneColumn(SSchema *pSchema, SStrToken *pToken, char *payload, cha return TSDB_CODE_SUCCESS; } +static FORCE_INLINE uint16_t tsSetColumnValue(char *payload, int16_t columnId, uint8_t columnType, void *value, + uint16_t valueLen) { + payloadColSetId(payload, columnId); + payloadColSetType(payload, columnType); + + memcpy(payloadColValue(payload), value, valueLen); + return PAYLOAD_ID_TYPE_LEN + valueLen; +} + +static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *payload, char *msg, char **str, + bool primaryKey, int16_t timePrec, uint16_t *sizeAppend, bool *isColNull, + TDRowLenT *dataRowColDeltaLen, TDRowLenT *kvRowColLen) { + int64_t iv; + int32_t ret; + char * endptr = NULL; + + if (IS_NUMERIC_TYPE(pSchema->type) && pToken->n == 0) { + return tscInvalidOperationMsg(msg, "invalid numeric data", pToken->z); + } + + switch (pSchema->type) { + case TSDB_DATA_TYPE_BOOL: { // bool + if (isNullStr(pToken)) { + // *((uint8_t *)payload) = TSDB_DATA_BOOL_NULL; + *isColNull = true; + } else { + if ((pToken->type == TK_BOOL || pToken->type == TK_STRING) && (pToken->n != 0)) { + if (strncmp(pToken->z, "true", pToken->n) == 0) { + // *(uint8_t *)payload = TSDB_TRUE; + *sizeAppend = + tsSetColumnValue(payload, pSchema->colId, pSchema->type, &TRUE_VALUE, TYPE_BYTES[TSDB_DATA_TYPE_BOOL]); + *kvRowColLen += sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_BOOL]; + } else if (strncmp(pToken->z, "false", pToken->n) == 0) { + // *(uint8_t *)payload = TSDB_FALSE; + *sizeAppend = + tsSetColumnValue(payload, pSchema->colId, pSchema->type, &FALSE_VALUE, TYPE_BYTES[TSDB_DATA_TYPE_BOOL]); + *kvRowColLen += sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_BOOL]; + } else { + return tscSQLSyntaxErrMsg(msg, "invalid bool data", pToken->z); + } + } else if (pToken->type == TK_INTEGER) { + iv = strtoll(pToken->z, NULL, 10); + // *(uint8_t *)payload = (int8_t)((iv == 0) ? TSDB_FALSE : TSDB_TRUE); + *sizeAppend = tsSetColumnValue(payload, pSchema->colId, pSchema->type, + ((iv == 0) ? &FALSE_VALUE : &TRUE_VALUE), TYPE_BYTES[TSDB_DATA_TYPE_BOOL]); + *kvRowColLen += sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_BOOL]; + } else if (pToken->type == TK_FLOAT) { + double dv = strtod(pToken->z, NULL); + // *(uint8_t *)payload = (int8_t)((dv == 0) ? TSDB_FALSE : TSDB_TRUE); + *sizeAppend = tsSetColumnValue(payload, pSchema->colId, pSchema->type, + ((dv == 0) ? &FALSE_VALUE : &TRUE_VALUE), TYPE_BYTES[TSDB_DATA_TYPE_BOOL]); + *kvRowColLen += sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_BOOL]; + } else { + return tscInvalidOperationMsg(msg, "invalid bool data", pToken->z); + } + } + break; + } + + case TSDB_DATA_TYPE_TINYINT: + if (isNullStr(pToken)) { + // *((uint8_t *)payload) = TSDB_DATA_TINYINT_NULL; + *isColNull = true; + } else { + ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); + if (ret != TSDB_CODE_SUCCESS) { + return tscInvalidOperationMsg(msg, "invalid tinyint data", pToken->z); + } else if (!IS_VALID_TINYINT(iv)) { + return tscInvalidOperationMsg(msg, "data overflow", pToken->z); + } + + // *((uint8_t *)payload) = (uint8_t)iv; + uint8_t tmpVal = (uint8_t)iv; + *sizeAppend = + tsSetColumnValue(payload, pSchema->colId, pSchema->type, &tmpVal, TYPE_BYTES[TSDB_DATA_TYPE_TINYINT]); + *kvRowColLen += sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_TINYINT]; + } + + break; + + case TSDB_DATA_TYPE_UTINYINT: + if (isNullStr(pToken)) { + // *((uint8_t *)payload) = TSDB_DATA_UTINYINT_NULL; + *isColNull = true; + } else { + ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); + if (ret != TSDB_CODE_SUCCESS) { + return tscInvalidOperationMsg(msg, "invalid unsigned tinyint data", pToken->z); + } else if (!IS_VALID_UTINYINT(iv)) { + return tscInvalidOperationMsg(msg, "unsigned tinyint data overflow", pToken->z); + } + + // *((uint8_t *)payload) = (uint8_t)iv; + uint8_t tmpVal = (uint8_t)iv; + *sizeAppend = + tsSetColumnValue(payload, pSchema->colId, pSchema->type, &tmpVal, TYPE_BYTES[TSDB_DATA_TYPE_UTINYINT]); + *kvRowColLen += sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_UTINYINT]; + } + + break; + + case TSDB_DATA_TYPE_SMALLINT: + if (isNullStr(pToken)) { + // *((int16_t *)payload) = TSDB_DATA_SMALLINT_NULL; + *isColNull = true; + } else { + ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); + if (ret != TSDB_CODE_SUCCESS) { + return tscInvalidOperationMsg(msg, "invalid smallint data", pToken->z); + } else if (!IS_VALID_SMALLINT(iv)) { + return tscInvalidOperationMsg(msg, "smallint data overflow", pToken->z); + } + + int16_t tmpVal = (int16_t)iv; + *sizeAppend = + tsSetColumnValue(payload, pSchema->colId, pSchema->type, &tmpVal, TYPE_BYTES[TSDB_DATA_TYPE_SMALLINT]); + *kvRowColLen += sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_SMALLINT]; + } + + break; + + case TSDB_DATA_TYPE_USMALLINT: + if (isNullStr(pToken)) { + // *((uint16_t *)payload) = TSDB_DATA_USMALLINT_NULL; + *isColNull = true; + } else { + ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); + if (ret != TSDB_CODE_SUCCESS) { + return tscInvalidOperationMsg(msg, "invalid unsigned smallint data", pToken->z); + } else if (!IS_VALID_USMALLINT(iv)) { + return tscInvalidOperationMsg(msg, "unsigned smallint data overflow", pToken->z); + } + + // *((uint16_t *)payload) = (uint16_t)iv; + uint16_t tmpVal = (uint16_t)iv; + *sizeAppend = + tsSetColumnValue(payload, pSchema->colId, pSchema->type, &tmpVal, TYPE_BYTES[TSDB_DATA_TYPE_USMALLINT]); + *kvRowColLen += sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_USMALLINT]; + } + + break; + + case TSDB_DATA_TYPE_INT: + if (isNullStr(pToken)) { + // *((int32_t *)payload) = TSDB_DATA_INT_NULL; + *isColNull = true; + } else { + ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); + if (ret != TSDB_CODE_SUCCESS) { + return tscInvalidOperationMsg(msg, "invalid int data", pToken->z); + } else if (!IS_VALID_INT(iv)) { + return tscInvalidOperationMsg(msg, "int data overflow", pToken->z); + } + + // *((int32_t *)payload) = (int32_t)iv; + int32_t tmpVal = (int32_t)iv; + *sizeAppend = tsSetColumnValue(payload, pSchema->colId, pSchema->type, &tmpVal, TYPE_BYTES[TSDB_DATA_TYPE_INT]); + *kvRowColLen += sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_INT]; + } + + break; + + case TSDB_DATA_TYPE_UINT: + if (isNullStr(pToken)) { + // *((uint32_t *)payload) = TSDB_DATA_UINT_NULL; + *isColNull = true; + } else { + ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); + if (ret != TSDB_CODE_SUCCESS) { + return tscInvalidOperationMsg(msg, "invalid unsigned int data", pToken->z); + } else if (!IS_VALID_UINT(iv)) { + return tscInvalidOperationMsg(msg, "unsigned int data overflow", pToken->z); + } + + // *((uint32_t *)payload) = (uint32_t)iv; + uint32_t tmpVal = (uint32_t)iv; + *sizeAppend = + tsSetColumnValue(payload, pSchema->colId, pSchema->type, &tmpVal, TYPE_BYTES[TSDB_DATA_TYPE_UINT]); + *kvRowColLen += sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_UINT]; + } + + break; + + case TSDB_DATA_TYPE_BIGINT: + if (isNullStr(pToken)) { + // *((int64_t *)payload) = TSDB_DATA_BIGINT_NULL; + *isColNull = true; + } else { + ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); + if (ret != TSDB_CODE_SUCCESS) { + return tscInvalidOperationMsg(msg, "invalid bigint data", pToken->z); + } else if (!IS_VALID_BIGINT(iv)) { + return tscInvalidOperationMsg(msg, "bigint data overflow", pToken->z); + } + + // *((int64_t *)payload) = iv; + // int64_t tmpVal = (int64_t)iv; + *sizeAppend = tsSetColumnValue(payload, pSchema->colId, pSchema->type, &iv, TYPE_BYTES[TSDB_DATA_TYPE_BIGINT]); + *kvRowColLen += sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_BIGINT]; + } + break; + + case TSDB_DATA_TYPE_UBIGINT: + if (isNullStr(pToken)) { + // *((uint64_t *)payload) = TSDB_DATA_UBIGINT_NULL; + *isColNull = true; + } else { + ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); + if (ret != TSDB_CODE_SUCCESS) { + return tscInvalidOperationMsg(msg, "invalid unsigned bigint data", pToken->z); + } else if (!IS_VALID_UBIGINT((uint64_t)iv)) { + return tscInvalidOperationMsg(msg, "unsigned bigint data overflow", pToken->z); + } + + // *((uint64_t *)payload) = iv; + *sizeAppend = tsSetColumnValue(payload, pSchema->colId, pSchema->type, &iv, TYPE_BYTES[TSDB_DATA_TYPE_UBIGINT]); + *kvRowColLen += sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_UBIGINT]; + } + break; + + case TSDB_DATA_TYPE_FLOAT: + if (isNullStr(pToken)) { + // *((int32_t *)payload) = TSDB_DATA_FLOAT_NULL; + *isColNull = true; + } else { + double dv; + if (TK_ILLEGAL == tscToDouble(pToken, &dv, &endptr)) { + return tscInvalidOperationMsg(msg, "illegal float data", pToken->z); + } + + if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || + isnan(dv)) { + return tscInvalidOperationMsg(msg, "illegal float data", pToken->z); + } + + // *((float *)payload) = (float)dv; + // SET_FLOAT_VAL(payload, dv); + float tmpVal = (float)dv; + *sizeAppend = + tsSetColumnValue(payload, pSchema->colId, pSchema->type, &tmpVal, TYPE_BYTES[TSDB_DATA_TYPE_FLOAT]); + *kvRowColLen += sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_FLOAT]; + } + break; + + case TSDB_DATA_TYPE_DOUBLE: + if (isNullStr(pToken)) { + // *((int64_t *)payload) = TSDB_DATA_DOUBLE_NULL; + *isColNull = true; + } else { + double dv; + if (TK_ILLEGAL == tscToDouble(pToken, &dv, &endptr)) { + return tscInvalidOperationMsg(msg, "illegal double data", pToken->z); + } + + if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) { + return tscInvalidOperationMsg(msg, "illegal double data", pToken->z); + } + + // *((double *)payload) = dv; + *sizeAppend = tsSetColumnValue(payload, pSchema->colId, pSchema->type, &dv, TYPE_BYTES[TSDB_DATA_TYPE_DOUBLE]); + *kvRowColLen += sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_DOUBLE]; + } + break; + + case TSDB_DATA_TYPE_BINARY: + // binary data cannot be null-terminated char string, otherwise the last char of the string is lost + if (pToken->type == TK_NULL) { + // setVardataNull(payload, TSDB_DATA_TYPE_BINARY); + *isColNull = true; + } else { // too long values will return invalid sql, not be truncated automatically + if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { // todo refactor + return tscInvalidOperationMsg(msg, "string data overflow", pToken->z); + } + + // STR_WITH_SIZE_TO_VARSTR(payload, pToken->z, pToken->n); + + payloadColSetId(payload, pSchema->colId); + payloadColSetType(payload, pSchema->type); + varDataSetLen(payload + PAYLOAD_ID_TYPE_LEN, pToken->n); + memcpy(varDataVal(payload + PAYLOAD_ID_TYPE_LEN), pToken->z, pToken->n); + *sizeAppend = PAYLOAD_ID_TYPE_LEN + sizeof(VarDataLenT) + pToken->n; + *dataRowColDeltaLen += (pToken->n - sizeof(uint8_t)); + *kvRowColLen += sizeof(SColIdx) + sizeof(VarDataLenT) + pToken->n; + } + + break; + + case TSDB_DATA_TYPE_NCHAR: + if (pToken->type == TK_NULL) { + // setVardataNull(payload, TSDB_DATA_TYPE_NCHAR); + *isColNull = true; + } else { + // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' + int32_t output = 0; + payloadColSetId(payload, pSchema->colId); + payloadColSetType(payload, pSchema->type); + if (!taosMbsToUcs4(pToken->z, pToken->n, varDataVal(payload + PAYLOAD_ID_TYPE_LEN), + pSchema->bytes - VARSTR_HEADER_SIZE, &output)) { + char buf[512] = {0}; + snprintf(buf, tListLen(buf), "%s", strerror(errno)); + return tscInvalidOperationMsg(msg, buf, pToken->z); + } + + varDataSetLen(payload + PAYLOAD_ID_TYPE_LEN, output); + + *sizeAppend = PAYLOAD_ID_TYPE_LEN + sizeof(VarDataLenT) + output; + *dataRowColDeltaLen += (output - sizeof(uint32_t)); + *kvRowColLen += sizeof(SColIdx) + sizeof(VarDataLenT) + output; + } + break; + + case TSDB_DATA_TYPE_TIMESTAMP: { + if (pToken->type == TK_NULL) { + if (primaryKey) { + // *((int64_t *)payload) = 0; + // When building SKVRow primaryKey, we should not skip even with NULL value. + int64_t tmpVal = 0; + *sizeAppend = + tsSetColumnValue(payload, pSchema->colId, pSchema->type, &tmpVal, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP]); + *kvRowColLen += sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP]; + } else { + // *((int64_t *)payload) = TSDB_DATA_BIGINT_NULL; + *isColNull = true; + } + } else { + int64_t tmpVal; + if (tsParseTime(pToken, &tmpVal, str, msg, timePrec) != TSDB_CODE_SUCCESS) { + return tscInvalidOperationMsg(msg, "invalid timestamp", pToken->z); + } + + // *((int64_t *)payload) = tmpVal; + *sizeAppend = + tsSetColumnValue(payload, pSchema->colId, pSchema->type, &tmpVal, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP]); + *kvRowColLen += sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP]; + } + + break; + } + } + + return TSDB_CODE_SUCCESS; +} + /* * The server time/client time should not be mixed up in one sql string * Do not employ sort operation is not involved if server time is used. @@ -414,23 +763,32 @@ int32_t tsCheckTimestamp(STableDataBlocks *pDataBlocks, const char *start) { return TSDB_CODE_SUCCESS; } -int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, int32_t *len, - char *tmpTokenBuf, SInsertStatementParam* pInsertParam) { - int32_t index = 0; - SStrToken sToken = {0}; - char *payload = pDataBlocks->pData + pDataBlocks->size; +int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, int32_t *len, char *tmpTokenBuf, + SInsertStatementParam *pInsertParam) { + int32_t index = 0; + SStrToken sToken = {0}; + + SMemRowBuilder *pBuilder = &pDataBlocks->rowBuilder; + char * payload = pDataBlocks->pData + pDataBlocks->size; SParsedDataColInfo *spd = &pDataBlocks->boundColumnInfo; - SSchema *schema = tscGetTableSchema(pDataBlocks->pTableMeta); + SSchema * schema = tscGetTableSchema(pDataBlocks->pTableMeta); // 1. set the parsed value from sql string - int32_t rowSize = 0; + int32_t rowSize = 0; + uint16_t rowSizeAppended = 0; + uint16_t nColsNotNull = 0; + TDRowLenT dataRowLen = pBuilder->allNullLen; + TDRowLenT kvRowLen = TD_MEM_ROW_KV_VER_SIZE; + + char *kvStart = payload; for (int i = 0; i < spd->numOfBound; ++i) { // the start position in data block buffer of current value in sql - int32_t colIndex = spd->boundedColumns[i]; + int32_t colIndex = spd->boundedColumns[i]; // ordered - char *start = payload + spd->cols[colIndex].offset; - SSchema *pSchema = &schema[colIndex]; + char *start = payload + spd->cols[colIndex].offset; + + SSchema *pSchema = &schema[colIndex]; // get colId here rowSize += pSchema->bytes; index = 0; @@ -453,7 +811,8 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i int16_t type = sToken.type; if ((type != TK_NOW && type != TK_INTEGER && type != TK_STRING && type != TK_FLOAT && type != TK_BOOL && - type != TK_NULL && type != TK_HEX && type != TK_OCT && type != TK_BIN) || (sToken.n == 0) || (type == TK_RP)) { + type != TK_NULL && type != TK_HEX && type != TK_OCT && type != TK_BIN) || + (sToken.n == 0) || (type == TK_RP)) { return tscSQLSyntaxErrMsg(pInsertParam->msg, "invalid data or symbol", sToken.z); } @@ -467,10 +826,10 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i if (sToken.n >= TSDB_MAX_BYTES_PER_ROW) { return tscSQLSyntaxErrMsg(pInsertParam->msg, "too long string", sToken.z); } - + for (uint32_t k = 1; k < sToken.n - 1; ++k) { if (sToken.z[k] == '\\' || (sToken.z[k] == delim && sToken.z[k + 1] == delim)) { - tmpTokenBuf[j] = sToken.z[k + 1]; + tmpTokenBuf[j] = sToken.z[k + 1]; cnt++; j++; @@ -487,42 +846,44 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i sToken.n -= 2 + cnt; } - bool isPrimaryKey = (colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX); - int32_t ret = tsParseOneColumn(pSchema, &sToken, start, pInsertParam->msg, str, isPrimaryKey, timePrec); + bool isPrimaryKey = (colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX); + bool isColNull = false; + TDRowLenT dataRowDeltaColLen = 0; // When combine the data as SDataRow, the delta len between all NULL columns. + TDRowLenT kvRowColLen = 0; + uint16_t colSizeAppended = 0; + int32_t ret = + tsParseOneColumnKV(pSchema, &sToken, kvStart + PAYLOAD_HEADER_LEN, pInsertParam->msg, str, isPrimaryKey, + timePrec, &colSizeAppended, &isColNull, &dataRowDeltaColLen, &kvRowColLen); if (ret != TSDB_CODE_SUCCESS) { return ret; } - if (isPrimaryKey && tsCheckTimestamp(pDataBlocks, start) != TSDB_CODE_SUCCESS) { + if (isPrimaryKey && + tsCheckTimestamp(pDataBlocks, payloadKeyAddr(kvStart)) != TSDB_CODE_SUCCESS) { tscInvalidOperationMsg(pInsertParam->msg, "client time/server time can not be mixed up", sToken.z); return TSDB_CODE_TSC_INVALID_TIME_STAMP; } - } - - // 2. set the null value for the columns that do not assign values - if (spd->numOfBound < spd->numOfCols) { - char *ptr = payload; - - for (int32_t i = 0; i < spd->numOfCols; ++i) { - if (!spd->cols[i].hasVal) { // current column do not have any value to insert, set it to null - if (schema[i].type == TSDB_DATA_TYPE_BINARY) { - varDataSetLen(ptr, sizeof(int8_t)); - *(uint8_t*) varDataVal(ptr) = TSDB_DATA_BINARY_NULL; - } else if (schema[i].type == TSDB_DATA_TYPE_NCHAR) { - varDataSetLen(ptr, sizeof(int32_t)); - *(uint32_t*) varDataVal(ptr) = TSDB_DATA_NCHAR_NULL; - } else { - setNull(ptr, schema[i].type, schema[i].bytes); - } - } - - ptr += schema[i].bytes; + if (isColNull == false) { + ++nColsNotNull; } - - rowSize = (int32_t)(ptr - payload); + kvRowLen += kvRowColLen; + dataRowLen += dataRowDeltaColLen; + kvStart += colSizeAppended; // move to next column + rowSizeAppended += colSizeAppended; // calculate rowLen } - *len = rowSize; + if (kvRowLen < dataRowLen) { + payloadSetType(payload, SMEM_ROW_KV); + } else { + payloadSetType(payload, SMEM_ROW_DATA); + } + + *(uint16_t *)(payload + sizeof(uint8_t)) = nColsNotNull; + *len = PAYLOAD_HEADER_LEN + rowSizeAppended; + + payloadSetNCols(payload, nColsNotNull); + payloadSetTLen(payload, *len); + return TSDB_CODE_SUCCESS; } @@ -551,21 +912,26 @@ int32_t tsParseValues(char **str, STableDataBlocks *pDataBlock, int maxRows, SIn int32_t precision = tinfo.precision; + int32_t rowSizeWithColIdType = getPaddingRowSize(&tinfo); + + initSMemRowBuilder(&pDataBlock->rowBuilder, tscGetTableSchema(pDataBlock->pTableMeta), + tscGetNumOfColumns(pDataBlock->pTableMeta), 0); + while (1) { index = 0; sToken = tStrGetToken(*str, &index, false); if (sToken.n == 0 || sToken.type != TK_LP) break; *str += index; - if ((*numOfRows) >= maxRows || pDataBlock->size + tinfo.rowSize >= pDataBlock->nAllocSize) { + if ((*numOfRows) >= maxRows || pDataBlock->size + rowSizeWithColIdType >= pDataBlock->nAllocSize) { int32_t tSize; - code = tscAllocateMemIfNeed(pDataBlock, tinfo.rowSize, &tSize); + code = tscAllocateMemIfNeed(pDataBlock, rowSizeWithColIdType, &tSize); if (code != TSDB_CODE_SUCCESS) { //TODO pass the correct error code to client strcpy(pInsertParam->msg, "client out of memory"); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - ASSERT(tSize > maxRows); + ASSERT(tSize >= maxRows); maxRows = tSize; } @@ -623,7 +989,7 @@ int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int3 // expand the allocated size if (remain < rowSize * factor) { while (remain < rowSize * factor) { - pDataBlock->nAllocSize = (uint32_t)(pDataBlock->nAllocSize * 1.5); + pDataBlock->nAllocSize = (uint32_t)(pDataBlock->nAllocSize * 2); remain = pDataBlock->nAllocSize - pDataBlock->size; } @@ -657,7 +1023,7 @@ static int32_t tsSetBlockInfo(SSubmitBlk *pBlocks, const STableMeta *pTableMeta, } // data block is disordered, sort it in ascending order -void tscSortRemoveDataBlockDupRows(STableDataBlocks *dataBuf) { +void tscSortRemoveDataBlockDupRowsXX(STableDataBlocks *dataBuf) { SSubmitBlk *pBlocks = (SSubmitBlk *)dataBuf->pData; // size is less than the total size, since duplicated rows may be removed yet. @@ -701,11 +1067,83 @@ void tscSortRemoveDataBlockDupRows(STableDataBlocks *dataBuf) { dataBuf->prevTS = INT64_MIN; } +// data block is disordered, sort it in ascending order +int tscSortRemoveDataBlockDupRows(STableDataBlocks *dataBuf, SBlockKeyInfo *pBlkKeyInfo) { + SSubmitBlk *pBlocks = (SSubmitBlk *)dataBuf->pData; + int16_t nRows = pBlocks->numOfRows; + + // size is less than the total size, since duplicated rows may be removed yet. + + // if use server time, this block must be ordered + if (dataBuf->tsSource == TSDB_USE_SERVER_TS) { + assert(dataBuf->ordered); + } + // allocate memory + size_t curBlkTupleSize = nRows * sizeof(SBlockKeyTuple); + if (pBlkKeyInfo->pKeyTuple == NULL || pBlkKeyInfo->nBytesAlloc < curBlkTupleSize) { + char *tmp = realloc(pBlkKeyInfo->pKeyTuple, curBlkTupleSize); + if (tmp == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + pBlkKeyInfo->pKeyTuple = (SBlockKeyTuple *)tmp; + pBlkKeyInfo->nBytesAlloc = curBlkTupleSize; + } + memset(pBlkKeyInfo->pKeyTuple, 0, curBlkTupleSize); + + SBlockKeyTuple *pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; + char * pBlockData = pBlocks->data; + int n = 0; + uint32_t totolPayloadLen = 0; + TDRowLenT payloadTLen = 0; + while (n < nRows) { + pBlkKeyTuple->skey = payloadKey(pBlockData); + pBlkKeyTuple->payloadAddr = pBlockData; + payloadTLen = payloadTLen(pBlockData); + totolPayloadLen += payloadTLen; + // next loop + pBlockData += payloadTLen; + ++pBlkKeyTuple; + ++n; + } + + if (!dataBuf->ordered) { + qsort(pBlkKeyTuple, nRows, sizeof(SBlockKeyTuple), rowDataCompar); + + pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; + int32_t i = 0; + int32_t j = 1; + while (j < nRows) { + TSKEY ti = *(TSKEY *)(pBlkKeyTuple + sizeof(SBlockKeyTuple) * i); + TSKEY tj = *(TSKEY *)(pBlkKeyTuple + sizeof(SBlockKeyTuple) * j); + + if (ti == tj) { + totolPayloadLen -= payloadTLen(pBlkKeyTuple + sizeof(SBlockKeyTuple) * j); + ++j; + continue; + } + + int32_t nextPos = (++i); + if (nextPos != j) { + memmove(pBlkKeyTuple + sizeof(SBlockKeyTuple) * nextPos, pBlkKeyTuple + sizeof(SBlockKeyTuple) * j, sizeof(SBlockKeyTuple)); + } + ++j; + } + + dataBuf->ordered = true; + pBlocks->numOfRows = i + 1; + } + + dataBuf->size = sizeof(SSubmitBlk) + totolPayloadLen; + dataBuf->prevTS = INT64_MIN; + + return 0; +} + static int32_t doParseInsertStatement(SInsertStatementParam *pInsertParam, char **str, STableDataBlocks* dataBuf, int32_t *totalNum) { STableComInfo tinfo = tscGetTableInfo(dataBuf->pTableMeta); int32_t maxNumOfRows; - int32_t code = tscAllocateMemIfNeed(dataBuf, tinfo.rowSize, &maxNumOfRows); + int32_t code = tscAllocateMemIfNeed(dataBuf, getPaddingRowSize(&tinfo), &maxNumOfRows); if (TSDB_CODE_SUCCESS != code) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -1489,21 +1927,24 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int32_t numOfRow } STableDataBlocks *pTableDataBlock = NULL; - int32_t ret = - tscGetDataBlockFromList(pInsertParam->pTableBlockHashList, pTableMeta->id.uid, TSDB_PAYLOAD_SIZE, sizeof(SSubmitBlk), - tinfo.rowSize, &pTableMetaInfo->name, pTableMeta, &pTableDataBlock, NULL); + int32_t ret = tscGetDataBlockFromList(pInsertParam->pTableBlockHashList, pTableMeta->id.uid, TSDB_PAYLOAD_SIZE, + sizeof(SSubmitBlk), tinfo.rowSize, &pTableMetaInfo->name, pTableMeta, + &pTableDataBlock, NULL); if (ret != TSDB_CODE_SUCCESS) { pParentSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY; goto _error; } - tscAllocateMemIfNeed(pTableDataBlock, tinfo.rowSize, &maxRows); + tscAllocateMemIfNeed(pTableDataBlock, getPaddingRowSize(&tinfo), &maxRows); tokenBuf = calloc(1, TSDB_MAX_BYTES_PER_ROW); if (tokenBuf == NULL) { code = TSDB_CODE_TSC_OUT_OF_MEMORY; goto _error; } + initSMemRowBuilder(&pTableDataBlock->rowBuilder, tscGetTableSchema(pTableDataBlock->pTableMeta), + tscGetNumOfColumns(pTableDataBlock->pTableMeta), 0); + while ((readLen = tgetline(&line, &n, fp)) != -1) { if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) { line[--readLen] = 0; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index fb09ceb5cb..8f3455814f 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1639,61 +1639,29 @@ int32_t tscGetDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, i return TSDB_CODE_SUCCESS; } -static FORCE_INLINE uint8_t checkTdRowType(SSchema* pSchema, void* pData, int32_t nCols, int32_t flen, - uint16_t* nColsNotNull) { - ASSERT(pData != NULL); - if (nCols < KvRowNColsThresh) { - return SMEM_ROW_DATA; - } - int32_t dataRowLength = flen; - int32_t kvRowLength = TD_MEM_ROW_KV_VER_SIZE; - uint16_t nColsNull = 0; - char* p = (char*)pData; - for (int i = 0; i < nCols; ++i) { - if (IS_VAR_DATA_TYPE(pSchema[i].type)) { - dataRowLength += varDataTLen(p); - if (!isNull(p, pSchema[i].type)) { - kvRowLength += (sizeof(SColIdx) + varDataTLen(p)); - } else { - ++nColsNull; - } - } else { - if (!isNull(p, pSchema[i].type)) { - kvRowLength += (sizeof(SColIdx) + TYPE_BYTES[pSchema[i].type]); - } else { - ++nColsNull; - } - } - // next column - p += pSchema[i].bytes; - } - tscDebug("nColsNull %d, nCols: %d, kvRowLen: %d, dataRowLen: %d", (int32_t)nColsNull, nCols, kvRowLength, - dataRowLength); - if (kvRowLength < dataRowLength) { - if (nColsNotNull) { - *nColsNotNull = nCols - nColsNull; - } - return SMEM_ROW_KV; - } - - return SMEM_ROW_DATA; -} -SMemRow tdGenMemRowFromBuilder(SMemRowBuilder* pBuilder) { +static SMemRow tdGenMemRowFromBuilder(SMemRowBuilder* pBuilder) { SSchema* pSchema = pBuilder->pSchema; char* p = (char*)pBuilder->buf; int toffset = 0; + uint16_t nCols = pBuilder->nCols; - if(pBuilder->nCols <= 0){ +// RawRow payload structure: +// |<---------- header ------------->|<------- column data array ------->| +// |SMemRowType| dataLen | nCols | colId | colType | value |...|...| +// +-----------+----------+----------+---------------------------------->| +// | uint8_t | uint16_t | uint16_t | int16_t | uint8_t | ??? |...|...| +// +-----------+----------+----------+---------------------------------->| + uint8_t memRowType = payloadType(p); + uint16_t nColsNotNull = payloadNCols(p); + if (pBuilder->nCols <= 0 || nColsNotNull <= 0) { return NULL; } + ASSERT(nColsNotNull <= nCols); - uint16_t nColsNotNull = 0; - uint8_t memRowType = checkTdRowType(pSchema, p, pBuilder->nCols, pBuilder->flen, &nColsNotNull); - // nColsNotNull = pBuilder->nCols; SMemRow* memRow = (SMemRow)pBuilder->pDataBlock; memRowSetType(memRow, memRowType); @@ -1702,14 +1670,41 @@ SMemRow tdGenMemRowFromBuilder(SMemRowBuilder* pBuilder) { dataRowSetLen(trow, (uint16_t)(TD_DATA_ROW_HEAD_SIZE + pBuilder->flen)); dataRowSetVersion(trow, pBuilder->sversion); - p = (char*)pBuilder->buf; - for (int32_t j = 0; j < pBuilder->nCols; ++j) { - tdAppendColVal(trow, p, pSchema[j].type, pSchema[j].bytes, toffset); - toffset += TYPE_BYTES[pSchema[j].type]; - p += pSchema[j].bytes; + p = (char*)payloadBody(pBuilder->buf); + uint16_t i = 0, j = 0; + while (j < pBuilder->nCols) { + if (i >= nColsNotNull) { + break; + } + int16_t colId = *(int16_t*)p; + if (colId == pSchema[j].colId) { + tdAppendColVal(trow, payloadColValue(p), pSchema[j].type, toffset); + toffset += TYPE_BYTES[pSchema[j].type]; + p = skipToNextEles(p); + ++i; + ++j; + } else if (colId < pSchema[j].colId) { + p = skipToNextEles(p); + ++i; + } else { + tdAppendColVal(trow, tdGetNullVal(pSchema[j].type), pSchema[j].type, toffset); + toffset += TYPE_BYTES[pSchema[j].type]; + ++j; + } } + + while (j < pBuilder->nCols) { + tdAppendColVal(trow, tdGetNullVal(pSchema[j].type), pSchema[j].type, toffset); + toffset += TYPE_BYTES[pSchema[j].type]; + ++j; + } + while (i < nColsNotNull) { + p = skipToNextEles(p); + ++i; + } + pBuilder->buf = p; - } else if (memRowType == SMEM_ROW_KV) { + } else if (memRowType == SMEM_ROW_KV) { ASSERT(nColsNotNull <= pBuilder->nCols); SKVRow kvRow = (SKVRow)memRowKvBody(memRow); uint16_t tlen = TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * nColsNotNull; @@ -1717,14 +1712,17 @@ SMemRow tdGenMemRowFromBuilder(SMemRowBuilder* pBuilder) { kvRowSetNCols(kvRow, nColsNotNull); memRowKvSetVersion(memRow, pBuilder->sversion); - p = (char*)pBuilder->buf; - for (int32_t j = 0; j < pBuilder->nCols; ++j) { - if(!isNull(p, pSchema[j].type)) { - tdAppendKvColVal(kvRow, p, pSchema[j].colId, pSchema[j].type, toffset); - toffset += sizeof(SColIdx); - } - p += pSchema[j].bytes; + p = (char*)payloadBody(pBuilder->buf); + int i = 0; + while (i < nColsNotNull) { + int16_t colId = payloadColId(p); + uint8_t colType = payloadColType(p); + tdAppendKvColVal(kvRow, payloadColValue(p), colId, colType, toffset); + toffset += sizeof(SColIdx); + p = skipToNextEles(p); + ++i; } + pBuilder->buf = p; } else { @@ -1738,11 +1736,12 @@ SMemRow tdGenMemRowFromBuilder(SMemRowBuilder* pBuilder) { } // Erase the empty space reserved for binary data -static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, bool includeSchema) { +static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, bool includeSchema, SBlockKeyTuple *blkKeyTuple) { // TODO: optimize this function, handle the case while binary is not presented - STableMeta* pTableMeta = pTableDataBlock->pTableMeta; - STableComInfo tinfo = tscGetTableInfo(pTableMeta); - SSchema* pSchema = tscGetTableSchema(pTableMeta); + STableMeta* pTableMeta = pTableDataBlock->pTableMeta; + STableComInfo tinfo = tscGetTableInfo(pTableMeta); + SSchema* pSchema = tscGetTableSchema(pTableMeta); + SMemRowBuilder* pBuilder = &pTableDataBlock->rowBuilder; SSubmitBlk* pBlock = pDataBlock; memcpy(pDataBlock, pTableDataBlock->pData, sizeof(SSubmitBlk)); @@ -1778,18 +1777,18 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, bo pBlock->dataLen = 0; int32_t numOfRows = htons(pBlock->numOfRows); - SMemRowBuilder mRowBuilder; - mRowBuilder.pSchema = pSchema; - mRowBuilder.sversion = pTableMeta->sversion; - mRowBuilder.flen = flen; - mRowBuilder.nCols = tinfo.numOfColumns; - mRowBuilder.pDataBlock = pDataBlock; - mRowBuilder.pSubmitBlk = pBlock; - mRowBuilder.buf = p; - mRowBuilder.size = 0; + pBuilder->pSchema = pSchema; + pBuilder->sversion = pTableMeta->sversion; + pBuilder->flen = flen; + pBuilder->nCols = tinfo.numOfColumns; + pBuilder->pDataBlock = pDataBlock; + pBuilder->pSubmitBlk = pBlock; + pBuilder->buf = p; + pBuilder->size = 0; for (int32_t i = 0; i < numOfRows; ++i) { - tdGenMemRowFromBuilder(&mRowBuilder); + pBuilder->buf = (blkKeyTuple+i)->payloadAddr; + tdGenMemRowFromBuilder(pBuilder); } int32_t len = pBlock->dataLen + pBlock->schemaLen; @@ -1807,9 +1806,8 @@ static int32_t getRowExpandSize(STableMeta* pTableMeta) { if (IS_VAR_DATA_TYPE((pSchema + i)->type)) { result += TYPE_BYTES[TSDB_DATA_TYPE_BINARY]; } - result += sizeof(SColIdx); } - result += TD_MEM_ROW_TYPE_SIZE; // add len of SMemRow flag + result += TD_MEM_ROW_KV_TYPE_VER_SIZE; // add prefix len of KV type SMemRow(we may use SDataRow or SKVRow) return result; } @@ -1837,14 +1835,17 @@ static void extractTableNameList(SInsertStatementParam *pInsertParam, bool freeB } int32_t tscMergeTableDataBlocks(SInsertStatementParam *pInsertParam, bool freeBlockMap) { - const int INSERT_HEAD_SIZE = sizeof(SMsgDesc) + sizeof(SSubmitMsg); - - void* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); + const int INSERT_HEAD_SIZE = sizeof(SMsgDesc) + sizeof(SSubmitMsg); + int code = 0; + void* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); SArray* pVnodeDataBlockList = taosArrayInit(8, POINTER_BYTES); STableDataBlocks** p = taosHashIterate(pInsertParam->pTableBlockHashList, NULL); STableDataBlocks* pOneTableBlock = *p; + + SBlockKeyInfo blkKeyInfo = {0}; // share by pOneTableBlock + while(pOneTableBlock) { SSubmitBlk* pBlocks = (SSubmitBlk*) pOneTableBlock->pData; if (pBlocks->numOfRows > 0) { @@ -1858,6 +1859,7 @@ int32_t tscMergeTableDataBlocks(SInsertStatementParam *pInsertParam, bool freeBl tscError("0x%"PRIx64" failed to prepare the data block buffer for merging table data, code:%d", pInsertParam->objectId, ret); taosHashCleanup(pVnodeDataBlockHashList); tscDestroyBlockArrayList(pVnodeDataBlockList); + tfree(blkKeyInfo.pKeyTuple); return ret; } @@ -1878,16 +1880,26 @@ int32_t tscMergeTableDataBlocks(SInsertStatementParam *pInsertParam, bool freeBl taosHashCleanup(pVnodeDataBlockHashList); tscDestroyBlockArrayList(pVnodeDataBlockList); tfree(dataBuf->pData); + tfree(blkKeyInfo.pKeyTuple); return TSDB_CODE_TSC_OUT_OF_MEMORY; } } - tscSortRemoveDataBlockDupRows(pOneTableBlock); - char* ekey = (char*)pBlocks->data + pOneTableBlock->rowSize*(pBlocks->numOfRows-1); + if((code = tscSortRemoveDataBlockDupRows(pOneTableBlock, &blkKeyInfo)) != 0){ + taosHashCleanup(pVnodeDataBlockHashList); + tscDestroyBlockArrayList(pVnodeDataBlockList); + tfree(dataBuf->pData); + tfree(blkKeyInfo.pKeyTuple); + return code; + } - tscDebug("0x%"PRIx64" name:%s, tid:%d rows:%d sversion:%d skey:%" PRId64 ", ekey:%" PRId64, pInsertParam->objectId, tNameGetTableName(&pOneTableBlock->tableName), - pBlocks->tid, pBlocks->numOfRows, pBlocks->sversion, GET_INT64_VAL(pBlocks->data), GET_INT64_VAL(ekey)); + ASSERT(blkKeyInfo.pKeyTuple != NULL && pBlocks->numOfRows > 0); + + SBlockKeyTuple* pLastKeyTuple = blkKeyInfo.pKeyTuple + pBlocks->numOfRows - 1; + tscDebug("0x%" PRIx64 " name:%s, tid:%d rows:%d sversion:%d skey:%" PRId64 ", ekey:%" PRId64, + pInsertParam->objectId, tNameGetTableName(&pOneTableBlock->tableName), pBlocks->tid, pBlocks->numOfRows, + pBlocks->sversion, blkKeyInfo.pKeyTuple->skey, pLastKeyTuple->skey); int32_t len = pBlocks->numOfRows * (pOneTableBlock->rowSize + expandSize) + sizeof(STColumn) * tscGetNumOfColumns(pOneTableBlock->pTableMeta); @@ -1898,7 +1910,7 @@ int32_t tscMergeTableDataBlocks(SInsertStatementParam *pInsertParam, bool freeBl pBlocks->schemaLen = 0; // erase the empty space reserved for binary data - int32_t finalLen = trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock, pInsertParam->schemaAttached); + int32_t finalLen = trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock, pInsertParam->schemaAttached, blkKeyInfo.pKeyTuple); assert(finalLen <= len); dataBuf->size += (finalLen + sizeof(SSubmitBlk)); @@ -1926,6 +1938,7 @@ int32_t tscMergeTableDataBlocks(SInsertStatementParam *pInsertParam, bool freeBl // free the table data blocks; pInsertParam->pDataBlocks = pVnodeDataBlockList; taosHashCleanup(pVnodeDataBlockHashList); + tfree(blkKeyInfo.pKeyTuple); return TSDB_CODE_SUCCESS; } diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index b0402d4674..49a0c0f120 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -24,6 +24,33 @@ extern "C" { #endif +#pragma pack(push, 1) +typedef struct { + VarDataLenT len; + uint8_t data; +} SBinaryNullT; + +typedef struct { + VarDataLenT len; + uint32_t data; +} SNCharNullT; +#pragma pack(pop) + +extern const uint8_t BoolNull; +extern const uint8_t TinyintNull; +extern const uint16_t SmallintNull; +extern const uint32_t IntNull; +extern const uint64_t BigintNull; +extern const uint64_t TimestampNull; +extern const uint8_t UTinyintNull; +extern const uint16_t USmallintNull; +extern const uint32_t UIntNull; +extern const uint64_t UBigintNull; +extern const uint32_t FloatNull; +extern const uint64_t DoubleNull; +extern const SBinaryNullT BinaryNull; +extern const SNCharNullT NcharNull; + #define STR_TO_VARSTR(x, str) \ do { \ VarDataLenT __len = (VarDataLenT)strlen(str); \ @@ -207,7 +234,7 @@ SDataRow tdDataRowDup(SDataRow row); SMemRow tdMemRowDup(SMemRow row); // offset here not include dataRow header length -static FORCE_INLINE int tdAppendColVal(SDataRow row, void *value, int8_t type, int32_t bytes, int32_t offset) { +static FORCE_INLINE int tdAppendColVal(SDataRow row, const void *value, int8_t type, int32_t offset) { ASSERT(value != NULL); int32_t toffset = offset + TD_DATA_ROW_HEAD_SIZE; char * ptr = (char *)POINTER_SHIFT(row, dataRowLen(row)); @@ -260,6 +287,42 @@ void dataColSetOffset(SDataCol *pCol, int nEle); bool isNEleNull(SDataCol *pCol, int nEle); void dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints); +static FORCE_INLINE const void *tdGetNullVal(int8_t type) { + switch (type) { + case TSDB_DATA_TYPE_BOOL: + return &BoolNull; + case TSDB_DATA_TYPE_TINYINT: + return &TinyintNull; + case TSDB_DATA_TYPE_SMALLINT: + return &SmallintNull; + case TSDB_DATA_TYPE_INT: + return &IntNull; + case TSDB_DATA_TYPE_BIGINT: + return &BigintNull; + case TSDB_DATA_TYPE_FLOAT: + return &FloatNull; + case TSDB_DATA_TYPE_DOUBLE: + return &DoubleNull; + case TSDB_DATA_TYPE_BINARY: + return &BinaryNull; + case TSDB_DATA_TYPE_TIMESTAMP: + return &TimestampNull; + case TSDB_DATA_TYPE_NCHAR: + return &NcharNull; + case TSDB_DATA_TYPE_UTINYINT: + return &UTinyintNull; + case TSDB_DATA_TYPE_USMALLINT: + return &USmallintNull; + case TSDB_DATA_TYPE_UINT: + return &UIntNull; + case TSDB_DATA_TYPE_UBIGINT: + return &UBigintNull; + default: + ASSERT(0); + return NULL; + } +} + // Get the data pointer from a column-wised data static FORCE_INLINE void *tdGetColDataOfRow(SDataCol *pCol, int row) { if (IS_VAR_DATA_TYPE(pCol->type)) { @@ -545,6 +608,47 @@ static FORCE_INLINE void *tdGetMemRowDataOfCol(void *row, int8_t type, int32_t o return NULL; } +// RawRow payload structure: +// |<---------- header ------------->|<---- body: column data tuple ---->| +// |SMemRowType| dataLen | nCols | colId | colType | value |...|...| +// +-----------+----------+----------+---------------------------------->| +// | uint8_t | uint16_t | uint16_t | int16_t | uint8_t | ??? |...|...| +// +-----------+----------+----------+---------------------------------->| + +#define PAYLOAD_NCOLS_LEN sizeof(uint16_t) +#define PAYLOAD_NCOLS_OFFSET (sizeof(uint8_t) + sizeof(TDRowLenT)) +#define PAYLOAD_HEADER_LEN (PAYLOAD_NCOLS_OFFSET + PAYLOAD_NCOLS_LEN) +#define PAYLOAD_ID_LEN sizeof(int16_t) +#define PAYLOAD_ID_TYPE_LEN (sizeof(int16_t) + sizeof(uint8_t)) + +#define payloadBody(r) POINTER_SHIFT(r, PAYLOAD_HEADER_LEN) +#define payloadType(r) (*(uint8_t *)(r)) +#define payloadSetType(r, t) (payloadType(r) = (t)) +#define payloadTLen(r) (*(TDRowLenT *)POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE)) // including total header +#define payloadSetTLen(r, l) (payloadTLen(r) = (l)) +#define payloadNCols(r) (*(TDRowLenT *)POINTER_SHIFT(r, PAYLOAD_NCOLS_OFFSET)) +#define payloadSetNCols(r, n) (payloadNCols(r) = (n)) + +#define payloadColId(r) (*(int16_t *)(r)) +#define payloadColType(r) (*(uint8_t *)POINTER_SHIFT(r, PAYLOAD_ID_LEN)) +#define payloadColValue(r) POINTER_SHIFT(r, PAYLOAD_ID_TYPE_LEN) + +#define payloadColSetId(r, i) (payloadColId(r) = (i)) +#define payloadColSetType(r, t) (payloadColType(r) = (t)) + +#define payloadKeyAddr(r) POINTER_SHIFT(r, PAYLOAD_HEADER_LEN + PAYLOAD_ID_TYPE_LEN) +#define payloadTKey(r) (*(TKEY *)(payloadKeyAddr(r))) +#define payloadKey(r) tdGetKey(payloadTKey(r)) + +static FORCE_INLINE char *skipToNextEles(char *p) { + uint8_t colType = payloadColType(p); + if (IS_VAR_DATA_TYPE(colType)) { + return POINTER_SHIFT(p, PAYLOAD_ID_TYPE_LEN + varDataTLen(payloadColValue(p))); + } else { + return POINTER_SHIFT(p, PAYLOAD_ID_TYPE_LEN + TYPE_BYTES[colType]); + } +} + #ifdef __cplusplus } #endif diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index a46d2e84b0..5392cebe84 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -18,6 +18,21 @@ #include "tcoding.h" #include "wchar.h" +const uint8_t BoolNull = TSDB_DATA_BOOL_NULL; +const uint8_t TinyintNull = TSDB_DATA_TINYINT_NULL; +const uint16_t SmallintNull = TSDB_DATA_SMALLINT_NULL; +const uint32_t IntNull = TSDB_DATA_INT_NULL; +const uint64_t BigintNull = TSDB_DATA_BIGINT_NULL; +const uint64_t TimestampNull = TSDB_DATA_BIGINT_NULL; +const uint8_t UTinyintNull = TSDB_DATA_UTINYINT_NULL; +const uint16_t USmallintNull = TSDB_DATA_USMALLINT_NULL; +const uint32_t UIntNull = TSDB_DATA_UINT_NULL; +const uint64_t UBigintNull = TSDB_DATA_UBIGINT_NULL; +const uint32_t FloatNull = TSDB_DATA_FLOAT_NULL; +const uint64_t DoubleNull = TSDB_DATA_DOUBLE_NULL; +const SBinaryNullT BinaryNull = {1, TSDB_DATA_BINARY_NULL}; +const SNCharNullT NcharNull = {4, TSDB_DATA_NCHAR_NULL}; + static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, int limit2, int tRows); diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c index ee1022ea0c..cd762f3110 100644 --- a/src/cq/src/cqMain.c +++ b/src/cq/src/cqMain.c @@ -502,7 +502,7 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) { memcpy((char *)val + sizeof(VarDataLenT), buf, len); varDataLen(val) = len; } - tdAppendColVal(dataRow, val, c->type, c->bytes, c->offset); + tdAppendColVal(dataRow, val, c->type, c->offset); } pBlk->dataLen = htonl(memRowDataTLen(trow)); pBlk->schemaLen = 0; diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index 69d855e57c..8718d5c35a 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -924,17 +924,17 @@ int tsdbWriteBlockImpl(STsdbRepo *pRepo, STable *pTable, SDFile *pDFile, SDataCo continue; } - memset(pBlockCol, 0, sizeof(*pBlockCol)); + memset(pBlockCol, 0, sizeof(*pBlockCol)); - pBlockCol->colId = pDataCol->colId; - pBlockCol->type = pDataCol->type; - if (tDataTypes[pDataCol->type].statisFunc) { - (*tDataTypes[pDataCol->type].statisFunc)(pDataCol->pData, rowsToWrite, &(pBlockCol->min), &(pBlockCol->max), - &(pBlockCol->sum), &(pBlockCol->minIndex), &(pBlockCol->maxIndex), - &(pBlockCol->numOfNull)); - } - nColsNotAllNull++; + pBlockCol->colId = pDataCol->colId; + pBlockCol->type = pDataCol->type; + if (tDataTypes[pDataCol->type].statisFunc) { + (*tDataTypes[pDataCol->type].statisFunc)(pDataCol->pData, rowsToWrite, &(pBlockCol->min), &(pBlockCol->max), + &(pBlockCol->sum), &(pBlockCol->minIndex), &(pBlockCol->maxIndex), + &(pBlockCol->numOfNull)); } + nColsNotAllNull++; + } ASSERT(nColsNotAllNull >= 0 && nColsNotAllNull <= pDataCols->numOfCols); diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 272a915d61..fc152231de 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -720,7 +720,7 @@ static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pRea // OK,let's load row from backward to get not-null column for (int32_t rowId = pBlock->numOfRows - 1; rowId >= 0; rowId--) { SDataCol *pDataCol = pReadh->pDCols[0]->cols + i; - tdAppendColVal(memRowDataBody(row), tdGetColDataOfRow(pDataCol, rowId), pCol->type, pCol->bytes, pCol->offset); + tdAppendColVal(memRowDataBody(row), tdGetColDataOfRow(pDataCol, rowId), pCol->type, pCol->offset); //SDataCol *pDataCol = readh.pDCols[0]->cols + j; void *value = tdGetRowDataOfCol(memRowDataBody(row), (int8_t)pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset); if (isNull(value, pCol->type)) { @@ -742,7 +742,7 @@ static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pRea // save row ts(in column 0) pDataCol = pReadh->pDCols[0]->cols + 0; pCol = schemaColAt(pSchema, 0); - tdAppendColVal(memRowDataBody(row), tdGetColDataOfRow(pDataCol, rowId), pCol->type, pCol->bytes, pCol->offset); + tdAppendColVal(memRowDataBody(row), tdGetColDataOfRow(pDataCol, rowId), pCol->type, pCol->offset); pLastCol->ts = memRowKey(row); pTable->restoreColumnNum += 1; @@ -790,7 +790,7 @@ static int tsdbRestoreLastRow(STsdbRepo *pRepo, STable *pTable, SReadH* pReadh, STColumn *pCol = schemaColAt(pSchema, icol); SDataCol *pDataCol = pReadh->pDCols[0]->cols + icol; tdAppendColVal(memRowDataBody(pTable->lastRow), tdGetColDataOfRow(pDataCol, pBlock->numOfRows - 1), pCol->type, - pCol->bytes, pCol->offset); + pCol->offset); } return 0; diff --git a/src/tsdb/tests/tsdbTests.cpp b/src/tsdb/tests/tsdbTests.cpp index ac254d6c34..dc804856fd 100644 --- a/src/tsdb/tests/tsdbTests.cpp +++ b/src/tsdb/tests/tsdbTests.cpp @@ -55,10 +55,10 @@ static int insertData(SInsertInfo *pInfo) { for (int j = 0; j < schemaNCols(pInfo->pSchema); j++) { STColumn *pTCol = schemaColAt(pInfo->pSchema, j); if (j == 0) { // Just for timestamp - tdAppendColVal(row, (void *)(&start_time), pTCol->type, pTCol->bytes, pTCol->offset); + tdAppendColVal(row, (void *)(&start_time), pTCol->type, pTCol->offset); } else { // For int int val = 10; - tdAppendColVal(row, (void *)(&val), pTCol->type, pTCol->bytes, pTCol->offset); + tdAppendColVal(row, (void *)(&val), pTCol->type, pTCol->offset); } } pBlock->dataLen += dataRowLen(row); From c10d065e1a5253446e5d0f2bfe79250677515d8c Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 7 Jul 2021 08:07:13 +0800 Subject: [PATCH 16/67] fix windows compiler err --- src/client/inc/tsclient.h | 22 ---------- src/client/src/tscParseInsert.c | 71 +++++++++++++++++++++------------ 2 files changed, 46 insertions(+), 47 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 428cbf3391..3771dbe575 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -110,28 +110,6 @@ typedef struct { uint16_t allNullLen; } SMemRowBuilder; -int FORCE_INLINE initSMemRowBuilder(SMemRowBuilder *pBuilder, SSchema *pSSchema, uint16_t nCols, - uint16_t allNullColsLen) { - ASSERT(nCols > 0); - pBuilder->pSchema = pSSchema; - pBuilder->allNullLen = allNullColsLen; // TODO: get allNullColsLen when creating or altering table meta - if (pBuilder->allNullLen == 0) { - for (uint16_t i = 0; i < nCols; ++i) { - uint8_t type = pSSchema[i].type; - int32_t typeLen = TYPE_BYTES[type]; - ASSERT(typeLen > 0); - pBuilder->allNullLen += typeLen; - if (TSDB_DATA_TYPE_BINARY == type) { - pBuilder->allNullLen += (sizeof(VarDataLenT) + CHAR_BYTES); - } else if (TSDB_DATA_TYPE_NCHAR == type) { - int len = sizeof(VarDataLenT) + TSDB_NCHAR_SIZE; - pBuilder->allNullLen += len; - } - } - } - return 0; -} - typedef struct STableDataBlocks { SName tableName; int8_t tsSource; // where does the UNIX timestamp come from, server or client diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index e8da633217..beb07dda65 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -48,6 +48,26 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat static FORCE_INLINE int32_t getPaddingRowSize(STableComInfo *tinfo) { return tinfo->rowSize + PAYLOAD_HEADER_LEN + PAYLOAD_ID_TYPE_LEN * tinfo->numOfColumns; } +int initSMemRowBuilder(SMemRowBuilder *pBuilder, SSchema *pSSchema, uint16_t nCols, uint16_t allNullColsLen) { + ASSERT(nCols > 0); + pBuilder->pSchema = pSSchema; + pBuilder->allNullLen = allNullColsLen; // TODO: get allNullColsLen when creating or altering table meta + if (pBuilder->allNullLen == 0) { + for (uint16_t i = 0; i < nCols; ++i) { + uint8_t type = pSSchema[i].type; + int32_t typeLen = TYPE_BYTES[type]; + ASSERT(typeLen > 0); + pBuilder->allNullLen += typeLen; + if (TSDB_DATA_TYPE_BINARY == type) { + pBuilder->allNullLen += (sizeof(VarDataLenT) + CHAR_BYTES); + } else if (TSDB_DATA_TYPE_NCHAR == type) { + int len = sizeof(VarDataLenT) + TSDB_NCHAR_SIZE; + pBuilder->allNullLen += len; + } + } + } + return 0; +} static int32_t tscToDouble(SStrToken *pToken, double *value, char **endPtr) { errno = 0; *value = strtold(pToken->z, endPtr); @@ -394,7 +414,7 @@ static FORCE_INLINE uint16_t tsSetColumnValue(char *payload, int16_t columnId, u } static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *payload, char *msg, char **str, - bool primaryKey, int16_t timePrec, uint16_t *sizeAppend, bool *isColNull, + bool primaryKey, int16_t timePrec, TDRowLenT *sizeAppend, bool *isColNull, TDRowLenT *dataRowColDeltaLen, TDRowLenT *kvRowColLen) { int64_t iv; int32_t ret; @@ -415,12 +435,12 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay // *(uint8_t *)payload = TSDB_TRUE; *sizeAppend = tsSetColumnValue(payload, pSchema->colId, pSchema->type, &TRUE_VALUE, TYPE_BYTES[TSDB_DATA_TYPE_BOOL]); - *kvRowColLen += sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_BOOL]; + *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_BOOL]); } else if (strncmp(pToken->z, "false", pToken->n) == 0) { // *(uint8_t *)payload = TSDB_FALSE; *sizeAppend = tsSetColumnValue(payload, pSchema->colId, pSchema->type, &FALSE_VALUE, TYPE_BYTES[TSDB_DATA_TYPE_BOOL]); - *kvRowColLen += sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_BOOL]; + *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_BOOL]); } else { return tscSQLSyntaxErrMsg(msg, "invalid bool data", pToken->z); } @@ -429,13 +449,13 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay // *(uint8_t *)payload = (int8_t)((iv == 0) ? TSDB_FALSE : TSDB_TRUE); *sizeAppend = tsSetColumnValue(payload, pSchema->colId, pSchema->type, ((iv == 0) ? &FALSE_VALUE : &TRUE_VALUE), TYPE_BYTES[TSDB_DATA_TYPE_BOOL]); - *kvRowColLen += sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_BOOL]; + *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_BOOL]); } else if (pToken->type == TK_FLOAT) { double dv = strtod(pToken->z, NULL); // *(uint8_t *)payload = (int8_t)((dv == 0) ? TSDB_FALSE : TSDB_TRUE); *sizeAppend = tsSetColumnValue(payload, pSchema->colId, pSchema->type, ((dv == 0) ? &FALSE_VALUE : &TRUE_VALUE), TYPE_BYTES[TSDB_DATA_TYPE_BOOL]); - *kvRowColLen += sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_BOOL]; + *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_BOOL]); } else { return tscInvalidOperationMsg(msg, "invalid bool data", pToken->z); } @@ -459,7 +479,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay uint8_t tmpVal = (uint8_t)iv; *sizeAppend = tsSetColumnValue(payload, pSchema->colId, pSchema->type, &tmpVal, TYPE_BYTES[TSDB_DATA_TYPE_TINYINT]); - *kvRowColLen += sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_TINYINT]; + *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_TINYINT]); } break; @@ -480,7 +500,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay uint8_t tmpVal = (uint8_t)iv; *sizeAppend = tsSetColumnValue(payload, pSchema->colId, pSchema->type, &tmpVal, TYPE_BYTES[TSDB_DATA_TYPE_UTINYINT]); - *kvRowColLen += sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_UTINYINT]; + *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_UTINYINT]); } break; @@ -500,7 +520,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay int16_t tmpVal = (int16_t)iv; *sizeAppend = tsSetColumnValue(payload, pSchema->colId, pSchema->type, &tmpVal, TYPE_BYTES[TSDB_DATA_TYPE_SMALLINT]); - *kvRowColLen += sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_SMALLINT]; + *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_SMALLINT]); } break; @@ -521,7 +541,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay uint16_t tmpVal = (uint16_t)iv; *sizeAppend = tsSetColumnValue(payload, pSchema->colId, pSchema->type, &tmpVal, TYPE_BYTES[TSDB_DATA_TYPE_USMALLINT]); - *kvRowColLen += sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_USMALLINT]; + *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_USMALLINT]); } break; @@ -541,7 +561,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay // *((int32_t *)payload) = (int32_t)iv; int32_t tmpVal = (int32_t)iv; *sizeAppend = tsSetColumnValue(payload, pSchema->colId, pSchema->type, &tmpVal, TYPE_BYTES[TSDB_DATA_TYPE_INT]); - *kvRowColLen += sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_INT]; + *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_INT]); } break; @@ -562,7 +582,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay uint32_t tmpVal = (uint32_t)iv; *sizeAppend = tsSetColumnValue(payload, pSchema->colId, pSchema->type, &tmpVal, TYPE_BYTES[TSDB_DATA_TYPE_UINT]); - *kvRowColLen += sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_UINT]; + *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_UINT]); } break; @@ -582,7 +602,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay // *((int64_t *)payload) = iv; // int64_t tmpVal = (int64_t)iv; *sizeAppend = tsSetColumnValue(payload, pSchema->colId, pSchema->type, &iv, TYPE_BYTES[TSDB_DATA_TYPE_BIGINT]); - *kvRowColLen += sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_BIGINT]; + *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_BIGINT]); } break; @@ -600,7 +620,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay // *((uint64_t *)payload) = iv; *sizeAppend = tsSetColumnValue(payload, pSchema->colId, pSchema->type, &iv, TYPE_BYTES[TSDB_DATA_TYPE_UBIGINT]); - *kvRowColLen += sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_UBIGINT]; + *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_UBIGINT]); } break; @@ -624,7 +644,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay float tmpVal = (float)dv; *sizeAppend = tsSetColumnValue(payload, pSchema->colId, pSchema->type, &tmpVal, TYPE_BYTES[TSDB_DATA_TYPE_FLOAT]); - *kvRowColLen += sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_FLOAT]; + *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_FLOAT]); } break; @@ -644,7 +664,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay // *((double *)payload) = dv; *sizeAppend = tsSetColumnValue(payload, pSchema->colId, pSchema->type, &dv, TYPE_BYTES[TSDB_DATA_TYPE_DOUBLE]); - *kvRowColLen += sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_DOUBLE]; + *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_DOUBLE]); } break; @@ -664,9 +684,9 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay payloadColSetType(payload, pSchema->type); varDataSetLen(payload + PAYLOAD_ID_TYPE_LEN, pToken->n); memcpy(varDataVal(payload + PAYLOAD_ID_TYPE_LEN), pToken->z, pToken->n); - *sizeAppend = PAYLOAD_ID_TYPE_LEN + sizeof(VarDataLenT) + pToken->n; - *dataRowColDeltaLen += (pToken->n - sizeof(uint8_t)); - *kvRowColLen += sizeof(SColIdx) + sizeof(VarDataLenT) + pToken->n; + *sizeAppend = (TDRowLenT)(PAYLOAD_ID_TYPE_LEN + sizeof(VarDataLenT) + pToken->n); + *dataRowColDeltaLen += (TDRowLenT)(pToken->n - sizeof(uint8_t)); + *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + sizeof(VarDataLenT) + pToken->n); } break; @@ -689,9 +709,9 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay varDataSetLen(payload + PAYLOAD_ID_TYPE_LEN, output); - *sizeAppend = PAYLOAD_ID_TYPE_LEN + sizeof(VarDataLenT) + output; - *dataRowColDeltaLen += (output - sizeof(uint32_t)); - *kvRowColLen += sizeof(SColIdx) + sizeof(VarDataLenT) + output; + *sizeAppend = (TDRowLenT)(PAYLOAD_ID_TYPE_LEN + sizeof(VarDataLenT) + output); + *dataRowColDeltaLen += (TDRowLenT)(output - sizeof(uint32_t)); + *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + sizeof(VarDataLenT) + output); } break; @@ -703,7 +723,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay int64_t tmpVal = 0; *sizeAppend = tsSetColumnValue(payload, pSchema->colId, pSchema->type, &tmpVal, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP]); - *kvRowColLen += sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP]; + *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP]); } else { // *((int64_t *)payload) = TSDB_DATA_BIGINT_NULL; *isColNull = true; @@ -717,7 +737,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay // *((int64_t *)payload) = tmpVal; *sizeAppend = tsSetColumnValue(payload, pSchema->colId, pSchema->type, &tmpVal, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP]); - *kvRowColLen += sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP]; + *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP]); } break; @@ -850,7 +870,7 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i bool isColNull = false; TDRowLenT dataRowDeltaColLen = 0; // When combine the data as SDataRow, the delta len between all NULL columns. TDRowLenT kvRowColLen = 0; - uint16_t colSizeAppended = 0; + TDRowLenT colSizeAppended = 0; int32_t ret = tsParseOneColumnKV(pSchema, &sToken, kvStart + PAYLOAD_HEADER_LEN, pInsertParam->msg, str, isPrimaryKey, timePrec, &colSizeAppended, &isColNull, &dataRowDeltaColLen, &kvRowColLen); @@ -1086,7 +1106,7 @@ int tscSortRemoveDataBlockDupRows(STableDataBlocks *dataBuf, SBlockKeyInfo *pBlk return TSDB_CODE_TSC_OUT_OF_MEMORY; } pBlkKeyInfo->pKeyTuple = (SBlockKeyTuple *)tmp; - pBlkKeyInfo->nBytesAlloc = curBlkTupleSize; + pBlkKeyInfo->nBytesAlloc = (int32_t)curBlkTupleSize; } memset(pBlkKeyInfo->pKeyTuple, 0, curBlkTupleSize); @@ -1107,6 +1127,7 @@ int tscSortRemoveDataBlockDupRows(STableDataBlocks *dataBuf, SBlockKeyInfo *pBlk } if (!dataBuf->ordered) { + pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; qsort(pBlkKeyTuple, nRows, sizeof(SBlockKeyTuple), rowDataCompar); pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; From 4a365c0d91137de824cfff791deb19038f996070 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 7 Jul 2021 11:22:46 +0800 Subject: [PATCH 17/67] compile err and bug fix --- src/client/src/tscParseInsert.c | 34 +++++++++++++++++++-------------- src/client/src/tscUtil.c | 3 --- src/common/inc/tdataformat.h | 5 +++-- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index beb07dda65..70d4f8979e 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -413,8 +413,8 @@ static FORCE_INLINE uint16_t tsSetColumnValue(char *payload, int16_t columnId, u return PAYLOAD_ID_TYPE_LEN + valueLen; } -static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *payload, char *msg, char **str, - bool primaryKey, int16_t timePrec, TDRowLenT *sizeAppend, bool *isColNull, +static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *primaryKeyStart, char *payload, char *msg, + char **str, bool primaryKey, int16_t timePrec, TDRowLenT *sizeAppend, bool *isColNull, TDRowLenT *dataRowColDeltaLen, TDRowLenT *kvRowColLen) { int64_t iv; int32_t ret; @@ -721,8 +721,9 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay // *((int64_t *)payload) = 0; // When building SKVRow primaryKey, we should not skip even with NULL value. int64_t tmpVal = 0; - *sizeAppend = - tsSetColumnValue(payload, pSchema->colId, pSchema->type, &tmpVal, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP]); + + *sizeAppend = tsSetColumnValue(primaryKeyStart, pSchema->colId, pSchema->type, &tmpVal, + TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP]); *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP]); } else { // *((int64_t *)payload) = TSDB_DATA_BIGINT_NULL; @@ -735,8 +736,9 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay } // *((int64_t *)payload) = tmpVal; - *sizeAppend = - tsSetColumnValue(payload, pSchema->colId, pSchema->type, &tmpVal, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP]); + + *sizeAppend = tsSetColumnValue(primaryKey ? primaryKeyStart : payload, pSchema->colId, pSchema->type, &tmpVal, + TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP]); *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP]); } @@ -801,7 +803,9 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i TDRowLenT dataRowLen = pBuilder->allNullLen; TDRowLenT kvRowLen = TD_MEM_ROW_KV_VER_SIZE; - char *kvStart = payload; + char *kvPrimayKeyStart = payload + PAYLOAD_HEADER_LEN; + char *kvStart = kvPrimayKeyStart + PLAYLOAD_PRIMARY_COL_LEN; // make sure 1st column tuple is primaryKey + for (int i = 0; i < spd->numOfBound; ++i) { // the start position in data block buffer of current value in sql int32_t colIndex = spd->boundedColumns[i]; // ordered @@ -871,9 +875,9 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i TDRowLenT dataRowDeltaColLen = 0; // When combine the data as SDataRow, the delta len between all NULL columns. TDRowLenT kvRowColLen = 0; TDRowLenT colSizeAppended = 0; - int32_t ret = - tsParseOneColumnKV(pSchema, &sToken, kvStart + PAYLOAD_HEADER_LEN, pInsertParam->msg, str, isPrimaryKey, - timePrec, &colSizeAppended, &isColNull, &dataRowDeltaColLen, &kvRowColLen); + // make sure the Primarykey locates in the 1st column + int32_t ret = tsParseOneColumnKV(pSchema, &sToken, kvPrimayKeyStart, kvStart, pInsertParam->msg, str, isPrimaryKey, + timePrec, &colSizeAppended, &isColNull, &dataRowDeltaColLen, &kvRowColLen); if (ret != TSDB_CODE_SUCCESS) { return ret; } @@ -888,7 +892,9 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i } kvRowLen += kvRowColLen; dataRowLen += dataRowDeltaColLen; - kvStart += colSizeAppended; // move to next column + if (!isPrimaryKey) { + kvStart += colSizeAppended; // move to next column + } rowSizeAppended += colSizeAppended; // calculate rowLen } @@ -1134,11 +1140,11 @@ int tscSortRemoveDataBlockDupRows(STableDataBlocks *dataBuf, SBlockKeyInfo *pBlk int32_t i = 0; int32_t j = 1; while (j < nRows) { - TSKEY ti = *(TSKEY *)(pBlkKeyTuple + sizeof(SBlockKeyTuple) * i); - TSKEY tj = *(TSKEY *)(pBlkKeyTuple + sizeof(SBlockKeyTuple) * j); + TSKEY ti = (pBlkKeyTuple + i)->skey; + TSKEY tj = (pBlkKeyTuple + j)->skey; if (ti == tj) { - totolPayloadLen -= payloadTLen(pBlkKeyTuple + sizeof(SBlockKeyTuple) * j); + totolPayloadLen -= payloadTLen(pBlkKeyTuple + j); ++j; continue; } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 8f3455814f..9870262b65 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1703,7 +1703,6 @@ static SMemRow tdGenMemRowFromBuilder(SMemRowBuilder* pBuilder) { ++i; } - pBuilder->buf = p; } else if (memRowType == SMEM_ROW_KV) { ASSERT(nColsNotNull <= pBuilder->nCols); SKVRow kvRow = (SKVRow)memRowKvBody(memRow); @@ -1723,8 +1722,6 @@ static SMemRow tdGenMemRowFromBuilder(SMemRowBuilder* pBuilder) { ++i; } - pBuilder->buf = p; - } else { ASSERT(0); } diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index 49a0c0f120..dd35e94cfc 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -620,6 +620,7 @@ static FORCE_INLINE void *tdGetMemRowDataOfCol(void *row, int8_t type, int32_t o #define PAYLOAD_HEADER_LEN (PAYLOAD_NCOLS_OFFSET + PAYLOAD_NCOLS_LEN) #define PAYLOAD_ID_LEN sizeof(int16_t) #define PAYLOAD_ID_TYPE_LEN (sizeof(int16_t) + sizeof(uint8_t)) +#define PLAYLOAD_PRIMARY_COL_LEN (PAYLOAD_ID_TYPE_LEN + sizeof(TSKEY)) #define payloadBody(r) POINTER_SHIFT(r, PAYLOAD_HEADER_LEN) #define payloadType(r) (*(uint8_t *)(r)) @@ -643,9 +644,9 @@ static FORCE_INLINE void *tdGetMemRowDataOfCol(void *row, int8_t type, int32_t o static FORCE_INLINE char *skipToNextEles(char *p) { uint8_t colType = payloadColType(p); if (IS_VAR_DATA_TYPE(colType)) { - return POINTER_SHIFT(p, PAYLOAD_ID_TYPE_LEN + varDataTLen(payloadColValue(p))); + return (char *)POINTER_SHIFT(p, PAYLOAD_ID_TYPE_LEN + varDataTLen(payloadColValue(p))); } else { - return POINTER_SHIFT(p, PAYLOAD_ID_TYPE_LEN + TYPE_BYTES[colType]); + return (char *)POINTER_SHIFT(p, PAYLOAD_ID_TYPE_LEN + TYPE_BYTES[colType]); } } From 16a05a6914e1ff4892c1300affade3026b50e8c6 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 7 Jul 2021 18:30:13 +0800 Subject: [PATCH 18/67] naming convention --- src/client/inc/tscUtil.h | 2 +- src/client/src/tscParseInsert.c | 101 ++++++++++++-------------------- src/client/src/tscUtil.c | 16 ++--- src/common/inc/tdataformat.h | 2 +- 4 files changed, 48 insertions(+), 73 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index e62661e6f6..049d03a438 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -103,7 +103,7 @@ typedef struct SBlockKeyTuple { } SBlockKeyTuple; typedef struct SBlockKeyInfo { - int32_t nBytesAlloc; + int32_t maxBytesAlloc; SBlockKeyTuple* pKeyTuple; } SBlockKeyInfo; diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 70d4f8979e..dd5980053c 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -45,7 +45,7 @@ static int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSiz static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDataColInfo *pColInfo, SSchema *pSchema, char *str, char **end); -static FORCE_INLINE int32_t getPaddingRowSize(STableComInfo *tinfo) { +static FORCE_INLINE int32_t getExtendedRowSize(STableComInfo *tinfo) { return tinfo->rowSize + PAYLOAD_HEADER_LEN + PAYLOAD_ID_TYPE_LEN * tinfo->numOfColumns; } int initSMemRowBuilder(SMemRowBuilder *pBuilder, SSchema *pSSchema, uint16_t nCols, uint16_t allNullColsLen) { @@ -427,17 +427,14 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pri switch (pSchema->type) { case TSDB_DATA_TYPE_BOOL: { // bool if (isNullStr(pToken)) { - // *((uint8_t *)payload) = TSDB_DATA_BOOL_NULL; *isColNull = true; } else { if ((pToken->type == TK_BOOL || pToken->type == TK_STRING) && (pToken->n != 0)) { if (strncmp(pToken->z, "true", pToken->n) == 0) { - // *(uint8_t *)payload = TSDB_TRUE; *sizeAppend = tsSetColumnValue(payload, pSchema->colId, pSchema->type, &TRUE_VALUE, TYPE_BYTES[TSDB_DATA_TYPE_BOOL]); *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_BOOL]); } else if (strncmp(pToken->z, "false", pToken->n) == 0) { - // *(uint8_t *)payload = TSDB_FALSE; *sizeAppend = tsSetColumnValue(payload, pSchema->colId, pSchema->type, &FALSE_VALUE, TYPE_BYTES[TSDB_DATA_TYPE_BOOL]); *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_BOOL]); @@ -446,13 +443,11 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pri } } else if (pToken->type == TK_INTEGER) { iv = strtoll(pToken->z, NULL, 10); - // *(uint8_t *)payload = (int8_t)((iv == 0) ? TSDB_FALSE : TSDB_TRUE); *sizeAppend = tsSetColumnValue(payload, pSchema->colId, pSchema->type, ((iv == 0) ? &FALSE_VALUE : &TRUE_VALUE), TYPE_BYTES[TSDB_DATA_TYPE_BOOL]); *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_BOOL]); } else if (pToken->type == TK_FLOAT) { double dv = strtod(pToken->z, NULL); - // *(uint8_t *)payload = (int8_t)((dv == 0) ? TSDB_FALSE : TSDB_TRUE); *sizeAppend = tsSetColumnValue(payload, pSchema->colId, pSchema->type, ((dv == 0) ? &FALSE_VALUE : &TRUE_VALUE), TYPE_BYTES[TSDB_DATA_TYPE_BOOL]); *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_BOOL]); @@ -465,7 +460,6 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pri case TSDB_DATA_TYPE_TINYINT: if (isNullStr(pToken)) { - // *((uint8_t *)payload) = TSDB_DATA_TINYINT_NULL; *isColNull = true; } else { ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); @@ -475,7 +469,6 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pri return tscInvalidOperationMsg(msg, "data overflow", pToken->z); } - // *((uint8_t *)payload) = (uint8_t)iv; uint8_t tmpVal = (uint8_t)iv; *sizeAppend = tsSetColumnValue(payload, pSchema->colId, pSchema->type, &tmpVal, TYPE_BYTES[TSDB_DATA_TYPE_TINYINT]); @@ -486,7 +479,6 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pri case TSDB_DATA_TYPE_UTINYINT: if (isNullStr(pToken)) { - // *((uint8_t *)payload) = TSDB_DATA_UTINYINT_NULL; *isColNull = true; } else { ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); @@ -496,7 +488,6 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pri return tscInvalidOperationMsg(msg, "unsigned tinyint data overflow", pToken->z); } - // *((uint8_t *)payload) = (uint8_t)iv; uint8_t tmpVal = (uint8_t)iv; *sizeAppend = tsSetColumnValue(payload, pSchema->colId, pSchema->type, &tmpVal, TYPE_BYTES[TSDB_DATA_TYPE_UTINYINT]); @@ -527,7 +518,6 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pri case TSDB_DATA_TYPE_USMALLINT: if (isNullStr(pToken)) { - // *((uint16_t *)payload) = TSDB_DATA_USMALLINT_NULL; *isColNull = true; } else { ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); @@ -537,7 +527,6 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pri return tscInvalidOperationMsg(msg, "unsigned smallint data overflow", pToken->z); } - // *((uint16_t *)payload) = (uint16_t)iv; uint16_t tmpVal = (uint16_t)iv; *sizeAppend = tsSetColumnValue(payload, pSchema->colId, pSchema->type, &tmpVal, TYPE_BYTES[TSDB_DATA_TYPE_USMALLINT]); @@ -548,7 +537,6 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pri case TSDB_DATA_TYPE_INT: if (isNullStr(pToken)) { - // *((int32_t *)payload) = TSDB_DATA_INT_NULL; *isColNull = true; } else { ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); @@ -558,7 +546,6 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pri return tscInvalidOperationMsg(msg, "int data overflow", pToken->z); } - // *((int32_t *)payload) = (int32_t)iv; int32_t tmpVal = (int32_t)iv; *sizeAppend = tsSetColumnValue(payload, pSchema->colId, pSchema->type, &tmpVal, TYPE_BYTES[TSDB_DATA_TYPE_INT]); *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_INT]); @@ -568,7 +555,6 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pri case TSDB_DATA_TYPE_UINT: if (isNullStr(pToken)) { - // *((uint32_t *)payload) = TSDB_DATA_UINT_NULL; *isColNull = true; } else { ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); @@ -578,7 +564,6 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pri return tscInvalidOperationMsg(msg, "unsigned int data overflow", pToken->z); } - // *((uint32_t *)payload) = (uint32_t)iv; uint32_t tmpVal = (uint32_t)iv; *sizeAppend = tsSetColumnValue(payload, pSchema->colId, pSchema->type, &tmpVal, TYPE_BYTES[TSDB_DATA_TYPE_UINT]); @@ -589,7 +574,6 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pri case TSDB_DATA_TYPE_BIGINT: if (isNullStr(pToken)) { - // *((int64_t *)payload) = TSDB_DATA_BIGINT_NULL; *isColNull = true; } else { ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); @@ -599,8 +583,6 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pri return tscInvalidOperationMsg(msg, "bigint data overflow", pToken->z); } - // *((int64_t *)payload) = iv; - // int64_t tmpVal = (int64_t)iv; *sizeAppend = tsSetColumnValue(payload, pSchema->colId, pSchema->type, &iv, TYPE_BYTES[TSDB_DATA_TYPE_BIGINT]); *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_BIGINT]); } @@ -608,7 +590,6 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pri case TSDB_DATA_TYPE_UBIGINT: if (isNullStr(pToken)) { - // *((uint64_t *)payload) = TSDB_DATA_UBIGINT_NULL; *isColNull = true; } else { ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); @@ -618,15 +599,15 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pri return tscInvalidOperationMsg(msg, "unsigned bigint data overflow", pToken->z); } - // *((uint64_t *)payload) = iv; - *sizeAppend = tsSetColumnValue(payload, pSchema->colId, pSchema->type, &iv, TYPE_BYTES[TSDB_DATA_TYPE_UBIGINT]); + uint64_t tmpVal = (uint64_t)iv; + *sizeAppend = + tsSetColumnValue(payload, pSchema->colId, pSchema->type, &tmpVal, TYPE_BYTES[TSDB_DATA_TYPE_UBIGINT]); *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_UBIGINT]); } break; case TSDB_DATA_TYPE_FLOAT: if (isNullStr(pToken)) { - // *((int32_t *)payload) = TSDB_DATA_FLOAT_NULL; *isColNull = true; } else { double dv; @@ -639,8 +620,6 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pri return tscInvalidOperationMsg(msg, "illegal float data", pToken->z); } - // *((float *)payload) = (float)dv; - // SET_FLOAT_VAL(payload, dv); float tmpVal = (float)dv; *sizeAppend = tsSetColumnValue(payload, pSchema->colId, pSchema->type, &tmpVal, TYPE_BYTES[TSDB_DATA_TYPE_FLOAT]); @@ -650,7 +629,6 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pri case TSDB_DATA_TYPE_DOUBLE: if (isNullStr(pToken)) { - // *((int64_t *)payload) = TSDB_DATA_DOUBLE_NULL; *isColNull = true; } else { double dv; @@ -662,7 +640,6 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pri return tscInvalidOperationMsg(msg, "illegal double data", pToken->z); } - // *((double *)payload) = dv; *sizeAppend = tsSetColumnValue(payload, pSchema->colId, pSchema->type, &dv, TYPE_BYTES[TSDB_DATA_TYPE_DOUBLE]); *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_DOUBLE]); } @@ -682,43 +659,41 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pri payloadColSetId(payload, pSchema->colId); payloadColSetType(payload, pSchema->type); - varDataSetLen(payload + PAYLOAD_ID_TYPE_LEN, pToken->n); - memcpy(varDataVal(payload + PAYLOAD_ID_TYPE_LEN), pToken->z, pToken->n); - *sizeAppend = (TDRowLenT)(PAYLOAD_ID_TYPE_LEN + sizeof(VarDataLenT) + pToken->n); + varDataSetLen(payloadColValue(payload), pToken->n); + memcpy(varDataVal(payloadColValue(payload)), pToken->z, pToken->n); + *sizeAppend = (TDRowLenT)(PAYLOAD_ID_TYPE_LEN + VARSTR_HEADER_SIZE + pToken->n); *dataRowColDeltaLen += (TDRowLenT)(pToken->n - sizeof(uint8_t)); - *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + sizeof(VarDataLenT) + pToken->n); + *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + VARSTR_HEADER_SIZE + pToken->n); } break; case TSDB_DATA_TYPE_NCHAR: if (pToken->type == TK_NULL) { - // setVardataNull(payload, TSDB_DATA_TYPE_NCHAR); *isColNull = true; } else { // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' int32_t output = 0; payloadColSetId(payload, pSchema->colId); payloadColSetType(payload, pSchema->type); - if (!taosMbsToUcs4(pToken->z, pToken->n, varDataVal(payload + PAYLOAD_ID_TYPE_LEN), + if (!taosMbsToUcs4(pToken->z, pToken->n, varDataVal(payloadColValue(payload)), pSchema->bytes - VARSTR_HEADER_SIZE, &output)) { char buf[512] = {0}; snprintf(buf, tListLen(buf), "%s", strerror(errno)); return tscInvalidOperationMsg(msg, buf, pToken->z); } - varDataSetLen(payload + PAYLOAD_ID_TYPE_LEN, output); + varDataSetLen(payloadColValue(payload), output); - *sizeAppend = (TDRowLenT)(PAYLOAD_ID_TYPE_LEN + sizeof(VarDataLenT) + output); + *sizeAppend = (TDRowLenT)(PAYLOAD_ID_TYPE_LEN + VARSTR_HEADER_SIZE + output); *dataRowColDeltaLen += (TDRowLenT)(output - sizeof(uint32_t)); - *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + sizeof(VarDataLenT) + output); + *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + VARSTR_HEADER_SIZE + output); } break; case TSDB_DATA_TYPE_TIMESTAMP: { if (pToken->type == TK_NULL) { if (primaryKey) { - // *((int64_t *)payload) = 0; // When building SKVRow primaryKey, we should not skip even with NULL value. int64_t tmpVal = 0; @@ -726,7 +701,6 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pri TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP]); *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP]); } else { - // *((int64_t *)payload) = TSDB_DATA_BIGINT_NULL; *isColNull = true; } } else { @@ -735,8 +709,6 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pri return tscInvalidOperationMsg(msg, "invalid timestamp", pToken->z); } - // *((int64_t *)payload) = tmpVal; - *sizeAppend = tsSetColumnValue(primaryKey ? primaryKeyStart : payload, pSchema->colId, pSchema->type, &tmpVal, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP]); *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP]); @@ -803,8 +775,10 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i TDRowLenT dataRowLen = pBuilder->allNullLen; TDRowLenT kvRowLen = TD_MEM_ROW_KV_VER_SIZE; - char *kvPrimayKeyStart = payload + PAYLOAD_HEADER_LEN; - char *kvStart = kvPrimayKeyStart + PLAYLOAD_PRIMARY_COL_LEN; // make sure 1st column tuple is primaryKey + ASSERT(dataRowLen > 0); + + char *kvPrimayKeyStart = payload + PAYLOAD_HEADER_LEN; // primaryKey in 1st column tuple + char *kvStart = kvPrimayKeyStart + PAYLOAD_PRIMARY_COL_LEN; // the column tuple behind the primaryKey for (int i = 0; i < spd->numOfBound; ++i) { // the start position in data block buffer of current value in sql @@ -882,8 +856,7 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i return ret; } - if (isPrimaryKey && - tsCheckTimestamp(pDataBlocks, payloadKeyAddr(kvStart)) != TSDB_CODE_SUCCESS) { + if (isPrimaryKey && tsCheckTimestamp(pDataBlocks, payloadColValue(kvPrimayKeyStart)) != TSDB_CODE_SUCCESS) { tscInvalidOperationMsg(pInsertParam->msg, "client time/server time can not be mixed up", sToken.z); return TSDB_CODE_TSC_INVALID_TIME_STAMP; } @@ -904,7 +877,6 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i payloadSetType(payload, SMEM_ROW_DATA); } - *(uint16_t *)(payload + sizeof(uint8_t)) = nColsNotNull; *len = PAYLOAD_HEADER_LEN + rowSizeAppended; payloadSetNCols(payload, nColsNotNull); @@ -938,7 +910,7 @@ int32_t tsParseValues(char **str, STableDataBlocks *pDataBlock, int maxRows, SIn int32_t precision = tinfo.precision; - int32_t rowSizeWithColIdType = getPaddingRowSize(&tinfo); + int32_t extendedRowSize = getExtendedRowSize(&tinfo); initSMemRowBuilder(&pDataBlock->rowBuilder, tscGetTableSchema(pDataBlock->pTableMeta), tscGetNumOfColumns(pDataBlock->pTableMeta), 0); @@ -949,9 +921,9 @@ int32_t tsParseValues(char **str, STableDataBlocks *pDataBlock, int maxRows, SIn if (sToken.n == 0 || sToken.type != TK_LP) break; *str += index; - if ((*numOfRows) >= maxRows || pDataBlock->size + rowSizeWithColIdType >= pDataBlock->nAllocSize) { + if ((*numOfRows) >= maxRows || pDataBlock->size + extendedRowSize >= pDataBlock->nAllocSize) { int32_t tSize; - code = tscAllocateMemIfNeed(pDataBlock, rowSizeWithColIdType, &tSize); + code = tscAllocateMemIfNeed(pDataBlock, extendedRowSize, &tSize); if (code != TSDB_CODE_SUCCESS) { //TODO pass the correct error code to client strcpy(pInsertParam->msg, "client out of memory"); return TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -1048,8 +1020,9 @@ static int32_t tsSetBlockInfo(SSubmitBlk *pBlocks, const STableMeta *pTableMeta, } } +#if 0 // data block is disordered, sort it in ascending order -void tscSortRemoveDataBlockDupRowsXX(STableDataBlocks *dataBuf) { +static void tscSortRemoveDataBlockDupRowsOld(STableDataBlocks *dataBuf) { SSubmitBlk *pBlocks = (SSubmitBlk *)dataBuf->pData; // size is less than the total size, since duplicated rows may be removed yet. @@ -1092,12 +1065,13 @@ void tscSortRemoveDataBlockDupRowsXX(STableDataBlocks *dataBuf) { dataBuf->prevTS = INT64_MIN; } +#endif // data block is disordered, sort it in ascending order int tscSortRemoveDataBlockDupRows(STableDataBlocks *dataBuf, SBlockKeyInfo *pBlkKeyInfo) { SSubmitBlk *pBlocks = (SSubmitBlk *)dataBuf->pData; - int16_t nRows = pBlocks->numOfRows; - + int16_t nRows = pBlocks->numOfRows; + // size is less than the total size, since duplicated rows may be removed yet. // if use server time, this block must be ordered @@ -1105,22 +1079,23 @@ int tscSortRemoveDataBlockDupRows(STableDataBlocks *dataBuf, SBlockKeyInfo *pBlk assert(dataBuf->ordered); } // allocate memory - size_t curBlkTupleSize = nRows * sizeof(SBlockKeyTuple); - if (pBlkKeyInfo->pKeyTuple == NULL || pBlkKeyInfo->nBytesAlloc < curBlkTupleSize) { - char *tmp = realloc(pBlkKeyInfo->pKeyTuple, curBlkTupleSize); + size_t nAlloc = nRows * sizeof(SBlockKeyTuple); + if (pBlkKeyInfo->pKeyTuple == NULL || pBlkKeyInfo->maxBytesAlloc < nAlloc) { + size_t nRealAlloc = nAlloc + 10 * sizeof(SBlockKeyTuple); + char * tmp = realloc(pBlkKeyInfo->pKeyTuple, nRealAlloc); if (tmp == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_TSC_OUT_OF_MEMORY; } pBlkKeyInfo->pKeyTuple = (SBlockKeyTuple *)tmp; - pBlkKeyInfo->nBytesAlloc = (int32_t)curBlkTupleSize; + pBlkKeyInfo->maxBytesAlloc = (int32_t)nRealAlloc; } - memset(pBlkKeyInfo->pKeyTuple, 0, curBlkTupleSize); + memset(pBlkKeyInfo->pKeyTuple, 0, nAlloc); SBlockKeyTuple *pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; - char * pBlockData = pBlocks->data; - int n = 0; - uint32_t totolPayloadLen = 0; - TDRowLenT payloadTLen = 0; + char * pBlockData = pBlocks->data; + uint32_t totolPayloadLen = 0; + TDRowLenT payloadTLen = 0; + int n = 0; while (n < nRows) { pBlkKeyTuple->skey = payloadKey(pBlockData); pBlkKeyTuple->payloadAddr = pBlockData; @@ -1170,7 +1145,7 @@ static int32_t doParseInsertStatement(SInsertStatementParam *pInsertParam, char STableComInfo tinfo = tscGetTableInfo(dataBuf->pTableMeta); int32_t maxNumOfRows; - int32_t code = tscAllocateMemIfNeed(dataBuf, getPaddingRowSize(&tinfo), &maxNumOfRows); + int32_t code = tscAllocateMemIfNeed(dataBuf, getExtendedRowSize(&tinfo), &maxNumOfRows); if (TSDB_CODE_SUCCESS != code) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -1962,7 +1937,7 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int32_t numOfRow goto _error; } - tscAllocateMemIfNeed(pTableDataBlock, getPaddingRowSize(&tinfo), &maxRows); + tscAllocateMemIfNeed(pTableDataBlock, getExtendedRowSize(&tinfo), &maxRows); tokenBuf = calloc(1, TSDB_MAX_BYTES_PER_ROW); if (tokenBuf == NULL) { code = TSDB_CODE_TSC_OUT_OF_MEMORY; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 9870262b65..8a3e8cad32 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1796,7 +1796,8 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, bo } static int32_t getRowExpandSize(STableMeta* pTableMeta) { - int32_t result = TD_DATA_ROW_HEAD_SIZE; + // add prefix len of KV type SMemRow(we may use SDataRow or SKVRow) + int32_t result = TD_DATA_ROW_HEAD_SIZE + TD_MEM_ROW_KV_TYPE_VER_SIZE; int32_t columns = tscGetNumOfColumns(pTableMeta); SSchema* pSchema = tscGetTableSchema(pTableMeta); for(int32_t i = 0; i < columns; i++) { @@ -1804,7 +1805,6 @@ static int32_t getRowExpandSize(STableMeta* pTableMeta) { result += TYPE_BYTES[TSDB_DATA_TYPE_BINARY]; } } - result += TD_MEM_ROW_KV_TYPE_VER_SIZE; // add prefix len of KV type SMemRow(we may use SDataRow or SKVRow) return result; } @@ -1835,7 +1835,7 @@ int32_t tscMergeTableDataBlocks(SInsertStatementParam *pInsertParam, bool freeBl const int INSERT_HEAD_SIZE = sizeof(SMsgDesc) + sizeof(SSubmitMsg); int code = 0; void* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); - SArray* pVnodeDataBlockList = taosArrayInit(8, POINTER_BYTES); + SArray* pVnodeDataBlockList = taosArrayInit(8, POINTER_BYTES); STableDataBlocks** p = taosHashIterate(pInsertParam->pTableBlockHashList, NULL); @@ -1883,11 +1883,11 @@ int32_t tscMergeTableDataBlocks(SInsertStatementParam *pInsertParam, bool freeBl } } - if((code = tscSortRemoveDataBlockDupRows(pOneTableBlock, &blkKeyInfo)) != 0){ - taosHashCleanup(pVnodeDataBlockHashList); - tscDestroyBlockArrayList(pVnodeDataBlockList); - tfree(dataBuf->pData); - tfree(blkKeyInfo.pKeyTuple); + if ((code = tscSortRemoveDataBlockDupRows(pOneTableBlock, &blkKeyInfo)) != 0) { + taosHashCleanup(pVnodeDataBlockHashList); + tscDestroyBlockArrayList(pVnodeDataBlockList); + tfree(dataBuf->pData); + tfree(blkKeyInfo.pKeyTuple); return code; } diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index dd35e94cfc..093b7c5188 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -620,7 +620,7 @@ static FORCE_INLINE void *tdGetMemRowDataOfCol(void *row, int8_t type, int32_t o #define PAYLOAD_HEADER_LEN (PAYLOAD_NCOLS_OFFSET + PAYLOAD_NCOLS_LEN) #define PAYLOAD_ID_LEN sizeof(int16_t) #define PAYLOAD_ID_TYPE_LEN (sizeof(int16_t) + sizeof(uint8_t)) -#define PLAYLOAD_PRIMARY_COL_LEN (PAYLOAD_ID_TYPE_LEN + sizeof(TSKEY)) +#define PAYLOAD_PRIMARY_COL_LEN (PAYLOAD_ID_TYPE_LEN + sizeof(TSKEY)) #define payloadBody(r) POINTER_SHIFT(r, PAYLOAD_HEADER_LEN) #define payloadType(r) (*(uint8_t *)(r)) From 1cd331a37618388b8efa1be7822736963c97cf28 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sun, 11 Jul 2021 14:44:13 +0800 Subject: [PATCH 19/67] payload refactor to support 4096 --- src/client/inc/tscUtil.h | 3 - src/client/inc/tsclient.h | 30 ++- src/client/src/tscParseInsert.c | 349 +++++++++++++++++++++----------- src/client/src/tscUtil.c | 98 ++++----- src/common/inc/tdataformat.h | 105 +++++----- src/inc/taosdef.h | 6 +- src/inc/ttype.h | 7 +- src/tsdb/src/tsdbMemTable.c | 2 +- 8 files changed, 365 insertions(+), 235 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index c962a1ef42..f1a7af01a5 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -40,9 +40,6 @@ extern "C" { #define UTIL_TABLE_IS_TMP_TABLE(metaInfo) \ (((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_TEMP_TABLE)) -#define KvRowNColsThresh 1 // default 1200 -#define KVRowRatio 0.85 // for NonVarType, we get value from SDataRow directly, while needs readdressing for SKVRow - #pragma pack(push,1) // this struct is transfered as binary, padding two bytes to avoid // an 'uid' whose low bytes is 0xff being recoginized as NULL, diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 3771dbe575..c9428504b8 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -41,6 +41,9 @@ extern "C" { // forward declaration struct SSqlInfo; +#define KvRowNColsThresh 128 // default 128 +#define KVRowRatio 0.9 // for NonVarType, we get value from SDataRow directly, while needs readdressing for SKVRow + typedef void (*__async_cb_func_t)(void *param, TAOS_RES *tres, int32_t numOfRows); typedef struct SNewVgroupInfo { @@ -87,12 +90,18 @@ typedef struct SBoundColumn { bool hasVal; // denote if current column has bound or not int32_t offset; // all column offset value } SBoundColumn; - +typedef struct { + uint16_t schemaColIdx; + uint16_t boundIdx; + uint16_t finalIdx; +} SBoundIdxInfo; typedef struct SParsedDataColInfo { - int16_t numOfCols; - int16_t numOfBound; - int32_t *boundedColumns; - SBoundColumn *cols; + bool isOrdered; // bounded columns + int16_t numOfCols; + int16_t numOfBound; + int32_t * boundedColumns; // bounded column idx according to schema + SBoundColumn * cols; + SBoundIdxInfo *colIdxInfo; } SParsedDataColInfo; typedef struct { @@ -107,9 +116,12 @@ typedef struct { void * pDataBlock; SSubmitBlk *pSubmitBlk; - uint16_t allNullLen; } SMemRowBuilder; +typedef struct { + TDRowLenT allNullLen; +} SMemRowHelper; + typedef struct STableDataBlocks { SName tableName; int8_t tsSource; // where does the UNIX timestamp come from, server or client @@ -130,7 +142,7 @@ typedef struct STableDataBlocks { uint32_t numOfAllocedParams; uint32_t numOfParams; SParamInfo * params; - SMemRowBuilder rowBuilder; + SMemRowHelper rowHelper; } STableDataBlocks; typedef struct { @@ -398,7 +410,7 @@ extern int tscRefId; extern int tscNumOfObj; // number of existed sqlObj in current process. extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo); - + void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArray* tables); int16_t getNewResColId(SSqlCmd* pCmd); @@ -406,4 +418,4 @@ int16_t getNewResColId(SSqlCmd* pCmd); } #endif -#endif +#endif \ No newline at end of file diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index dd5980053c..f0a9421f30 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -46,23 +46,22 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat char *str, char **end); static FORCE_INLINE int32_t getExtendedRowSize(STableComInfo *tinfo) { - return tinfo->rowSize + PAYLOAD_HEADER_LEN + PAYLOAD_ID_TYPE_LEN * tinfo->numOfColumns; + return tinfo->rowSize + PAYLOAD_HEADER_LEN + PAYLOAD_COL_HEAD_LEN * tinfo->numOfColumns; } -int initSMemRowBuilder(SMemRowBuilder *pBuilder, SSchema *pSSchema, uint16_t nCols, uint16_t allNullColsLen) { +int initSMemRowHelper(SMemRowHelper *pHelper, SSchema *pSSchema, uint16_t nCols, uint16_t allNullColsLen) { ASSERT(nCols > 0); - pBuilder->pSchema = pSSchema; - pBuilder->allNullLen = allNullColsLen; // TODO: get allNullColsLen when creating or altering table meta - if (pBuilder->allNullLen == 0) { + pHelper->allNullLen = allNullColsLen; // TODO: get allNullColsLen when creating or altering table meta + if (pHelper->allNullLen == 0) { for (uint16_t i = 0; i < nCols; ++i) { uint8_t type = pSSchema[i].type; int32_t typeLen = TYPE_BYTES[type]; ASSERT(typeLen > 0); - pBuilder->allNullLen += typeLen; + pHelper->allNullLen += typeLen; if (TSDB_DATA_TYPE_BINARY == type) { - pBuilder->allNullLen += (sizeof(VarDataLenT) + CHAR_BYTES); + pHelper->allNullLen += (sizeof(VarDataLenT) + CHAR_BYTES); } else if (TSDB_DATA_TYPE_NCHAR == type) { int len = sizeof(VarDataLenT) + TSDB_NCHAR_SIZE; - pBuilder->allNullLen += len; + pHelper->allNullLen += len; } } } @@ -404,18 +403,19 @@ int32_t tsParseOneColumn(SSchema *pSchema, SStrToken *pToken, char *payload, cha return TSDB_CODE_SUCCESS; } -static FORCE_INLINE uint16_t tsSetColumnValue(char *payload, int16_t columnId, uint8_t columnType, void *value, - uint16_t valueLen) { +static FORCE_INLINE TDRowLenT tsSetPayloadColValue(char *payloadStart, char *payload, int16_t columnId, + uint8_t columnType, const void *value, uint16_t valueLen, TDRowTLenT tOffset) { payloadColSetId(payload, columnId); payloadColSetType(payload, columnType); - - memcpy(payloadColValue(payload), value, valueLen); - return PAYLOAD_ID_TYPE_LEN + valueLen; + memcpy(POINTER_SHIFT(payloadStart,tOffset), value, valueLen); + payloadSetTLen(payloadStart, payloadTLen(payloadStart) + valueLen); + return valueLen; } -static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *primaryKeyStart, char *payload, char *msg, - char **str, bool primaryKey, int16_t timePrec, TDRowLenT *sizeAppend, bool *isColNull, - TDRowLenT *dataRowColDeltaLen, TDRowLenT *kvRowColLen) { +static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *payloadStart, char *primaryKeyStart, + char *payload, char *msg, char **str, bool primaryKey, int16_t timePrec, + TDRowTLenT tOffset, TDRowLenT *sizeAppend, TDRowLenT *dataRowColDeltaLen, + TDRowLenT *kvRowColLen) { int64_t iv; int32_t ret; char * endptr = NULL; @@ -427,29 +427,30 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pri switch (pSchema->type) { case TSDB_DATA_TYPE_BOOL: { // bool if (isNullStr(pToken)) { - *isColNull = true; + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, + tdGetNullVal(TSDB_DATA_TYPE_BOOL), TYPE_BYTES[TSDB_DATA_TYPE_BOOL], tOffset); } else { if ((pToken->type == TK_BOOL || pToken->type == TK_STRING) && (pToken->n != 0)) { if (strncmp(pToken->z, "true", pToken->n) == 0) { - *sizeAppend = - tsSetColumnValue(payload, pSchema->colId, pSchema->type, &TRUE_VALUE, TYPE_BYTES[TSDB_DATA_TYPE_BOOL]); + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &TRUE_VALUE, + TYPE_BYTES[TSDB_DATA_TYPE_BOOL], tOffset); *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_BOOL]); } else if (strncmp(pToken->z, "false", pToken->n) == 0) { - *sizeAppend = - tsSetColumnValue(payload, pSchema->colId, pSchema->type, &FALSE_VALUE, TYPE_BYTES[TSDB_DATA_TYPE_BOOL]); + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &FALSE_VALUE, + TYPE_BYTES[TSDB_DATA_TYPE_BOOL], tOffset); *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_BOOL]); } else { return tscSQLSyntaxErrMsg(msg, "invalid bool data", pToken->z); } } else if (pToken->type == TK_INTEGER) { iv = strtoll(pToken->z, NULL, 10); - *sizeAppend = tsSetColumnValue(payload, pSchema->colId, pSchema->type, - ((iv == 0) ? &FALSE_VALUE : &TRUE_VALUE), TYPE_BYTES[TSDB_DATA_TYPE_BOOL]); + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, + ((iv == 0) ? &FALSE_VALUE : &TRUE_VALUE), TYPE_BYTES[TSDB_DATA_TYPE_BOOL], tOffset); *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_BOOL]); } else if (pToken->type == TK_FLOAT) { double dv = strtod(pToken->z, NULL); - *sizeAppend = tsSetColumnValue(payload, pSchema->colId, pSchema->type, - ((dv == 0) ? &FALSE_VALUE : &TRUE_VALUE), TYPE_BYTES[TSDB_DATA_TYPE_BOOL]); + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, + ((dv == 0) ? &FALSE_VALUE : &TRUE_VALUE), TYPE_BYTES[TSDB_DATA_TYPE_BOOL], tOffset); *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_BOOL]); } else { return tscInvalidOperationMsg(msg, "invalid bool data", pToken->z); @@ -460,7 +461,8 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pri case TSDB_DATA_TYPE_TINYINT: if (isNullStr(pToken)) { - *isColNull = true; + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, + tdGetNullVal(TSDB_DATA_TYPE_TINYINT), TYPE_BYTES[TSDB_DATA_TYPE_TINYINT], tOffset); } else { ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); if (ret != TSDB_CODE_SUCCESS) { @@ -470,8 +472,8 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pri } uint8_t tmpVal = (uint8_t)iv; - *sizeAppend = - tsSetColumnValue(payload, pSchema->colId, pSchema->type, &tmpVal, TYPE_BYTES[TSDB_DATA_TYPE_TINYINT]); + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &tmpVal, + TYPE_BYTES[TSDB_DATA_TYPE_TINYINT], tOffset); *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_TINYINT]); } @@ -479,7 +481,8 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pri case TSDB_DATA_TYPE_UTINYINT: if (isNullStr(pToken)) { - *isColNull = true; + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, + tdGetNullVal(TSDB_DATA_TYPE_UTINYINT), TYPE_BYTES[TSDB_DATA_TYPE_UTINYINT], tOffset); } else { ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); if (ret != TSDB_CODE_SUCCESS) { @@ -489,8 +492,8 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pri } uint8_t tmpVal = (uint8_t)iv; - *sizeAppend = - tsSetColumnValue(payload, pSchema->colId, pSchema->type, &tmpVal, TYPE_BYTES[TSDB_DATA_TYPE_UTINYINT]); + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &tmpVal, + TYPE_BYTES[TSDB_DATA_TYPE_UTINYINT], tOffset); *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_UTINYINT]); } @@ -498,8 +501,8 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pri case TSDB_DATA_TYPE_SMALLINT: if (isNullStr(pToken)) { - // *((int16_t *)payload) = TSDB_DATA_SMALLINT_NULL; - *isColNull = true; + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, + tdGetNullVal(TSDB_DATA_TYPE_SMALLINT), TYPE_BYTES[TSDB_DATA_TYPE_SMALLINT], tOffset); } else { ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); if (ret != TSDB_CODE_SUCCESS) { @@ -509,8 +512,8 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pri } int16_t tmpVal = (int16_t)iv; - *sizeAppend = - tsSetColumnValue(payload, pSchema->colId, pSchema->type, &tmpVal, TYPE_BYTES[TSDB_DATA_TYPE_SMALLINT]); + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &tmpVal, + TYPE_BYTES[TSDB_DATA_TYPE_SMALLINT], tOffset); *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_SMALLINT]); } @@ -518,7 +521,9 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pri case TSDB_DATA_TYPE_USMALLINT: if (isNullStr(pToken)) { - *isColNull = true; + *sizeAppend = + tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, + tdGetNullVal(TSDB_DATA_TYPE_USMALLINT), TYPE_BYTES[TSDB_DATA_TYPE_USMALLINT], tOffset); } else { ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); if (ret != TSDB_CODE_SUCCESS) { @@ -528,8 +533,8 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pri } uint16_t tmpVal = (uint16_t)iv; - *sizeAppend = - tsSetColumnValue(payload, pSchema->colId, pSchema->type, &tmpVal, TYPE_BYTES[TSDB_DATA_TYPE_USMALLINT]); + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &tmpVal, + TYPE_BYTES[TSDB_DATA_TYPE_USMALLINT], tOffset); *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_USMALLINT]); } @@ -537,7 +542,8 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pri case TSDB_DATA_TYPE_INT: if (isNullStr(pToken)) { - *isColNull = true; + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, + tdGetNullVal(TSDB_DATA_TYPE_INT), TYPE_BYTES[TSDB_DATA_TYPE_INT], tOffset); } else { ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); if (ret != TSDB_CODE_SUCCESS) { @@ -547,7 +553,8 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pri } int32_t tmpVal = (int32_t)iv; - *sizeAppend = tsSetColumnValue(payload, pSchema->colId, pSchema->type, &tmpVal, TYPE_BYTES[TSDB_DATA_TYPE_INT]); + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &tmpVal, + TYPE_BYTES[TSDB_DATA_TYPE_INT], tOffset); *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_INT]); } @@ -555,7 +562,8 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pri case TSDB_DATA_TYPE_UINT: if (isNullStr(pToken)) { - *isColNull = true; + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, + tdGetNullVal(TSDB_DATA_TYPE_UINT), TYPE_BYTES[TSDB_DATA_TYPE_UINT], tOffset); } else { ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); if (ret != TSDB_CODE_SUCCESS) { @@ -565,8 +573,8 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pri } uint32_t tmpVal = (uint32_t)iv; - *sizeAppend = - tsSetColumnValue(payload, pSchema->colId, pSchema->type, &tmpVal, TYPE_BYTES[TSDB_DATA_TYPE_UINT]); + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &tmpVal, + TYPE_BYTES[TSDB_DATA_TYPE_UINT], tOffset); *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_UINT]); } @@ -574,7 +582,8 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pri case TSDB_DATA_TYPE_BIGINT: if (isNullStr(pToken)) { - *isColNull = true; + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, + tdGetNullVal(TSDB_DATA_TYPE_BIGINT), TYPE_BYTES[TSDB_DATA_TYPE_BIGINT], tOffset); } else { ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); if (ret != TSDB_CODE_SUCCESS) { @@ -583,14 +592,16 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pri return tscInvalidOperationMsg(msg, "bigint data overflow", pToken->z); } - *sizeAppend = tsSetColumnValue(payload, pSchema->colId, pSchema->type, &iv, TYPE_BYTES[TSDB_DATA_TYPE_BIGINT]); + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &iv, + TYPE_BYTES[TSDB_DATA_TYPE_BIGINT], tOffset); *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_BIGINT]); } break; case TSDB_DATA_TYPE_UBIGINT: if (isNullStr(pToken)) { - *isColNull = true; + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, + tdGetNullVal(TSDB_DATA_TYPE_UBIGINT), TYPE_BYTES[TSDB_DATA_TYPE_UBIGINT], tOffset); } else { ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); if (ret != TSDB_CODE_SUCCESS) { @@ -600,15 +611,16 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pri } uint64_t tmpVal = (uint64_t)iv; - *sizeAppend = - tsSetColumnValue(payload, pSchema->colId, pSchema->type, &tmpVal, TYPE_BYTES[TSDB_DATA_TYPE_UBIGINT]); + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &tmpVal, + TYPE_BYTES[TSDB_DATA_TYPE_UBIGINT], tOffset); *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_UBIGINT]); } break; case TSDB_DATA_TYPE_FLOAT: if (isNullStr(pToken)) { - *isColNull = true; + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, + tdGetNullVal(TSDB_DATA_TYPE_FLOAT), TYPE_BYTES[TSDB_DATA_TYPE_FLOAT], tOffset); } else { double dv; if (TK_ILLEGAL == tscToDouble(pToken, &dv, &endptr)) { @@ -621,15 +633,16 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pri } float tmpVal = (float)dv; - *sizeAppend = - tsSetColumnValue(payload, pSchema->colId, pSchema->type, &tmpVal, TYPE_BYTES[TSDB_DATA_TYPE_FLOAT]); + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &tmpVal, + TYPE_BYTES[TSDB_DATA_TYPE_FLOAT], tOffset); *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_FLOAT]); } break; case TSDB_DATA_TYPE_DOUBLE: if (isNullStr(pToken)) { - *isColNull = true; + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, + tdGetNullVal(TSDB_DATA_TYPE_DOUBLE), TYPE_BYTES[TSDB_DATA_TYPE_DOUBLE], tOffset); } else { double dv; if (TK_ILLEGAL == tscToDouble(pToken, &dv, &endptr)) { @@ -640,7 +653,8 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pri return tscInvalidOperationMsg(msg, "illegal double data", pToken->z); } - *sizeAppend = tsSetColumnValue(payload, pSchema->colId, pSchema->type, &dv, TYPE_BYTES[TSDB_DATA_TYPE_DOUBLE]); + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &dv, + TYPE_BYTES[TSDB_DATA_TYPE_DOUBLE], tOffset); *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_DOUBLE]); } break; @@ -648,21 +662,22 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pri case TSDB_DATA_TYPE_BINARY: // binary data cannot be null-terminated char string, otherwise the last char of the string is lost if (pToken->type == TK_NULL) { - // setVardataNull(payload, TSDB_DATA_TYPE_BINARY); - *isColNull = true; + payloadColSetId(payload, pSchema->colId); + payloadColSetType(payload, pSchema->type); + memcpy(POINTER_SHIFT(payloadStart, tOffset), tdGetNullVal(TSDB_DATA_TYPE_BINARY), VARSTR_HEADER_SIZE + CHAR_BYTES); + *sizeAppend = (TDRowLenT)(VARSTR_HEADER_SIZE + CHAR_BYTES); } else { // too long values will return invalid sql, not be truncated automatically if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { // todo refactor return tscInvalidOperationMsg(msg, "string data overflow", pToken->z); } - // STR_WITH_SIZE_TO_VARSTR(payload, pToken->z, pToken->n); payloadColSetId(payload, pSchema->colId); payloadColSetType(payload, pSchema->type); - varDataSetLen(payloadColValue(payload), pToken->n); - memcpy(varDataVal(payloadColValue(payload)), pToken->z, pToken->n); - *sizeAppend = (TDRowLenT)(PAYLOAD_ID_TYPE_LEN + VARSTR_HEADER_SIZE + pToken->n); - *dataRowColDeltaLen += (TDRowLenT)(pToken->n - sizeof(uint8_t)); + varDataSetLen(POINTER_SHIFT(payloadStart,tOffset), pToken->n); + memcpy(varDataVal(POINTER_SHIFT(payloadStart,tOffset)), pToken->z, pToken->n); + *sizeAppend = (TDRowLenT)(VARSTR_HEADER_SIZE + pToken->n); + *dataRowColDeltaLen += (TDRowLenT)(pToken->n - CHAR_BYTES); *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + VARSTR_HEADER_SIZE + pToken->n); } @@ -670,22 +685,25 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pri case TSDB_DATA_TYPE_NCHAR: if (pToken->type == TK_NULL) { - *isColNull = true; + payloadColSetId(payload, pSchema->colId); + payloadColSetType(payload, pSchema->type); + memcpy(POINTER_SHIFT(payloadStart,tOffset), tdGetNullVal(TSDB_DATA_TYPE_NCHAR), VARSTR_HEADER_SIZE + INT_BYTES); + *sizeAppend = (TDRowLenT)(VARSTR_HEADER_SIZE + INT_BYTES); } else { // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' int32_t output = 0; payloadColSetId(payload, pSchema->colId); payloadColSetType(payload, pSchema->type); - if (!taosMbsToUcs4(pToken->z, pToken->n, varDataVal(payloadColValue(payload)), + if (!taosMbsToUcs4(pToken->z, pToken->n, varDataVal(POINTER_SHIFT(payloadStart,tOffset)), pSchema->bytes - VARSTR_HEADER_SIZE, &output)) { char buf[512] = {0}; snprintf(buf, tListLen(buf), "%s", strerror(errno)); return tscInvalidOperationMsg(msg, buf, pToken->z); } - varDataSetLen(payloadColValue(payload), output); + varDataSetLen(POINTER_SHIFT(payloadStart,tOffset), output); - *sizeAppend = (TDRowLenT)(PAYLOAD_ID_TYPE_LEN + VARSTR_HEADER_SIZE + output); + *sizeAppend = (TDRowLenT)(VARSTR_HEADER_SIZE + output); *dataRowColDeltaLen += (TDRowLenT)(output - sizeof(uint32_t)); *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + VARSTR_HEADER_SIZE + output); } @@ -696,12 +714,13 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pri if (primaryKey) { // When building SKVRow primaryKey, we should not skip even with NULL value. int64_t tmpVal = 0; - - *sizeAppend = tsSetColumnValue(primaryKeyStart, pSchema->colId, pSchema->type, &tmpVal, - TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP]); + *sizeAppend = tsSetPayloadColValue(payloadStart, primaryKeyStart, pSchema->colId, pSchema->type, &tmpVal, + TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], tOffset); *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP]); } else { - *isColNull = true; + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, + tdGetNullVal(TSDB_DATA_TYPE_TIMESTAMP), + TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], tOffset); } } else { int64_t tmpVal; @@ -709,8 +728,8 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pri return tscInvalidOperationMsg(msg, "invalid timestamp", pToken->z); } - *sizeAppend = tsSetColumnValue(primaryKey ? primaryKeyStart : payload, pSchema->colId, pSchema->type, &tmpVal, - TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP]); + *sizeAppend = tsSetPayloadColValue(payloadStart, primaryKey ? primaryKeyStart : payload, pSchema->colId, + pSchema->type, &tmpVal, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], tOffset); *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP]); } @@ -762,27 +781,31 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i int32_t index = 0; SStrToken sToken = {0}; - SMemRowBuilder *pBuilder = &pDataBlocks->rowBuilder; - char * payload = pDataBlocks->pData + pDataBlocks->size; + SMemRowHelper *pHelper = &pDataBlocks->rowHelper; + char * payload = pDataBlocks->pData + pDataBlocks->size; SParsedDataColInfo *spd = &pDataBlocks->boundColumnInfo; SSchema * schema = tscGetTableSchema(pDataBlocks->pTableMeta); - // 1. set the parsed value from sql string + int32_t rowSize = 0; - uint16_t rowSizeAppended = 0; - uint16_t nColsNotNull = 0; - TDRowLenT dataRowLen = pBuilder->allNullLen; - TDRowLenT kvRowLen = TD_MEM_ROW_KV_VER_SIZE; - + int32_t dataRowLen = pHelper->allNullLen; + int32_t kvRowLen = TD_MEM_ROW_KV_VER_SIZE; + TDRowTLenT payloadValOffset = 0; + TDRowLenT colValOffset = 0; ASSERT(dataRowLen > 0); + + payloadSetNCols(payload, spd->numOfBound); + payloadValOffset = payloadValuesOffset(payload); // rely on payloadNCols + payloadSetTLen(payload, payloadValOffset); - char *kvPrimayKeyStart = payload + PAYLOAD_HEADER_LEN; // primaryKey in 1st column tuple - char *kvStart = kvPrimayKeyStart + PAYLOAD_PRIMARY_COL_LEN; // the column tuple behind the primaryKey + char *kvPrimaryKeyStart = payload + PAYLOAD_HEADER_LEN; // primaryKey in 1st column tuple + char *kvStart = kvPrimaryKeyStart + PAYLOAD_COL_HEAD_LEN; // the column tuple behind the primaryKey + // 1. set the parsed value from sql string for (int i = 0; i < spd->numOfBound; ++i) { // the start position in data block buffer of current value in sql - int32_t colIndex = spd->boundedColumns[i]; // ordered + int32_t colIndex = spd->boundedColumns[i]; char *start = payload + spd->cols[colIndex].offset; @@ -845,43 +868,61 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i } bool isPrimaryKey = (colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX); - bool isColNull = false; TDRowLenT dataRowDeltaColLen = 0; // When combine the data as SDataRow, the delta len between all NULL columns. TDRowLenT kvRowColLen = 0; - TDRowLenT colSizeAppended = 0; - // make sure the Primarykey locates in the 1st column - int32_t ret = tsParseOneColumnKV(pSchema, &sToken, kvPrimayKeyStart, kvStart, pInsertParam->msg, str, isPrimaryKey, - timePrec, &colSizeAppended, &isColNull, &dataRowDeltaColLen, &kvRowColLen); + TDRowLenT colValAppended = 0; + + if(!spd->isOrdered) { + ASSERT(spd->colIdxInfo != NULL); + if(!isPrimaryKey) { + kvStart = POINTER_SHIFT(kvPrimaryKeyStart, spd->colIdxInfo[i].finalIdx * PAYLOAD_COL_HEAD_LEN); + } else { + ASSERT(spd->colIdxInfo[i].finalIdx == 0); + } + } else { + ASSERT(spd->colIdxInfo == NULL); + } + // the primary key locates in 1st column + int32_t ret = tsParseOneColumnKV(pSchema, &sToken, payload, kvPrimaryKeyStart, kvStart, pInsertParam->msg, str, + isPrimaryKey, timePrec, payloadValOffset + colValOffset, &colValAppended, + &dataRowDeltaColLen, &kvRowColLen); if (ret != TSDB_CODE_SUCCESS) { return ret; } - if (isPrimaryKey && tsCheckTimestamp(pDataBlocks, payloadColValue(kvPrimayKeyStart)) != TSDB_CODE_SUCCESS) { - tscInvalidOperationMsg(pInsertParam->msg, "client time/server time can not be mixed up", sToken.z); - return TSDB_CODE_TSC_INVALID_TIME_STAMP; - } - if (isColNull == false) { - ++nColsNotNull; + if (isPrimaryKey) { + if (tsCheckTimestamp(pDataBlocks, payloadValues(payload)) != TSDB_CODE_SUCCESS) { + tscInvalidOperationMsg(pInsertParam->msg, "client time/server time can not be mixed up", sToken.z); + return TSDB_CODE_TSC_INVALID_TIME_STAMP; + } + payloadColSetOffset(kvPrimaryKeyStart, colValOffset); + } else { + payloadColSetOffset(kvStart, colValOffset); + if(spd->isOrdered) { + kvStart += PAYLOAD_COL_HEAD_LEN; // move to next column + } } + + colValOffset += colValAppended; kvRowLen += kvRowColLen; dataRowLen += dataRowDeltaColLen; - if (!isPrimaryKey) { - kvStart += colSizeAppended; // move to next column - } - rowSizeAppended += colSizeAppended; // calculate rowLen } - if (kvRowLen < dataRowLen) { + if (kvRowLen < dataRowLen * KVRowRatio) { payloadSetType(payload, SMEM_ROW_KV); } else { payloadSetType(payload, SMEM_ROW_DATA); } - *len = PAYLOAD_HEADER_LEN + rowSizeAppended; - - payloadSetNCols(payload, nColsNotNull); + ASSERT(colValOffset <= TSDB_MAX_BYTES_PER_ROW); + + *len = (int32_t)(payloadValOffset + colValOffset); payloadSetTLen(payload, *len); + // TSKEY tsKey = payloadKey(payload); + // ASSERT((tsKey < 1627747200000000 && tsKey > 1498838400000000) || (tsKey < 1627747200000 && tsKey > 1498838400000) || + // (tsKey < 1627747200 && tsKey > 1498838400)); + return TSDB_CODE_SUCCESS; } @@ -895,6 +936,27 @@ static int32_t rowDataCompar(const void *lhs, const void *rhs) { return left > right ? 1 : -1; } } +static int32_t schemaIdxCompar(const void *lhs, const void *rhs) { + uint16_t left = *(uint16_t *)lhs; + uint16_t right = *(uint16_t *)rhs; + + if (left == right) { + return 0; + } else { + return left > right ? 1 : -1; + } +} + +static int32_t boundIdxCompar(const void *lhs, const void *rhs) { + uint16_t left = *(uint16_t *)POINTER_SHIFT(lhs, sizeof(uint16_t)); + uint16_t right = *(uint16_t *)POINTER_SHIFT(rhs, sizeof(uint16_t)); + + if (left == right) { + return 0; + } else { + return left > right ? 1 : -1; + } +} int32_t tsParseValues(char **str, STableDataBlocks *pDataBlock, int maxRows, SInsertStatementParam *pInsertParam, int32_t* numOfRows, char *tmpTokenBuf) { @@ -912,8 +974,8 @@ int32_t tsParseValues(char **str, STableDataBlocks *pDataBlock, int maxRows, SIn int32_t extendedRowSize = getExtendedRowSize(&tinfo); - initSMemRowBuilder(&pDataBlock->rowBuilder, tscGetTableSchema(pDataBlock->pTableMeta), - tscGetNumOfColumns(pDataBlock->pTableMeta), 0); + initSMemRowHelper(&pDataBlock->rowHelper, tscGetTableSchema(pDataBlock->pTableMeta), + tscGetNumOfColumns(pDataBlock->pTableMeta), 0); while (1) { index = 0; @@ -965,9 +1027,10 @@ int32_t tsParseValues(char **str, STableDataBlocks *pDataBlock, int maxRows, SIn void tscSetBoundColumnInfo(SParsedDataColInfo *pColInfo, SSchema *pSchema, int32_t numOfCols) { pColInfo->numOfCols = numOfCols; pColInfo->numOfBound = numOfCols; - + pColInfo->isOrdered = true; pColInfo->boundedColumns = calloc(pColInfo->numOfCols, sizeof(int32_t)); pColInfo->cols = calloc(pColInfo->numOfCols, sizeof(SBoundColumn)); + pColInfo->colIdxInfo = NULL; for (int32_t i = 0; i < pColInfo->numOfCols; ++i) { if (i > 0) { @@ -1093,14 +1156,23 @@ int tscSortRemoveDataBlockDupRows(STableDataBlocks *dataBuf, SBlockKeyInfo *pBlk SBlockKeyTuple *pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; char * pBlockData = pBlocks->data; - uint32_t totolPayloadLen = 0; - TDRowLenT payloadTLen = 0; + TDRowTLenT totolPayloadTLen = 0; + TDRowTLenT payloadTLen = 0; int n = 0; while (n < nRows) { pBlkKeyTuple->skey = payloadKey(pBlockData); pBlkKeyTuple->payloadAddr = pBlockData; payloadTLen = payloadTLen(pBlockData); - totolPayloadLen += payloadTLen; + + ASSERT(payloadNCols(pBlockData) <= 4096); + ASSERT(payloadTLen(pBlockData) < 65536); + ASSERT(pBlkKeyTuple->payloadAddr != NULL); + + ASSERT((pBlkKeyTuple->skey < 1627747200000000 && pBlkKeyTuple->skey > 1498838400000000) || + (pBlkKeyTuple->skey < 1627747200000 && pBlkKeyTuple->skey > 1498838400000) || + (pBlkKeyTuple->skey < 1627747200 && pBlkKeyTuple->skey > 1498838400)); + + totolPayloadTLen += payloadTLen; // next loop pBlockData += payloadTLen; ++pBlkKeyTuple; @@ -1119,23 +1191,36 @@ int tscSortRemoveDataBlockDupRows(STableDataBlocks *dataBuf, SBlockKeyInfo *pBlk TSKEY tj = (pBlkKeyTuple + j)->skey; if (ti == tj) { - totolPayloadLen -= payloadTLen(pBlkKeyTuple + j); + totolPayloadTLen -= payloadTLen(pBlkKeyTuple + j); ++j; continue; } int32_t nextPos = (++i); if (nextPos != j) { - memmove(pBlkKeyTuple + sizeof(SBlockKeyTuple) * nextPos, pBlkKeyTuple + sizeof(SBlockKeyTuple) * j, sizeof(SBlockKeyTuple)); + memmove(pBlkKeyTuple + nextPos, pBlkKeyTuple + j, sizeof(SBlockKeyTuple)); } ++j; } dataBuf->ordered = true; pBlocks->numOfRows = i + 1; + + ASSERT(pBlocks->numOfRows <= nRows); + + int tt = 0; + pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; + while (tt < pBlocks->numOfRows) { + ASSERT(pBlkKeyTuple->payloadAddr != NULL); + ASSERT((pBlkKeyTuple->skey < 1627747200000000 && pBlkKeyTuple->skey > 1498838400000000) || + (pBlkKeyTuple->skey < 1627747200000 && pBlkKeyTuple->skey > 1498838400000) || + (pBlkKeyTuple->skey < 1627747200 && pBlkKeyTuple->skey > 1498838400)); + ++pBlkKeyTuple; + ++tt; + } } - - dataBuf->size = sizeof(SSubmitBlk) + totolPayloadLen; + + dataBuf->size = sizeof(SSubmitBlk) + totolPayloadTLen; dataBuf->prevTS = INT64_MIN; return 0; @@ -1475,7 +1560,6 @@ static int32_t validateDataSource(SInsertStatementParam *pInsertParam, int32_t t static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDataColInfo* pColInfo, SSchema* pSchema, char* str, char **end) { pColInfo->numOfBound = 0; - memset(pColInfo->boundedColumns, 0, sizeof(int32_t) * pColInfo->numOfCols); for(int32_t i = 0; i < pColInfo->numOfCols; ++i) { pColInfo->cols[i].hasVal = false; @@ -1483,7 +1567,7 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat int32_t code = TSDB_CODE_SUCCESS; - int32_t index = 0; + int32_t index = 0; SStrToken sToken = tStrGetToken(str, &index, false); str += index; @@ -1491,7 +1575,8 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat code = tscSQLSyntaxErrMsg(pInsertParam->msg, "( is expected", sToken.z); goto _clean; } - + bool isOrdered = true; + int32_t lastColIdx = -1; while (1) { index = 0; sToken = tStrGetToken(str, &index, false); @@ -1523,6 +1608,14 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat pColInfo->boundedColumns[pColInfo->numOfBound] = t; pColInfo->numOfBound += 1; findColumnIndex = true; + + if (isOrdered) { + if (lastColIdx > t) { + isOrdered = false; + } else { + lastColIdx = t; + } + } break; } } @@ -1533,10 +1626,32 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat } } - memset(&pColInfo->boundedColumns[pColInfo->numOfBound], 0 , sizeof(int32_t) * (pColInfo->numOfCols - pColInfo->numOfBound)); + pColInfo->isOrdered = isOrdered; + + if (!isOrdered) { + pColInfo->colIdxInfo = calloc(pColInfo->numOfBound, sizeof(SBoundIdxInfo)); + if (pColInfo->colIdxInfo == NULL) { + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _clean; + } + SBoundIdxInfo *pColIdx = pColInfo->colIdxInfo; + for (int i = 0; i < pColInfo->numOfBound; ++i) { + pColIdx[i].schemaColIdx = (uint16_t)pColInfo->boundedColumns[i]; + pColIdx[i].boundIdx = (uint16_t)i; + } + qsort(pColIdx, pColInfo->numOfBound, sizeof(SBoundIdxInfo), schemaIdxCompar); + for (int i = 0; i < pColInfo->numOfBound; ++i) { + pColIdx[i].finalIdx = (uint16_t)i; + } + qsort(pColIdx, pColInfo->numOfBound, sizeof(SBoundIdxInfo), boundIdxCompar); + } + + memset(&pColInfo->boundedColumns[pColInfo->numOfBound], 0, + sizeof(int32_t) * (pColInfo->numOfCols - pColInfo->numOfBound)); + return TSDB_CODE_SUCCESS; - _clean: +_clean: pInsertParam->sql = NULL; return code; } @@ -1944,8 +2059,8 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int32_t numOfRow goto _error; } - initSMemRowBuilder(&pTableDataBlock->rowBuilder, tscGetTableSchema(pTableDataBlock->pTableMeta), - tscGetNumOfColumns(pTableDataBlock->pTableMeta), 0); + initSMemRowHelper(&pTableDataBlock->rowHelper, tscGetTableSchema(pTableDataBlock->pTableMeta), + tscGetNumOfColumns(pTableDataBlock->pTableMeta), 0); while ((readLen = tgetline(&line, &n, fp)) != -1) { if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) { diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 729fae3bfa..9246b86ea6 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1433,6 +1433,7 @@ void tscFreeSqlObj(SSqlObj* pSql) { void tscDestroyBoundColumnInfo(SParsedDataColInfo* pColInfo) { tfree(pColInfo->boundedColumns); tfree(pColInfo->cols); + tfree(pColInfo->colIdxInfo); } void tscDestroyDataBlock(STableDataBlocks* pDataBlock, bool removeMeta) { @@ -1646,51 +1647,54 @@ int32_t tscGetDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, i return TSDB_CODE_SUCCESS; } - - - static SMemRow tdGenMemRowFromBuilder(SMemRowBuilder* pBuilder) { SSchema* pSchema = pBuilder->pSchema; char* p = (char*)pBuilder->buf; int toffset = 0; uint16_t nCols = pBuilder->nCols; -// RawRow payload structure: -// |<---------- header ------------->|<------- column data array ------->| -// |SMemRowType| dataLen | nCols | colId | colType | value |...|...| -// +-----------+----------+----------+---------------------------------->| -// | uint8_t | uint16_t | uint16_t | int16_t | uint8_t | ??? |...|...| -// +-----------+----------+----------+---------------------------------->| uint8_t memRowType = payloadType(p); - uint16_t nColsNotNull = payloadNCols(p); - if (pBuilder->nCols <= 0 || nColsNotNull <= 0) { + uint16_t nColsBound = payloadNCols(p); + if (pBuilder->nCols <= 0 || nColsBound <= 0) { return NULL; } - ASSERT(nColsNotNull <= nCols); - + char* pVals = POINTER_SHIFT(p, payloadValuesOffset(p)); SMemRow* memRow = (SMemRow)pBuilder->pDataBlock; memRowSetType(memRow, memRowType); + // ----------------- Raw payload structure for row: + /* |<------------ Head ------------->|<----------- body of column data tuple ------------------->| + * | |<----------------- flen ------------->|<--- value part --->| + * |SMemRowType| dataTLen | nCols | colId | colType | offset | ... | value |...|...|... | + * +-----------+----------+----------+--------------------------------------|--------------------| + * | uint8_t | uint32_t | uint16_t | int16_t | uint8_t | uint16_t | ... |.......|...|...|... | + * +-----------+----------+----------+--------------------------------------+--------------------| + * 1. offset in column data tuple starts from the value part in case of uint16_t overflow. + * 2. dataTLen: total length including the header and body. + */ + if (memRowType == SMEM_ROW_DATA) { + ASSERT(nColsBound <= nCols); SDataRow trow = (SDataRow)memRowDataBody(memRow); - dataRowSetLen(trow, (uint16_t)(TD_DATA_ROW_HEAD_SIZE + pBuilder->flen)); + dataRowSetLen(trow, (TDRowLenT)(TD_DATA_ROW_HEAD_SIZE + pBuilder->flen)); dataRowSetVersion(trow, pBuilder->sversion); p = (char*)payloadBody(pBuilder->buf); uint16_t i = 0, j = 0; - while (j < pBuilder->nCols) { - if (i >= nColsNotNull) { + while (j < nCols) { + if (i >= nColsBound) { break; } - int16_t colId = *(int16_t*)p; + int16_t colId = payloadColId(p); if (colId == pSchema[j].colId) { - tdAppendColVal(trow, payloadColValue(p), pSchema[j].type, toffset); + // ASSERT(payloadColType(p) == pSchema[j].type); + tdAppendColVal(trow, POINTER_SHIFT(pVals, payloadColOffset(p)), pSchema[j].type, toffset); toffset += TYPE_BYTES[pSchema[j].type]; - p = skipToNextEles(p); + p = payloadNextCol(p); ++i; ++j; } else if (colId < pSchema[j].colId) { - p = skipToNextEles(p); + p = payloadNextCol(p); ++i; } else { tdAppendColVal(trow, tdGetNullVal(pSchema[j].type), pSchema[j].type, toffset); @@ -1699,41 +1703,43 @@ static SMemRow tdGenMemRowFromBuilder(SMemRowBuilder* pBuilder) { } } - while (j < pBuilder->nCols) { + while (j < nCols) { tdAppendColVal(trow, tdGetNullVal(pSchema[j].type), pSchema[j].type, toffset); toffset += TYPE_BYTES[pSchema[j].type]; ++j; } - while (i < nColsNotNull) { - p = skipToNextEles(p); + + #if 0 // no need anymore + while (i < nColsBound) { + p = payloadNextCol(p); ++i; } + #endif } else if (memRowType == SMEM_ROW_KV) { - ASSERT(nColsNotNull <= pBuilder->nCols); + ASSERT(nColsBound <= nCols); SKVRow kvRow = (SKVRow)memRowKvBody(memRow); - uint16_t tlen = TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * nColsNotNull; - kvRowSetLen(kvRow, tlen); - kvRowSetNCols(kvRow, nColsNotNull); - memRowKvSetVersion(memRow, pBuilder->sversion); + kvRowSetLen(kvRow, (TDRowLenT)(TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * nColsBound)); + kvRowSetNCols(kvRow, nColsBound); + memRowSetKvVersion(memRow, pBuilder->sversion); p = (char*)payloadBody(pBuilder->buf); int i = 0; - while (i < nColsNotNull) { + while (i < nColsBound) { int16_t colId = payloadColId(p); uint8_t colType = payloadColType(p); - tdAppendKvColVal(kvRow, payloadColValue(p), colId, colType, toffset); + tdAppendKvColVal(kvRow, POINTER_SHIFT(pVals,payloadColOffset(p)), colId, colType, toffset); toffset += sizeof(SColIdx); - p = skipToNextEles(p); + p = payloadNextCol(p); ++i; } } else { ASSERT(0); } - - pBuilder->pDataBlock = (char*)pBuilder->pDataBlock + memRowTLen(memRow); // next row - pBuilder->pSubmitBlk->dataLen += memRowTLen(memRow); + int32_t rowTLen = memRowTLen(memRow); + pBuilder->pDataBlock = (char*)pBuilder->pDataBlock + rowTLen; // next row + pBuilder->pSubmitBlk->dataLen += rowTLen; return memRow; } @@ -1744,7 +1750,6 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, bo STableMeta* pTableMeta = pTableDataBlock->pTableMeta; STableComInfo tinfo = tscGetTableInfo(pTableMeta); SSchema* pSchema = tscGetTableSchema(pTableMeta); - SMemRowBuilder* pBuilder = &pTableDataBlock->rowBuilder; SSubmitBlk* pBlock = pDataBlock; memcpy(pDataBlock, pTableDataBlock->pData, sizeof(SSubmitBlk)); @@ -1780,18 +1785,19 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, bo pBlock->dataLen = 0; int32_t numOfRows = htons(pBlock->numOfRows); - pBuilder->pSchema = pSchema; - pBuilder->sversion = pTableMeta->sversion; - pBuilder->flen = flen; - pBuilder->nCols = tinfo.numOfColumns; - pBuilder->pDataBlock = pDataBlock; - pBuilder->pSubmitBlk = pBlock; - pBuilder->buf = p; - pBuilder->size = 0; + SMemRowBuilder rowBuilder; + rowBuilder.pSchema = pSchema; + rowBuilder.sversion = pTableMeta->sversion; + rowBuilder.flen = flen; + rowBuilder.nCols = tinfo.numOfColumns; + rowBuilder.pDataBlock = pDataBlock; + rowBuilder.pSubmitBlk = pBlock; + rowBuilder.buf = p; + rowBuilder.size = 0; for (int32_t i = 0; i < numOfRows; ++i) { - pBuilder->buf = (blkKeyTuple+i)->payloadAddr; - tdGenMemRowFromBuilder(pBuilder); + rowBuilder.buf = (blkKeyTuple + i)->payloadAddr; + tdGenMemRowFromBuilder(&rowBuilder); } int32_t len = pBlock->dataLen + pBlock->schemaLen; @@ -1803,7 +1809,7 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, bo static int32_t getRowExpandSize(STableMeta* pTableMeta) { // add prefix len of KV type SMemRow(we may use SDataRow or SKVRow) - int32_t result = TD_DATA_ROW_HEAD_SIZE + TD_MEM_ROW_KV_TYPE_VER_SIZE; + int32_t result = TD_MEM_ROW_DATA_HEAD_SIZE; int32_t columns = tscGetNumOfColumns(pTableMeta); SSchema* pSchema = tscGetTableSchema(pTableMeta); for(int32_t i = 0; i < columns; i++) { diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index 093b7c5188..49353115ae 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -186,18 +186,6 @@ static FORCE_INLINE int tkeyComparFn(const void *tkey1, const void *tkey2) { return 0; } } -/* A memory data row, the format is like below: - *|---------+---------------------+--------------------------- len ---------------------------------->| - *|<- type->|<-- Head -->|<--------- flen -------------->| | - *|---------+---------------------+---------------------------------+---------------------------------+ - *| uint8_t | uint16_t | int16_t | | | - *|---------+----------+----------+---------------------------------+---------------------------------+ - *| flag | len | sversion | First part | Second part | - *|---------+----------+----------+---------------------------------+---------------------------------+ - * - * NOTE: timestamp in this row structure is TKEY instead of TSKEY - */ -typedef void *SMemRow; // ----------------- Data row structure @@ -216,7 +204,7 @@ typedef void *SDataRow; #define TD_DATA_ROW_HEAD_SIZE (sizeof(uint16_t) + sizeof(int16_t)) -#define dataRowLen(r) (*(uint16_t *)(r)) +#define dataRowLen(r) (*(TDRowLenT *)(r)) // 0~65535 #define dataRowVersion(r) (*(int16_t *)POINTER_SHIFT(r, sizeof(int16_t))) #define dataRowTuple(r) POINTER_SHIFT(r, TD_DATA_ROW_HEAD_SIZE) #define dataRowTKey(r) (*(TKEY *)(dataRowTuple(r))) @@ -231,7 +219,6 @@ SDataRow tdNewDataRowFromSchema(STSchema *pSchema); void tdFreeDataRow(SDataRow row); void tdInitDataRow(SDataRow row, STSchema *pSchema); SDataRow tdDataRowDup(SDataRow row); -SMemRow tdMemRowDup(SMemRow row); // offset here not include dataRow header length static FORCE_INLINE int tdAppendColVal(SDataRow row, const void *value, int8_t type, int32_t offset) { @@ -247,7 +234,7 @@ static FORCE_INLINE int tdAppendColVal(SDataRow row, const void *value, int8_t t if (offset == 0) { ASSERT(type == TSDB_DATA_TYPE_TIMESTAMP); TKEY tvalue = tdGetTKEY(*(TSKEY *)value); - memcpy(POINTER_SHIFT(row, toffset), (void *)(&tvalue), TYPE_BYTES[type]); + memcpy(POINTER_SHIFT(row, toffset), (const void *)(&tvalue), TYPE_BYTES[type]); } else { memcpy(POINTER_SHIFT(row, toffset), value, TYPE_BYTES[type]); } @@ -287,7 +274,7 @@ void dataColSetOffset(SDataCol *pCol, int nEle); bool isNEleNull(SDataCol *pCol, int nEle); void dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints); -static FORCE_INLINE const void *tdGetNullVal(int8_t type) { +FORCE_INLINE const void *tdGetNullVal(int8_t type) { switch (type) { case TSDB_DATA_TYPE_BOOL: return &BoolNull; @@ -400,11 +387,11 @@ void tdResetDataCols(SDataCols *pCols); int tdInitDataCols(SDataCols *pCols, STSchema *pSchema); SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData); SDataCols *tdFreeDataCols(SDataCols *pCols); -void tdAppendMemRowToDataCol(SMemRow row, STSchema *pSchema, SDataCols *pCols); int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int *pOffset); // ----------------- K-V data row structure -/* +/* |<-------------------------------------- len -------------------------------------------->| + * |<----- header ----->|<--------------------------- body -------------------------------->| * +----------+----------+---------------------------------+---------------------------------+ * | uint16_t | int16_t | | | * +----------+----------+---------------------------------+---------------------------------+ @@ -420,7 +407,7 @@ typedef struct { #define TD_KV_ROW_HEAD_SIZE (sizeof(uint16_t) + sizeof(int16_t)) -#define kvRowLen(r) (*(uint16_t *)(r)) +#define kvRowLen(r) (*(TDRowLenT *)(r)) #define kvRowNCols(r) (*(int16_t *)POINTER_SHIFT(r, sizeof(uint16_t))) #define kvRowSetLen(r, len) kvRowLen(r) = (len) #define kvRowSetNCols(r, n) kvRowNCols(r) = (n) @@ -532,7 +519,7 @@ static FORCE_INLINE int tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t colId, // ----------------- SMemRow appended with sequential data row structure /* - * |-------------------------------+--------------------------- len ---------------------------------->| + * |---------|------------------------------------------------- len ---------------------------------->| * |<-------- Head ------>|<--------- flen -------------->| | * |---------+---------------------+---------------------------------+---------------------------------+ * | uint8_t | uint16_t | int16_t | | | @@ -544,7 +531,8 @@ static FORCE_INLINE int tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t colId, */ // ----------------- SMemRow appended with extended K-V data row structure -/* | +/* |--------------------|------------------------------------------------ len ---------------------------------->| + * |<------------- Head ------------>|<--------- flen -------------->| | * |--------------------+----------+--------------------------------------------+---------------------------------+ * | uint8_t | int16_t | uint16_t | int16_t | | | * |---------+----------+----------+----------+---------------------------------+---------------------------------+ @@ -552,11 +540,13 @@ static FORCE_INLINE int tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t colId, * |---------+----------+----------+----------+---------------------------------+---------------------------------+ */ +typedef void *SMemRow; + #define TD_MEM_ROW_TYPE_SIZE sizeof(uint8_t) #define TD_MEM_ROW_KV_VER_SIZE sizeof(int16_t) #define TD_MEM_ROW_KV_TYPE_VER_SIZE (TD_MEM_ROW_TYPE_SIZE + TD_MEM_ROW_KV_VER_SIZE) #define TD_MEM_ROW_DATA_HEAD_SIZE (TD_MEM_ROW_TYPE_SIZE + TD_DATA_ROW_HEAD_SIZE) -#define TD_MEM_ROW_KV_HEAD_SIZE (TD_MEM_ROW_TYPE_SIZE + TD_MEM_ROW_KV_VER_SIZE + TD_KV_ROW_HEAD_SIZE) +// #define TD_MEM_ROW_KV_HEAD_SIZE (TD_MEM_ROW_TYPE_SIZE + TD_MEM_ROW_KV_VER_SIZE + TD_KV_ROW_HEAD_SIZE) #define SMEM_ROW_DATA 0U // SDataRow #define SMEM_ROW_KV 1U // SKVRow @@ -567,21 +557,22 @@ static FORCE_INLINE int tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t colId, #define memRowDataBody(r) POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE) // section after flag #define memRowKvBody(r) \ - POINTER_SHIFT(r, TD_MEM_ROW_KV_TYPE_VER_SIZE) // section after flag + sversion as to reuse of SKVRow -// #define memRowBody(r) (isDataRow(r) ? memRowDataBody(r) : memRowKvBody(r)) + POINTER_SHIFT(r, TD_MEM_ROW_KV_TYPE_VER_SIZE) // section after flag + sversion as to reuse SKVRow -#define memRowDataLen(r) (*(TDRowLenT *)memRowDataBody(r)) -#define memRowKvLen(r) (*(TDRowLenT *)memRowKvBody(r)) -#define memRowDataTLen(r) (memRowDataLen(r) + (TDRowLenT)TD_MEM_ROW_TYPE_SIZE) -#define memRowKvTLen(r) (memRowKvLen(r) + (TDRowLenT)TD_MEM_ROW_KV_TYPE_VER_SIZE) +#define memRowDataLen(r) (*(TDRowLenT *)memRowDataBody(r)) // 0~65535 +#define memRowKvLen(r) (*(TDRowLenT *)memRowKvBody(r)) // 0~65535 + +#define memRowDataTLen(r) (memRowDataLen(r) + TD_MEM_ROW_TYPE_SIZE) // using uint32_t/int32_t to store the TLen + +#define memRowKvTLen(r) (memRowKvLen(r) + TD_MEM_ROW_KV_TYPE_VER_SIZE) #define memRowLen(r) (isDataRow(r) ? memRowDataLen(r) : memRowKvLen(r)) -#define memRowTLen(r) (isDataRow(r) ? memRowDataTLen(r) : memRowKvTLen(r)) +#define memRowTLen(r) (isDataRow(r) ? memRowDataTLen(r) : memRowKvTLen(r)) // using uint32_t/int32_t to store the TLen #define memRowDataVersion(r) dataRowVersion(memRowDataBody(r)) #define memRowKvVersion(r) (*(int16_t *)POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE)) #define memRowVersion(r) (isDataRow(r) ? memRowDataVersion(r) : memRowKvVersion(r)) // schema version -#define memRowKvSetVersion(r, v) (memRowKvVersion(r) = (v)) +#define memRowSetKvVersion(r, v) (memRowKvVersion(r) = (v)) #define memRowTuple(r) (isDataRow(r) ? dataRowTuple(memRowDataBody(r)) : kvRowValues(memRowKvBody(r))) #define memRowTKey(r) (isDataRow(r) ? dataRowTKey(memRowDataBody(r)) : kvRowTKey(memRowKvBody(r))) @@ -594,6 +585,8 @@ static FORCE_INLINE int tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t colId, #define memRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_MEM_ROW_DATA_HEAD_SIZE) #define memRowDeleted(r) TKEY_IS_DELETED(memRowTKey(r)) +SMemRow tdMemRowDup(SMemRow row); +void tdAppendMemRowToDataCol(SMemRow row, STSchema *pSchema, SDataCols *pCols); // NOTE: offset here including the header size static FORCE_INLINE void *tdGetKvRowDataOfCol(void *row, int32_t offset) { return POINTER_SHIFT(row, offset); } // NOTE: offset here including the header size @@ -608,50 +601,52 @@ static FORCE_INLINE void *tdGetMemRowDataOfCol(void *row, int8_t type, int32_t o return NULL; } -// RawRow payload structure: -// |<---------- header ------------->|<---- body: column data tuple ---->| -// |SMemRowType| dataLen | nCols | colId | colType | value |...|...| -// +-----------+----------+----------+---------------------------------->| -// | uint8_t | uint16_t | uint16_t | int16_t | uint8_t | ??? |...|...| -// +-----------+----------+----------+---------------------------------->| +// ----------------- Raw payload structure for row: +/* |<------------ Head ------------->|<----------- body of column data tuple ------------------->| + * | |<----------------- flen ------------->|<--- value part --->| + * |SMemRowType| dataTLen | nCols | colId | colType | offset | ... | value |...|...|... | + * +-----------+----------+----------+--------------------------------------|--------------------| + * | uint8_t | uint32_t | uint16_t | int16_t | uint8_t | uint16_t | ... |.......|...|...|... | + * +-----------+----------+----------+--------------------------------------+--------------------| + * 1. offset in column data tuple starts from the value part in case of uint16_t overflow. + * 2. dataTLen: total length including the header and body. + */ #define PAYLOAD_NCOLS_LEN sizeof(uint16_t) -#define PAYLOAD_NCOLS_OFFSET (sizeof(uint8_t) + sizeof(TDRowLenT)) +#define PAYLOAD_NCOLS_OFFSET (sizeof(uint8_t) + sizeof(TDRowTLenT)) #define PAYLOAD_HEADER_LEN (PAYLOAD_NCOLS_OFFSET + PAYLOAD_NCOLS_LEN) #define PAYLOAD_ID_LEN sizeof(int16_t) #define PAYLOAD_ID_TYPE_LEN (sizeof(int16_t) + sizeof(uint8_t)) +#define PAYLOAD_COL_HEAD_LEN (PAYLOAD_ID_TYPE_LEN + sizeof(uint16_t)) #define PAYLOAD_PRIMARY_COL_LEN (PAYLOAD_ID_TYPE_LEN + sizeof(TSKEY)) #define payloadBody(r) POINTER_SHIFT(r, PAYLOAD_HEADER_LEN) #define payloadType(r) (*(uint8_t *)(r)) #define payloadSetType(r, t) (payloadType(r) = (t)) -#define payloadTLen(r) (*(TDRowLenT *)POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE)) // including total header +#define payloadTLen(r) (*(TDRowTLenT *)POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE)) // including total header #define payloadSetTLen(r, l) (payloadTLen(r) = (l)) #define payloadNCols(r) (*(TDRowLenT *)POINTER_SHIFT(r, PAYLOAD_NCOLS_OFFSET)) #define payloadSetNCols(r, n) (payloadNCols(r) = (n)) +#define payloadValuesOffset(r) \ + (PAYLOAD_HEADER_LEN + payloadNCols(r) * PAYLOAD_COL_HEAD_LEN) // avoid using the macro in loop +#define payloadValues(r) POINTER_SHIFT(r, payloadValuesOffset(r)) // avoid using the macro in loop +#define payloadColId(c) (*(int16_t *)(c)) +#define payloadColType(c) (*(uint8_t *)POINTER_SHIFT(c, PAYLOAD_ID_LEN)) +#define payloadColOffset(c) (*(uint16_t *)POINTER_SHIFT(c, PAYLOAD_ID_TYPE_LEN)) +#define payloadColValue(c) POINTER_SHIFT(c, payloadColOffset(c)) -#define payloadColId(r) (*(int16_t *)(r)) -#define payloadColType(r) (*(uint8_t *)POINTER_SHIFT(r, PAYLOAD_ID_LEN)) -#define payloadColValue(r) POINTER_SHIFT(r, PAYLOAD_ID_TYPE_LEN) +#define payloadColSetId(c, i) (payloadColId(c) = (i)) +#define payloadColSetType(c, t) (payloadColType(c) = (t)) +#define payloadColSetOffset(c, o) (payloadColOffset(c) = (o)) -#define payloadColSetId(r, i) (payloadColId(r) = (i)) -#define payloadColSetType(r, t) (payloadColType(r) = (t)) - -#define payloadKeyAddr(r) POINTER_SHIFT(r, PAYLOAD_HEADER_LEN + PAYLOAD_ID_TYPE_LEN) -#define payloadTKey(r) (*(TKEY *)(payloadKeyAddr(r))) +#define payloadKeyOffset(r) (*(uint16_t *)POINTER_SHIFT(r, PAYLOAD_HEADER_LEN + PAYLOAD_ID_TYPE_LEN)) +#define payloadTKey(r) (*(TKEY *)POINTER_SHIFT(r, payloadValuesOffset(r) + payloadKeyOffset(r))) #define payloadKey(r) tdGetKey(payloadTKey(r)) -static FORCE_INLINE char *skipToNextEles(char *p) { - uint8_t colType = payloadColType(p); - if (IS_VAR_DATA_TYPE(colType)) { - return (char *)POINTER_SHIFT(p, PAYLOAD_ID_TYPE_LEN + varDataTLen(payloadColValue(p))); - } else { - return (char *)POINTER_SHIFT(p, PAYLOAD_ID_TYPE_LEN + TYPE_BYTES[colType]); - } -} +static FORCE_INLINE char *payloadNextCol(char *pCol) { return (char *)POINTER_SHIFT(pCol, PAYLOAD_COL_HEAD_LEN); } #ifdef __cplusplus } #endif -#endif // _TD_DATA_FORMAT_H_ +#endif // _TD_DATA_FORMAT_H_ \ No newline at end of file diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 6fa9a41f1f..9ab2542eb9 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -195,7 +195,11 @@ do { \ #define TSDB_APPNAME_LEN TSDB_UNI_LEN -#define TSDB_MAX_BYTES_PER_ROW 65536 + /** + * Don't change to 65536. As in some scenarios uint16_t (0~65535) is used to store the row len. + * Finally, we use 65531(65535 - 4), as the SDataRow and SKVRow including 4 bits header. + */ +#define TSDB_MAX_BYTES_PER_ROW 65531 #define TSDB_MAX_TAGS_LEN 16384 #define TSDB_MAX_TAGS 128 #define TSDB_MAX_TAG_CONDITIONS 1024 diff --git a/src/inc/ttype.h b/src/inc/ttype.h index 0900f45350..2581589563 100644 --- a/src/inc/ttype.h +++ b/src/inc/ttype.h @@ -10,9 +10,10 @@ extern "C" { #include "taosdef.h" // ----------------- For variable data types such as TSDB_DATA_TYPE_BINARY and TSDB_DATA_TYPE_NCHAR -typedef int32_t VarDataOffsetT; -typedef int16_t VarDataLenT; // maxVarDataLen: 32767 -typedef uint16_t TDRowLenT; +typedef int32_t VarDataOffsetT; +typedef int16_t VarDataLenT; // maxVarDataLen: 32767 +typedef uint16_t TDRowLenT; // not including overhead: 0 ~ 65535 +typedef uint32_t TDRowTLenT; // total length, including overhead typedef struct tstr { VarDataLenT len; diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index ad097ec0f5..d62595d96f 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -767,7 +767,7 @@ static int tsdbCopyRowToMem(STsdbRepo *pRepo, SMemRow row, STable *pTable, void void *pRow = tsdbAllocBytes(pRepo, memRowTLen(row)); if (pRow == NULL) { - tsdbError("vgId:%d failed to insert row with key %" PRId64 " to table %s while allocate %d bytes since %s", + tsdbError("vgId:%d failed to insert row with key %" PRId64 " to table %s while allocate %" PRIu64 " bytes since %s", REPO_ID(pRepo), key, TABLE_CHAR_NAME(pTable), memRowTLen(row), tstrerror(terrno)); return -1; } From c329bb4d0b5bd51a132fcdd20a23cd2120c6a9df Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 11 Jul 2021 10:20:53 +0000 Subject: [PATCH 20/67] fix compile err --- src/client/src/tscParseInsert.c | 22 ++---------------- src/common/inc/tdataformat.h | 41 ++++----------------------------- src/common/src/tdataformat.c | 35 ++++++++++++++++++++++++++++ src/tsdb/src/tsdbMemTable.c | 2 +- 4 files changed, 42 insertions(+), 58 deletions(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 89415da079..da1a085e95 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -1163,15 +1163,10 @@ int tscSortRemoveDataBlockDupRows(STableDataBlocks *dataBuf, SBlockKeyInfo *pBlk pBlkKeyTuple->skey = payloadKey(pBlockData); pBlkKeyTuple->payloadAddr = pBlockData; payloadTLen = payloadTLen(pBlockData); - +#if 0 ASSERT(payloadNCols(pBlockData) <= 4096); ASSERT(payloadTLen(pBlockData) < 65536); - ASSERT(pBlkKeyTuple->payloadAddr != NULL); - - ASSERT((pBlkKeyTuple->skey < 1627747200000000 && pBlkKeyTuple->skey > 1498838400000000) || - (pBlkKeyTuple->skey < 1627747200000 && pBlkKeyTuple->skey > 1498838400000) || - (pBlkKeyTuple->skey < 1627747200 && pBlkKeyTuple->skey > 1498838400)); - +#endif totolPayloadTLen += payloadTLen; // next loop pBlockData += payloadTLen; @@ -1205,19 +1200,6 @@ int tscSortRemoveDataBlockDupRows(STableDataBlocks *dataBuf, SBlockKeyInfo *pBlk dataBuf->ordered = true; pBlocks->numOfRows = i + 1; - - ASSERT(pBlocks->numOfRows <= nRows); - - int tt = 0; - pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; - while (tt < pBlocks->numOfRows) { - ASSERT(pBlkKeyTuple->payloadAddr != NULL); - ASSERT((pBlkKeyTuple->skey < 1627747200000000 && pBlkKeyTuple->skey > 1498838400000000) || - (pBlkKeyTuple->skey < 1627747200000 && pBlkKeyTuple->skey > 1498838400000) || - (pBlkKeyTuple->skey < 1627747200 && pBlkKeyTuple->skey > 1498838400)); - ++pBlkKeyTuple; - ++tt; - } } dataBuf->size = sizeof(SSubmitBlk) + totolPayloadTLen; diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index 49353115ae..573836a4e7 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -274,41 +274,7 @@ void dataColSetOffset(SDataCol *pCol, int nEle); bool isNEleNull(SDataCol *pCol, int nEle); void dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints); -FORCE_INLINE const void *tdGetNullVal(int8_t type) { - switch (type) { - case TSDB_DATA_TYPE_BOOL: - return &BoolNull; - case TSDB_DATA_TYPE_TINYINT: - return &TinyintNull; - case TSDB_DATA_TYPE_SMALLINT: - return &SmallintNull; - case TSDB_DATA_TYPE_INT: - return &IntNull; - case TSDB_DATA_TYPE_BIGINT: - return &BigintNull; - case TSDB_DATA_TYPE_FLOAT: - return &FloatNull; - case TSDB_DATA_TYPE_DOUBLE: - return &DoubleNull; - case TSDB_DATA_TYPE_BINARY: - return &BinaryNull; - case TSDB_DATA_TYPE_TIMESTAMP: - return &TimestampNull; - case TSDB_DATA_TYPE_NCHAR: - return &NcharNull; - case TSDB_DATA_TYPE_UTINYINT: - return &UTinyintNull; - case TSDB_DATA_TYPE_USMALLINT: - return &USmallintNull; - case TSDB_DATA_TYPE_UINT: - return &UIntNull; - case TSDB_DATA_TYPE_UBIGINT: - return &UBigintNull; - default: - ASSERT(0); - return NULL; - } -} +const void *tdGetNullVal(int8_t type); // Get the data pointer from a column-wised data static FORCE_INLINE void *tdGetColDataOfRow(SDataCol *pCol, int row) { @@ -562,9 +528,10 @@ typedef void *SMemRow; #define memRowDataLen(r) (*(TDRowLenT *)memRowDataBody(r)) // 0~65535 #define memRowKvLen(r) (*(TDRowLenT *)memRowKvBody(r)) // 0~65535 -#define memRowDataTLen(r) (memRowDataLen(r) + TD_MEM_ROW_TYPE_SIZE) // using uint32_t/int32_t to store the TLen +#define memRowDataTLen(r) \ + ((TDRowTLenT)(memRowDataLen(r) + TD_MEM_ROW_TYPE_SIZE)) // using uint32_t/int32_t to store the TLen -#define memRowKvTLen(r) (memRowKvLen(r) + TD_MEM_ROW_KV_TYPE_VER_SIZE) +#define memRowKvTLen(r) ((TDRowTLenT)(memRowKvLen(r) + TD_MEM_ROW_KV_TYPE_VER_SIZE)) #define memRowLen(r) (isDataRow(r) ? memRowDataLen(r) : memRowKvLen(r)) #define memRowTLen(r) (isDataRow(r) ? memRowDataTLen(r) : memRowKvTLen(r)) // using uint32_t/int32_t to store the TLen diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index 5392cebe84..2ac6fbe6dd 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -785,3 +785,38 @@ SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder) { return row; } +const void *tdGetNullVal(int8_t type) { + switch (type) { + case TSDB_DATA_TYPE_BOOL: + return &BoolNull; + case TSDB_DATA_TYPE_TINYINT: + return &TinyintNull; + case TSDB_DATA_TYPE_SMALLINT: + return &SmallintNull; + case TSDB_DATA_TYPE_INT: + return &IntNull; + case TSDB_DATA_TYPE_BIGINT: + return &BigintNull; + case TSDB_DATA_TYPE_FLOAT: + return &FloatNull; + case TSDB_DATA_TYPE_DOUBLE: + return &DoubleNull; + case TSDB_DATA_TYPE_BINARY: + return &BinaryNull; + case TSDB_DATA_TYPE_TIMESTAMP: + return &TimestampNull; + case TSDB_DATA_TYPE_NCHAR: + return &NcharNull; + case TSDB_DATA_TYPE_UTINYINT: + return &UTinyintNull; + case TSDB_DATA_TYPE_USMALLINT: + return &USmallintNull; + case TSDB_DATA_TYPE_UINT: + return &UIntNull; + case TSDB_DATA_TYPE_UBIGINT: + return &UBigintNull; + default: + ASSERT(0); + return NULL; + } +} diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index d62595d96f..9ac5503e5f 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -767,7 +767,7 @@ static int tsdbCopyRowToMem(STsdbRepo *pRepo, SMemRow row, STable *pTable, void void *pRow = tsdbAllocBytes(pRepo, memRowTLen(row)); if (pRow == NULL) { - tsdbError("vgId:%d failed to insert row with key %" PRId64 " to table %s while allocate %" PRIu64 " bytes since %s", + tsdbError("vgId:%d failed to insert row with key %" PRId64 " to table %s while allocate %" PRIu32 " bytes since %s", REPO_ID(pRepo), key, TABLE_CHAR_NAME(pTable), memRowTLen(row), tstrerror(terrno)); return -1; } From 91a5378f109414b4df0a1d24d7c7399fa1e678e0 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 11 Jul 2021 15:32:54 +0000 Subject: [PATCH 21/67] tiny code optimization --- src/client/src/tscParseInsert.c | 28 +++++++++++----------------- src/client/src/tscUtil.c | 3 +-- 2 files changed, 12 insertions(+), 19 deletions(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index da1a085e95..d665e2100a 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -49,13 +49,11 @@ static FORCE_INLINE int32_t getExtendedRowSize(STableComInfo *tinfo) { return tinfo->rowSize + PAYLOAD_HEADER_LEN + PAYLOAD_COL_HEAD_LEN * tinfo->numOfColumns; } int initSMemRowHelper(SMemRowHelper *pHelper, SSchema *pSSchema, uint16_t nCols, uint16_t allNullColsLen) { - ASSERT(nCols > 0); pHelper->allNullLen = allNullColsLen; // TODO: get allNullColsLen when creating or altering table meta if (pHelper->allNullLen == 0) { for (uint16_t i = 0; i < nCols; ++i) { uint8_t type = pSSchema[i].type; int32_t typeLen = TYPE_BYTES[type]; - ASSERT(typeLen > 0); pHelper->allNullLen += typeLen; if (TSDB_DATA_TYPE_BINARY == type) { pHelper->allNullLen += (sizeof(VarDataLenT) + CHAR_BYTES); @@ -787,14 +785,12 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i SParsedDataColInfo *spd = &pDataBlocks->boundColumnInfo; SSchema * schema = tscGetTableSchema(pDataBlocks->pTableMeta); - - int32_t rowSize = 0; - int32_t dataRowLen = pHelper->allNullLen; - int32_t kvRowLen = TD_MEM_ROW_KV_VER_SIZE; + TDRowTLenT dataRowLen = pHelper->allNullLen; + TDRowTLenT kvRowLen = TD_MEM_ROW_KV_VER_SIZE; TDRowTLenT payloadValOffset = 0; TDRowLenT colValOffset = 0; ASSERT(dataRowLen > 0); - + payloadSetNCols(payload, spd->numOfBound); payloadValOffset = payloadValuesOffset(payload); // rely on payloadNCols payloadSetTLen(payload, payloadValOffset); @@ -810,7 +806,6 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i char *start = payload + spd->cols[colIndex].offset; SSchema *pSchema = &schema[colIndex]; // get colId here - rowSize += pSchema->bytes; index = 0; sToken = tStrGetToken(*str, &index, true); @@ -879,9 +874,7 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i } else { ASSERT(spd->colIdxInfo[i].finalIdx == 0); } - } else { - ASSERT(spd->colIdxInfo == NULL); - } + } // the primary key locates in 1st column int32_t ret = tsParseOneColumnKV(pSchema, &sToken, payload, kvPrimaryKeyStart, kvStart, pInsertParam->msg, str, isPrimaryKey, timePrec, payloadValOffset + colValOffset, &colValAppended, @@ -1050,7 +1043,7 @@ int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int3 // expand the allocated size if (remain < rowSize * factor) { while (remain < rowSize * factor) { - pDataBlock->nAllocSize = (uint32_t)(pDataBlock->nAllocSize * 2); + pDataBlock->nAllocSize = (uint32_t)(pDataBlock->nAllocSize * 1.5); remain = pDataBlock->nAllocSize - pDataBlock->size; } @@ -1553,7 +1546,7 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat int32_t code = TSDB_CODE_SUCCESS; - int32_t index = 0; + int32_t index = 0; SStrToken sToken = tStrGetToken(str, &index, false); str += index; @@ -1561,6 +1554,7 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat code = tscSQLSyntaxErrMsg(pInsertParam->msg, "( is expected", sToken.z); goto _clean; } + bool isOrdered = true; int32_t lastColIdx = -1; while (1) { @@ -1621,13 +1615,13 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat goto _clean; } SBoundIdxInfo *pColIdx = pColInfo->colIdxInfo; - for (int i = 0; i < pColInfo->numOfBound; ++i) { + for (uint16_t i = 0; i < pColInfo->numOfBound; ++i) { pColIdx[i].schemaColIdx = (uint16_t)pColInfo->boundedColumns[i]; - pColIdx[i].boundIdx = (uint16_t)i; + pColIdx[i].boundIdx = i; } qsort(pColIdx, pColInfo->numOfBound, sizeof(SBoundIdxInfo), schemaIdxCompar); - for (int i = 0; i < pColInfo->numOfBound; ++i) { - pColIdx[i].finalIdx = (uint16_t)i; + for (uint16_t i = 0; i < pColInfo->numOfBound; ++i) { + pColIdx[i].finalIdx = i; } qsort(pColIdx, pColInfo->numOfBound, sizeof(SBoundIdxInfo), boundIdxCompar); } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index b601d5ea78..eed409fbcf 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1755,7 +1755,7 @@ static SMemRow tdGenMemRowFromBuilder(SMemRowBuilder* pBuilder) { #endif } else if (memRowType == SMEM_ROW_KV) { - ASSERT(nColsBound <= nCols); + ASSERT(nColsBound < nCols); SKVRow kvRow = (SKVRow)memRowKvBody(memRow); kvRowSetLen(kvRow, (TDRowLenT)(TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * nColsBound)); kvRowSetNCols(kvRow, nColsBound); @@ -1846,7 +1846,6 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, bo } static int32_t getRowExpandSize(STableMeta* pTableMeta) { - // add prefix len of KV type SMemRow(we may use SDataRow or SKVRow) int32_t result = TD_MEM_ROW_DATA_HEAD_SIZE; int32_t columns = tscGetNumOfColumns(pTableMeta); SSchema* pSchema = tscGetTableSchema(pTableMeta); From c5c1d65ab398d884879b464853dc8695f66544af Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 11 Jul 2021 15:51:20 +0000 Subject: [PATCH 22/67] restore and don't perform the ratio check --- src/client/inc/tsclient.h | 1 - src/client/src/tscParseInsert.c | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index c9428504b8..187b7393e6 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -42,7 +42,6 @@ extern "C" { struct SSqlInfo; #define KvRowNColsThresh 128 // default 128 -#define KVRowRatio 0.9 // for NonVarType, we get value from SDataRow directly, while needs readdressing for SKVRow typedef void (*__async_cb_func_t)(void *param, TAOS_RES *tres, int32_t numOfRows); diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index d665e2100a..9d50c1c62b 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -901,7 +901,7 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i dataRowLen += dataRowDeltaColLen; } - if (kvRowLen < dataRowLen * KVRowRatio) { + if (kvRowLen < dataRowLen) { payloadSetType(payload, SMEM_ROW_KV); } else { payloadSetType(payload, SMEM_ROW_DATA); From 51c540689b8c89e4a754429cc4d7d980110adfff Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 11 Jul 2021 16:12:25 +0000 Subject: [PATCH 23/67] remove some asserts --- src/client/src/tscUtil.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index eed409fbcf..ced48d35b2 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1712,7 +1712,6 @@ static SMemRow tdGenMemRowFromBuilder(SMemRowBuilder* pBuilder) { */ if (memRowType == SMEM_ROW_DATA) { - ASSERT(nColsBound <= nCols); SDataRow trow = (SDataRow)memRowDataBody(memRow); dataRowSetLen(trow, (TDRowLenT)(TD_DATA_ROW_HEAD_SIZE + pBuilder->flen)); dataRowSetVersion(trow, pBuilder->sversion); @@ -1755,7 +1754,6 @@ static SMemRow tdGenMemRowFromBuilder(SMemRowBuilder* pBuilder) { #endif } else if (memRowType == SMEM_ROW_KV) { - ASSERT(nColsBound < nCols); SKVRow kvRow = (SKVRow)memRowKvBody(memRow); kvRowSetLen(kvRow, (TDRowLenT)(TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * nColsBound)); kvRowSetNCols(kvRow, nColsBound); From 048c8dedd92c3dee91d7cc68ee59c0f970fcba8a Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 11 Jul 2021 21:24:35 +0000 Subject: [PATCH 24/67] test case adaption for 4096 --- tests/pytest/table/max_table_length.py | 2 +- .../pytest/tools/taosdemoAllTest/taosdemoTestInsertWithJson.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pytest/table/max_table_length.py b/tests/pytest/table/max_table_length.py index ec34f3008f..366e20456d 100644 --- a/tests/pytest/table/max_table_length.py +++ b/tests/pytest/table/max_table_length.py @@ -42,7 +42,7 @@ class TDTestCase: print("==============step3") tdLog.info("check int & binary") - tdSql.error("create table anal2 (ts timestamp ,i binary(16371),j int)") + # tdSql.error("create table anal2 (ts timestamp ,i binary(16371),j int)") tdSql.execute("create table anal2 (ts timestamp ,i binary(16370),j int)") tdSql.execute("create table anal3 (ts timestamp ,i binary(16366), j int, k int)") diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoTestInsertWithJson.py b/tests/pytest/tools/taosdemoAllTest/taosdemoTestInsertWithJson.py index 01e46eaaa0..128c63413b 100644 --- a/tests/pytest/tools/taosdemoAllTest/taosdemoTestInsertWithJson.py +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoTestInsertWithJson.py @@ -198,7 +198,7 @@ class TDTestCase: tdSql.checkRows(1) tdSql.query("select count(*) from db.stb1") tdSql.checkRows(1) - tdSql.error("select * from db.stb3") + # tdSql.error("select * from db.stb3") tdSql.error("select * from db.stb2") tdSql.execute("drop database if exists db") os.system("%staosdemo -f tools/taosdemoAllTest/insertNumOfrecordPerReq0.json -y " % binPath) From f97d290b31174e8f36b2cc7a845a8618176b2e96 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 12 Jul 2021 00:07:31 +0000 Subject: [PATCH 25/67] adjust the max bytes per row --- src/inc/taosdef.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 9ab2542eb9..30d79f7116 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -197,9 +197,10 @@ do { \ /** * Don't change to 65536. As in some scenarios uint16_t (0~65535) is used to store the row len. - * Finally, we use 65531(65535 - 4), as the SDataRow and SKVRow including 4 bits header. + * So, we use 65531(65535 - 4), as the SDataRow and SKVRow including 4 bits header. + * If all cols are VarType except primary key, we need 4 bits to store the offset. 65531-8-4095*4=49143 */ -#define TSDB_MAX_BYTES_PER_ROW 65531 +#define TSDB_MAX_BYTES_PER_ROW 49143 #define TSDB_MAX_TAGS_LEN 16384 #define TSDB_MAX_TAGS 128 #define TSDB_MAX_TAG_CONDITIONS 1024 From 21a8b0917475578bc3d7bae0c39f2be1da3a1e7d Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 12 Jul 2021 02:26:39 +0000 Subject: [PATCH 26/67] appendKvRow with NULL optimization --- src/common/inc/tdataformat.h | 8 ++++++-- src/common/src/tdataformat.c | 26 +++++++++++++++++++++----- src/tsdb/src/tsdbCommit.c | 3 ++- src/tsdb/src/tsdbRead.c | 2 +- 4 files changed, 30 insertions(+), 9 deletions(-) diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index 573836a4e7..46d13d7094 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -265,10 +265,11 @@ typedef struct SDataCol { TSKEY ts; // only used in last NULL column } SDataCol; +#define isAllRowsNull(pCol) ((pCol)->len == 0) static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; } void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints); -void dataColAppendVal(SDataCol *pCol, void *value, int numOfRows, int maxPoints); +void dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints); void dataColSetOffset(SDataCol *pCol, int nEle); bool isNEleNull(SDataCol *pCol, int nEle); @@ -277,7 +278,10 @@ void dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints); const void *tdGetNullVal(int8_t type); // Get the data pointer from a column-wised data -static FORCE_INLINE void *tdGetColDataOfRow(SDataCol *pCol, int row) { +static FORCE_INLINE const void *tdGetColDataOfRow(SDataCol *pCol, int row) { + if (isAllRowsNull(pCol)) { + return tdGetNullVal(pCol->type); + } if (IS_VAR_DATA_TYPE(pCol->type)) { return POINTER_SHIFT(pCol->pData, pCol->dataOff[row]); } else { diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index 2ac6fbe6dd..f26f7511c1 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -241,9 +241,21 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints) } } // value from timestamp should be TKEY here instead of TSKEY -void dataColAppendVal(SDataCol *pCol, void *value, int numOfRows, int maxPoints) { +void dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints) { ASSERT(pCol != NULL && value != NULL); + if (isAllRowsNull(pCol)) { + if (isNull(value, pCol->type)) { + // all null value yet, just return + return; + } + + if (numOfRows > 0) { + // Find the first not null value, fill all previouse values as NULL + dataColSetNEleNull(pCol, numOfRows, maxPoints); + } + } + if (IS_VAR_DATA_TYPE(pCol->type)) { // set offset pCol->dataOff[numOfRows] = pCol->len; @@ -440,7 +452,8 @@ static void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols while (dcol < pCols->numOfCols) { SDataCol *pDataCol = &(pCols->cols[dcol]); if (rcol >= schemaNCols(pSchema)) { - dataColSetNullAt(pDataCol, pCols->numOfRows); + // dataColSetNullAt(pDataCol, pCols->numOfRows); + dataColAppendVal(pDataCol, tdGetNullVal(pDataCol->type), pCols->numOfRows, pCols->maxPoints); dcol++; continue; } @@ -454,7 +467,8 @@ static void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols } else if (pRowCol->colId < pDataCol->colId) { rcol++; } else { - dataColSetNullAt(pDataCol, pCols->numOfRows); + // dataColSetNullAt(pDataCol, pCols->numOfRows); + dataColAppendVal(pDataCol, tdGetNullVal(pDataCol->type), pCols->numOfRows, pCols->maxPoints); dcol++; } } @@ -483,7 +497,8 @@ static void tdAppendKvRowToDataCol(SKVRow row, STSchema *pSchema, SDataCols *pCo while (dcol < pCols->numOfCols) { SDataCol *pDataCol = &(pCols->cols[dcol]); if (rcol >= nRowCols || rcol >= schemaNCols(pSchema)) { - dataColSetNullAt(pDataCol, pCols->numOfRows); + // dataColSetNullAt(pDataCol, pCols->numOfRows); + dataColAppendVal(pDataCol, tdGetNullVal(pDataCol->type), pCols->numOfRows, pCols->maxPoints); ++dcol; continue; } @@ -498,7 +513,8 @@ static void tdAppendKvRowToDataCol(SKVRow row, STSchema *pSchema, SDataCols *pCo } else if (colIdx->colId < pDataCol->colId) { ++rcol; } else { - dataColSetNullAt(pDataCol, pCols->numOfRows); + // dataColSetNullAt(pDataCol, pCols->numOfRows); + dataColAppendVal(pDataCol, tdGetNullVal(pDataCol->type), pCols->numOfRows, pCols->maxPoints); ++dcol; } } diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index 8718d5c35a..6330da6058 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -920,7 +920,8 @@ int tsdbWriteBlockImpl(STsdbRepo *pRepo, STable *pTable, SDFile *pDFile, SDataCo SDataCol * pDataCol = pDataCols->cols + ncol; SBlockCol *pBlockCol = pBlockData->cols + nColsNotAllNull; - if (isNEleNull(pDataCol, rowsToWrite)) { // all data to commit are NULL, just ignore it + // if (isNEleNull(pDataCol, rowsToWrite)) { // all data to commit are NULL, just ignore it + if (isAllRowsNull(pDataCol)) { // all data to commit are NULL, just ignore it continue; } diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 22162da5c6..1a301f4903 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -1391,7 +1391,7 @@ int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity // todo refactor, only copy one-by-one for (int32_t k = start; k < num + start; ++k) { - char* p = tdGetColDataOfRow(src, k); + const char* p = tdGetColDataOfRow(src, k); memcpy(dst, p, varDataTLen(p)); dst += bytes; } From 889aa8a11b74b57d165eb0e6ac8cb4d03d9063f6 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 12 Jul 2021 05:10:33 +0000 Subject: [PATCH 27/67] code refactor --- src/client/inc/tsclient.h | 14 +++++--------- src/client/src/tscParseInsert.c | 10 +++++----- src/client/src/tscUtil.c | 1 - src/common/inc/tdataformat.h | 5 ++--- src/inc/taosdef.h | 7 ++++--- 5 files changed, 16 insertions(+), 21 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 187b7393e6..bb14b28499 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -104,15 +104,11 @@ typedef struct SParsedDataColInfo { } SParsedDataColInfo; typedef struct { - // for SDataRow - SSchema *pSchema; - int16_t sversion; - int32_t flen; - // for SKVRow - uint16_t nCols; - uint16_t size; - void * buf; - + SSchema * pSchema; + int16_t sversion; + int32_t flen; + uint16_t nCols; + void * buf; void * pDataBlock; SSubmitBlk *pSubmitBlk; } SMemRowBuilder; diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 9d50c1c62b..3b21601caf 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -48,7 +48,7 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat static FORCE_INLINE int32_t getExtendedRowSize(STableComInfo *tinfo) { return tinfo->rowSize + PAYLOAD_HEADER_LEN + PAYLOAD_COL_HEAD_LEN * tinfo->numOfColumns; } -int initSMemRowHelper(SMemRowHelper *pHelper, SSchema *pSSchema, uint16_t nCols, uint16_t allNullColsLen) { +static int initSMemRowHelper(SMemRowHelper *pHelper, SSchema *pSSchema, uint16_t nCols, uint16_t allNullColsLen) { pHelper->allNullLen = allNullColsLen; // TODO: get allNullColsLen when creating or altering table meta if (pHelper->allNullLen == 0) { for (uint16_t i = 0; i < nCols; ++i) { @@ -406,7 +406,7 @@ static FORCE_INLINE TDRowLenT tsSetPayloadColValue(char *payloadStart, char *pay payloadColSetId(payload, columnId); payloadColSetType(payload, columnType); memcpy(POINTER_SHIFT(payloadStart,tOffset), value, valueLen); - payloadSetTLen(payloadStart, payloadTLen(payloadStart) + valueLen); + // payloadSetTLen(payloadStart, payloadTLen(payloadStart) + valueLen); return valueLen; } @@ -793,7 +793,7 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i payloadSetNCols(payload, spd->numOfBound); payloadValOffset = payloadValuesOffset(payload); // rely on payloadNCols - payloadSetTLen(payload, payloadValOffset); + // payloadSetTLen(payload, payloadValOffset); char *kvPrimaryKeyStart = payload + PAYLOAD_HEADER_LEN; // primaryKey in 1st column tuple char *kvStart = kvPrimaryKeyStart + PAYLOAD_COL_HEAD_LEN; // the column tuple behind the primaryKey @@ -1138,7 +1138,7 @@ int tscSortRemoveDataBlockDupRows(STableDataBlocks *dataBuf, SBlockKeyInfo *pBlk size_t nAlloc = nRows * sizeof(SBlockKeyTuple); if (pBlkKeyInfo->pKeyTuple == NULL || pBlkKeyInfo->maxBytesAlloc < nAlloc) { size_t nRealAlloc = nAlloc + 10 * sizeof(SBlockKeyTuple); - char * tmp = realloc(pBlkKeyInfo->pKeyTuple, nRealAlloc); + char * tmp = trealloc(pBlkKeyInfo->pKeyTuple, nRealAlloc); if (tmp == NULL) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -1609,7 +1609,7 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat pColInfo->isOrdered = isOrdered; if (!isOrdered) { - pColInfo->colIdxInfo = calloc(pColInfo->numOfBound, sizeof(SBoundIdxInfo)); + pColInfo->colIdxInfo = tcalloc(pColInfo->numOfBound, sizeof(SBoundIdxInfo)); if (pColInfo->colIdxInfo == NULL) { code = TSDB_CODE_TSC_OUT_OF_MEMORY; goto _clean; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index ced48d35b2..40f1b7b2fb 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1829,7 +1829,6 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, bo rowBuilder.pDataBlock = pDataBlock; rowBuilder.pSubmitBlk = pBlock; rowBuilder.buf = p; - rowBuilder.size = 0; for (int32_t i = 0; i < numOfRows; ++i) { rowBuilder.buf = (blkKeyTuple + i)->payloadAddr; diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index 46d13d7094..add2a656f5 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -190,7 +190,7 @@ static FORCE_INLINE int tkeyComparFn(const void *tkey1, const void *tkey2) { // ----------------- Data row structure /* A data row, the format is like below: - * |<--------------------+--------------------------- len ---------------------------------->| + * |<------------------------------------------------ len ---------------------------------->| * |<-- Head -->|<--------- flen -------------->| | * +---------------------+---------------------------------+---------------------------------+ * | uint16_t | int16_t | | | @@ -224,11 +224,10 @@ SDataRow tdDataRowDup(SDataRow row); static FORCE_INLINE int tdAppendColVal(SDataRow row, const void *value, int8_t type, int32_t offset) { ASSERT(value != NULL); int32_t toffset = offset + TD_DATA_ROW_HEAD_SIZE; - char * ptr = (char *)POINTER_SHIFT(row, dataRowLen(row)); if (IS_VAR_DATA_TYPE(type)) { *(VarDataOffsetT *)POINTER_SHIFT(row, toffset) = dataRowLen(row); - memcpy(ptr, value, varDataTLen(value)); + memcpy(POINTER_SHIFT(row, dataRowLen(row)), value, varDataTLen(value)); dataRowLen(row) += varDataTLen(value); } else { if (offset == 0) { diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 30d79f7116..acebdaa30a 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -196,9 +196,10 @@ do { \ #define TSDB_APPNAME_LEN TSDB_UNI_LEN /** - * Don't change to 65536. As in some scenarios uint16_t (0~65535) is used to store the row len. - * So, we use 65531(65535 - 4), as the SDataRow and SKVRow including 4 bits header. - * If all cols are VarType except primary key, we need 4 bits to store the offset. 65531-8-4095*4=49143 + * In some scenarios uint16_t (0~65535) is used to store the row len. + * - Firstly, we use 65531(65535 - 4), as the SDataRow and SKVRow including 4 bits header. + * - Secondly, if all cols are VarType except primary key, we need 4 bits to store the offset, thus + * the final value is 65531-8-4095*4 = 49143. */ #define TSDB_MAX_BYTES_PER_ROW 49143 #define TSDB_MAX_TAGS_LEN 16384 From f24dca541f8ba9a6a4ad79023ef87354209ff580 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 12 Jul 2021 23:21:47 +0000 Subject: [PATCH 28/67] refactor the payload to K_V format for prepare mode --- src/client/inc/tsclient.h | 15 ++- src/client/src/tscParseInsert.c | 22 +-- src/client/src/tscPrepare.c | 232 ++++++++++++++++++++++++++++++-- 3 files changed, 247 insertions(+), 22 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index bb14b28499..072db868d0 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -89,18 +89,26 @@ typedef struct SBoundColumn { bool hasVal; // denote if current column has bound or not int32_t offset; // all column offset value } SBoundColumn; + typedef struct { uint16_t schemaColIdx; uint16_t boundIdx; uint16_t finalIdx; } SBoundIdxInfo; + +typedef enum _COL_ORDER_STATUS { + ORDER_STATUS_UNKNOWN = 0, + ORDER_STATUS_ORDERED = 1, + ORDER_STATUS_DISORDERED = 2, +} EOrderStatus; + typedef struct SParsedDataColInfo { - bool isOrdered; // bounded columns int16_t numOfCols; int16_t numOfBound; int32_t * boundedColumns; // bounded column idx according to schema SBoundColumn * cols; SBoundIdxInfo *colIdxInfo; + int8_t orderStatus; // bounded columns: } SParsedDataColInfo; typedef struct { @@ -409,6 +417,11 @@ extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo); void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArray* tables); int16_t getNewResColId(SSqlCmd* pCmd); +int32_t schemaIdxCompar(const void *lhs, const void *rhs); +int32_t boundIdxCompar(const void *lhs, const void *rhs); +int initSMemRowHelper(SMemRowHelper *pHelper, SSchema *pSSchema, uint16_t nCols, uint16_t allNullColsLen); +int32_t getExtendedRowSize(STableComInfo *tinfo); + #ifdef __cplusplus } #endif diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 3b21601caf..edf2421746 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -45,10 +45,10 @@ static int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSiz static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDataColInfo *pColInfo, SSchema *pSchema, char *str, char **end); -static FORCE_INLINE int32_t getExtendedRowSize(STableComInfo *tinfo) { +int32_t getExtendedRowSize(STableComInfo *tinfo) { return tinfo->rowSize + PAYLOAD_HEADER_LEN + PAYLOAD_COL_HEAD_LEN * tinfo->numOfColumns; } -static int initSMemRowHelper(SMemRowHelper *pHelper, SSchema *pSSchema, uint16_t nCols, uint16_t allNullColsLen) { +int initSMemRowHelper(SMemRowHelper *pHelper, SSchema *pSSchema, uint16_t nCols, uint16_t allNullColsLen) { pHelper->allNullLen = allNullColsLen; // TODO: get allNullColsLen when creating or altering table meta if (pHelper->allNullLen == 0) { for (uint16_t i = 0; i < nCols; ++i) { @@ -56,9 +56,9 @@ static int initSMemRowHelper(SMemRowHelper *pHelper, SSchema *pSSchema, uint16_t int32_t typeLen = TYPE_BYTES[type]; pHelper->allNullLen += typeLen; if (TSDB_DATA_TYPE_BINARY == type) { - pHelper->allNullLen += (sizeof(VarDataLenT) + CHAR_BYTES); + pHelper->allNullLen += (VARSTR_HEADER_SIZE + CHAR_BYTES); } else if (TSDB_DATA_TYPE_NCHAR == type) { - int len = sizeof(VarDataLenT) + TSDB_NCHAR_SIZE; + int len = VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE; pHelper->allNullLen += len; } } @@ -867,14 +867,14 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i TDRowLenT kvRowColLen = 0; TDRowLenT colValAppended = 0; - if(!spd->isOrdered) { + if (spd->orderStatus == ORDER_STATUS_DISORDERED) { ASSERT(spd->colIdxInfo != NULL); if(!isPrimaryKey) { kvStart = POINTER_SHIFT(kvPrimaryKeyStart, spd->colIdxInfo[i].finalIdx * PAYLOAD_COL_HEAD_LEN); } else { ASSERT(spd->colIdxInfo[i].finalIdx == 0); } - } + } // the primary key locates in 1st column int32_t ret = tsParseOneColumnKV(pSchema, &sToken, payload, kvPrimaryKeyStart, kvStart, pInsertParam->msg, str, isPrimaryKey, timePrec, payloadValOffset + colValOffset, &colValAppended, @@ -891,7 +891,7 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i payloadColSetOffset(kvPrimaryKeyStart, colValOffset); } else { payloadColSetOffset(kvStart, colValOffset); - if(spd->isOrdered) { + if (spd->orderStatus == ORDER_STATUS_ORDERED) { kvStart += PAYLOAD_COL_HEAD_LEN; // move to next column } } @@ -929,7 +929,7 @@ static int32_t rowDataCompar(const void *lhs, const void *rhs) { return left > right ? 1 : -1; } } -static int32_t schemaIdxCompar(const void *lhs, const void *rhs) { +int32_t schemaIdxCompar(const void *lhs, const void *rhs) { uint16_t left = *(uint16_t *)lhs; uint16_t right = *(uint16_t *)rhs; @@ -940,7 +940,7 @@ static int32_t schemaIdxCompar(const void *lhs, const void *rhs) { } } -static int32_t boundIdxCompar(const void *lhs, const void *rhs) { +int32_t boundIdxCompar(const void *lhs, const void *rhs) { uint16_t left = *(uint16_t *)POINTER_SHIFT(lhs, sizeof(uint16_t)); uint16_t right = *(uint16_t *)POINTER_SHIFT(rhs, sizeof(uint16_t)); @@ -1020,7 +1020,7 @@ int32_t tsParseValues(char **str, STableDataBlocks *pDataBlock, int maxRows, SIn void tscSetBoundColumnInfo(SParsedDataColInfo *pColInfo, SSchema *pSchema, int32_t numOfCols) { pColInfo->numOfCols = numOfCols; pColInfo->numOfBound = numOfCols; - pColInfo->isOrdered = true; + pColInfo->orderStatus = ORDER_STATUS_UNKNOWN; pColInfo->boundedColumns = calloc(pColInfo->numOfCols, sizeof(int32_t)); pColInfo->cols = calloc(pColInfo->numOfCols, sizeof(SBoundColumn)); pColInfo->colIdxInfo = NULL; @@ -1606,7 +1606,7 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat } } - pColInfo->isOrdered = isOrdered; + pColInfo->orderStatus = isOrdered ? ORDER_STATUS_ORDERED : ORDER_STATUS_DISORDERED; if (!isOrdered) { pColInfo->colIdxInfo = tcalloc(pColInfo->numOfBound, sizeof(SBoundIdxInfo)); diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 08d3cc599e..68fa2454d1 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -291,6 +291,7 @@ static char* normalStmtBuildSql(STscStmt* stmt) { return taosStringBuilderGetResult(&sb, NULL); } +#if 0 static int fillColumnsNull(STableDataBlocks* pBlock, int32_t rowNum) { SParsedDataColInfo* spd = &pBlock->boundColumnInfo; int32_t offset = 0; @@ -318,8 +319,135 @@ static int fillColumnsNull(STableDataBlocks* pBlock, int32_t rowNum) { return TSDB_CODE_SUCCESS; } +#endif + +/** + * input: + * - schema: + * - payload: + * - spd: + * output: + * - pBlock with data block replaced by K-V format + */ +static int refactorPayload(STableDataBlocks* pBlock, int32_t rowNum) { + SParsedDataColInfo* spd = &pBlock->boundColumnInfo; + SSchema* schema = (SSchema*)pBlock->pTableMeta->schema; + SMemRowHelper* pHelper = &pBlock->rowHelper; + STableMeta* pTableMeta = pBlock->pTableMeta; + STableComInfo tinfo = tscGetTableInfo(pTableMeta); + int code = TSDB_CODE_SUCCESS; + int32_t extendedRowSize = getExtendedRowSize(&tinfo); + TDRowTLenT destPayloadSize = sizeof(SSubmitBlk); + + ASSERT(pHelper->allNullLen >= 8); + + TDRowTLenT destAllocSize = sizeof(SSubmitBlk) + rowNum * extendedRowSize; + SSubmitBlk* pDestBlock = tcalloc(destAllocSize, 1); + if (pDestBlock == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + memcpy(pDestBlock, pBlock->pData, sizeof(SSubmitBlk)); + char* destPayload = (char*)pDestBlock + sizeof(SSubmitBlk); + + char* srcPayload = (char*)pBlock->pData + sizeof(SSubmitBlk); + + for (int n = 0; n < rowNum; ++n) { + payloadSetNCols(destPayload, spd->numOfBound); + + TDRowTLenT dataRowLen = pHelper->allNullLen; + TDRowTLenT kvRowLen = TD_MEM_ROW_KV_VER_SIZE + sizeof(SColIdx) * spd->numOfBound; + TDRowTLenT payloadValOffset = payloadValuesOffset(destPayload); // rely on payloadNCols + TDRowLenT colValOffset = 0; + + char* kvPrimaryKeyStart = destPayload + PAYLOAD_HEADER_LEN; // primaryKey in 1st column tuple + char* kvStart = kvPrimaryKeyStart + PAYLOAD_COL_HEAD_LEN; // the column tuple behind the primaryKey + + for (int32_t i = 0; i < spd->numOfBound; ++i) { + int32_t colIndex = spd->boundedColumns[i]; + ASSERT(spd->cols[colIndex].hasVal); + char* start = srcPayload + spd->cols[colIndex].offset; + SSchema* pSchema = &schema[colIndex]; // get colId here + bool isPrimaryKey = (colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX); + + // the primary key locates in 1st column + if (spd->orderStatus == ORDER_STATUS_DISORDERED) { + ASSERT(spd->colIdxInfo != NULL); + if (!isPrimaryKey) { + kvStart = POINTER_SHIFT(kvPrimaryKeyStart, spd->colIdxInfo[i].finalIdx * PAYLOAD_COL_HEAD_LEN); + } else { + ASSERT(spd->colIdxInfo[i].finalIdx == 0); + } + } + if (isPrimaryKey) { + payloadColSetId(kvPrimaryKeyStart, pSchema->colId); + payloadColSetType(kvPrimaryKeyStart, pSchema->type); + payloadColSetOffset(kvPrimaryKeyStart, colValOffset); + memcpy(POINTER_SHIFT(destPayload, payloadValOffset + colValOffset), start, TYPE_BYTES[pSchema->type]); + colValOffset += TYPE_BYTES[pSchema->type]; + kvRowLen += TYPE_BYTES[pSchema->type]; + } else { + payloadColSetId(kvStart, pSchema->colId); + payloadColSetType(kvStart, pSchema->type); + payloadColSetOffset(kvStart, colValOffset); + if (IS_VAR_DATA_TYPE(pSchema->type)) { + varDataCopy(POINTER_SHIFT(destPayload, payloadValOffset + colValOffset), start); + colValOffset += varDataTLen(start); + kvRowLen += varDataTLen(start); + if (pSchema->type == TSDB_DATA_TYPE_BINARY) { + dataRowLen += (varDataLen(start) - CHAR_BYTES); + } else if (pSchema->type == TSDB_DATA_TYPE_NCHAR) { + dataRowLen += (varDataLen(start) - TSDB_NCHAR_SIZE); + } else { + ASSERT(0); + } + } else { + memcpy(POINTER_SHIFT(destPayload, payloadValOffset + colValOffset), start, TYPE_BYTES[pSchema->type]); + colValOffset += TYPE_BYTES[pSchema->type]; + kvRowLen += TYPE_BYTES[pSchema->type]; + } + + if (spd->orderStatus == ORDER_STATUS_ORDERED) { + kvStart += PAYLOAD_COL_HEAD_LEN; // move to next column + } + } + } // end of column + + if (kvRowLen < dataRowLen) { + payloadSetType(destPayload, SMEM_ROW_KV); + } else { + payloadSetType(destPayload, SMEM_ROW_DATA); + } + + ASSERT(colValOffset <= TSDB_MAX_BYTES_PER_ROW); + + TDRowTLenT len = payloadValOffset + colValOffset; + payloadSetTLen(destPayload, len); + +#if 0 + TSKEY tsKey = payloadKey(destPayload); + ASSERT((tsKey < 1627747200000000 && tsKey > 1498838400000000) || (tsKey < 1627747200000 && tsKey > 1498838400000) || + (tsKey < 1627747200 && tsKey > 1498838400)); +#endif + + // next loop + srcPayload += pBlock->rowSize; + destPayload += len; + + destPayloadSize += len; + } // end of row + + ASSERT(destPayloadSize <= destAllocSize); + + tfree(pBlock->pData); + pBlock->pData = (char*)pDestBlock; + pBlock->nAllocSize = destAllocSize; + pBlock->size = destPayloadSize; + + return code; +} +#if 0 int32_t fillTablesColumnsNull(SSqlObj* pSql) { SSqlCmd* pCmd = &pSql->cmd; @@ -342,17 +470,98 @@ int32_t fillTablesColumnsNull(SSqlObj* pSql) { return TSDB_CODE_SUCCESS; } +#endif - - -//////////////////////////////////////////////////////////////////////////////// -// functions for insertion statement preparation -static int doBindParam(STableDataBlocks* pBlock, char* data, SParamInfo* param, TAOS_BIND* bind, int32_t colNum) { - if (bind->is_null != NULL && *(bind->is_null)) { - setNull(data + param->offset, param->type, param->bytes); +/** + * check and sort + */ +static int initPayloadEnv(STableDataBlocks* pBlock, int32_t rowNum) { + SParsedDataColInfo* spd = &pBlock->boundColumnInfo; + if (spd->orderStatus != ORDER_STATUS_UNKNOWN) { return TSDB_CODE_SUCCESS; } + bool isOrdered = true; + int32_t lastColIdx = -1; + for (int32_t i = 0; i < spd->numOfBound; ++i) { + ASSERT(spd->cols[i].hasVal); + int32_t colIdx = spd->boundedColumns[i]; + if (isOrdered) { + if (lastColIdx > colIdx) { + isOrdered = false; + break; + } else { + lastColIdx = colIdx; + } + } + } + + spd->orderStatus = isOrdered ? ORDER_STATUS_ORDERED : ORDER_STATUS_DISORDERED; + + if (isOrdered) { + spd->colIdxInfo = NULL; + } else { + spd->colIdxInfo = calloc(spd->numOfBound, sizeof(SBoundIdxInfo)); + if (spd->colIdxInfo == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + SBoundIdxInfo* pColIdx = spd->colIdxInfo; + for (uint16_t i = 0; i < spd->numOfBound; ++i) { + pColIdx[i].schemaColIdx = (uint16_t)spd->boundedColumns[i]; + pColIdx[i].boundIdx = i; + } + qsort(pColIdx, spd->numOfBound, sizeof(SBoundIdxInfo), schemaIdxCompar); + for (uint16_t i = 0; i < spd->numOfBound; ++i) { + pColIdx[i].finalIdx = i; + } + qsort(pColIdx, spd->numOfBound, sizeof(SBoundIdxInfo), boundIdxCompar); + } + + return TSDB_CODE_SUCCESS; +} + +/** + * Refactor the raw payload structure to K-V format as the in tsParseOneRow() + */ +int32_t fillTablesPayload(SSqlObj* pSql) { + SSqlCmd* pCmd = &pSql->cmd; + int code = TSDB_CODE_SUCCESS; + + STableDataBlocks** p = taosHashIterate(pCmd->insertParam.pTableBlockHashList, NULL); + + STableDataBlocks* pOneTableBlock = *p; + while (pOneTableBlock) { + SSubmitBlk* pBlocks = (SSubmitBlk*)pOneTableBlock->pData; + + if (pBlocks->numOfRows > 0) { + initSMemRowHelper(&pOneTableBlock->rowHelper, tscGetTableSchema(pOneTableBlock->pTableMeta), + tscGetNumOfColumns(pOneTableBlock->pTableMeta), 0); + if ((code = initPayloadEnv(pOneTableBlock, pBlocks->numOfRows)) != TSDB_CODE_SUCCESS) { + return code; + } + if ((code = refactorPayload(pOneTableBlock, pBlocks->numOfRows)) != TSDB_CODE_SUCCESS) { + return code; + }; + } + + p = taosHashIterate(pCmd->insertParam.pTableBlockHashList, p); + if (p == NULL) { + break; + } + + pOneTableBlock = *p; + } + + return code; +} + //////////////////////////////////////////////////////////////////////////////// + // functions for insertion statement preparation + static int doBindParam(STableDataBlocks* pBlock, char* data, SParamInfo* param, TAOS_BIND* bind, int32_t colNum) { + if (bind->is_null != NULL && *(bind->is_null)) { + setNull(data + param->offset, param->type, param->bytes); + return TSDB_CODE_SUCCESS; + } + #if 0 if (0) { // allow user bind param data with different type @@ -1106,9 +1315,12 @@ static int insertStmtExecute(STscStmt* stmt) { pBlk->uid = pTableMeta->id.uid; pBlk->tid = pTableMeta->id.tid; - fillTablesColumnsNull(stmt->pSql); + int code = fillTablesPayload(stmt->pSql); + if (code != TSDB_CODE_SUCCESS) { + return code; + } - int code = tscMergeTableDataBlocks(&stmt->pSql->cmd.insertParam, false); + code = tscMergeTableDataBlocks(&stmt->pSql->cmd.insertParam, false); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1185,7 +1397,7 @@ static int insertBatchStmtExecute(STscStmt* pStmt) { return TSDB_CODE_TSC_APP_ERROR; } - fillTablesColumnsNull(pStmt->pSql); + fillTablesPayload(pStmt->pSql); if ((code = tscMergeTableDataBlocks(&pStmt->pSql->cmd.insertParam, false)) != TSDB_CODE_SUCCESS) { return code; From e65340519e8fe0f1c1d96d3f5c24ebc2321b6ed1 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 13 Jul 2021 03:00:33 +0000 Subject: [PATCH 29/67] refactor --- src/client/inc/tsclient.h | 2 ++ src/client/src/tscParseInsert.c | 4 ++-- src/client/src/tscPrepare.c | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 072db868d0..c23ca2bc98 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -111,6 +111,8 @@ typedef struct SParsedDataColInfo { int8_t orderStatus; // bounded columns: } SParsedDataColInfo; +#define IS_DATA_COL_ORDERED(s) (((s)->orderStatus) == (int8_t)ORDER_STATUS_ORDERED) + typedef struct { SSchema * pSchema; int16_t sversion; diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index edf2421746..5d3f41acd3 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -867,7 +867,7 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i TDRowLenT kvRowColLen = 0; TDRowLenT colValAppended = 0; - if (spd->orderStatus == ORDER_STATUS_DISORDERED) { + if (!IS_DATA_COL_ORDERED(spd)) { ASSERT(spd->colIdxInfo != NULL); if(!isPrimaryKey) { kvStart = POINTER_SHIFT(kvPrimaryKeyStart, spd->colIdxInfo[i].finalIdx * PAYLOAD_COL_HEAD_LEN); @@ -1020,7 +1020,7 @@ int32_t tsParseValues(char **str, STableDataBlocks *pDataBlock, int maxRows, SIn void tscSetBoundColumnInfo(SParsedDataColInfo *pColInfo, SSchema *pSchema, int32_t numOfCols) { pColInfo->numOfCols = numOfCols; pColInfo->numOfBound = numOfCols; - pColInfo->orderStatus = ORDER_STATUS_UNKNOWN; + pColInfo->orderStatus = ORDER_STATUS_ORDERED; pColInfo->boundedColumns = calloc(pColInfo->numOfCols, sizeof(int32_t)); pColInfo->cols = calloc(pColInfo->numOfCols, sizeof(SBoundColumn)); pColInfo->colIdxInfo = NULL; diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 68fa2454d1..3739fbaca5 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -370,7 +370,7 @@ static int refactorPayload(STableDataBlocks* pBlock, int32_t rowNum) { bool isPrimaryKey = (colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX); // the primary key locates in 1st column - if (spd->orderStatus == ORDER_STATUS_DISORDERED) { + if (!IS_DATA_COL_ORDERED(spd)) { ASSERT(spd->colIdxInfo != NULL); if (!isPrimaryKey) { kvStart = POINTER_SHIFT(kvPrimaryKeyStart, spd->colIdxInfo[i].finalIdx * PAYLOAD_COL_HEAD_LEN); @@ -406,7 +406,7 @@ static int refactorPayload(STableDataBlocks* pBlock, int32_t rowNum) { kvRowLen += TYPE_BYTES[pSchema->type]; } - if (spd->orderStatus == ORDER_STATUS_ORDERED) { + if (IS_DATA_COL_ORDERED(spd)) { kvStart += PAYLOAD_COL_HEAD_LEN; // move to next column } } From 9dcaa53e128c6048c332809e7bc46623247d2844 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 13 Jul 2021 06:43:49 +0000 Subject: [PATCH 30/67] code optimization --- src/client/src/tscParseInsert.c | 4 ++-- src/inc/taosdef.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 5d3f41acd3..0081dac11b 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -685,8 +685,8 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay if (pToken->type == TK_NULL) { payloadColSetId(payload, pSchema->colId); payloadColSetType(payload, pSchema->type); - memcpy(POINTER_SHIFT(payloadStart,tOffset), tdGetNullVal(TSDB_DATA_TYPE_NCHAR), VARSTR_HEADER_SIZE + INT_BYTES); - *sizeAppend = (TDRowLenT)(VARSTR_HEADER_SIZE + INT_BYTES); + memcpy(POINTER_SHIFT(payloadStart,tOffset), tdGetNullVal(TSDB_DATA_TYPE_NCHAR), VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE); + *sizeAppend = (TDRowLenT)(VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE); } else { // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' int32_t output = 0; diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index acebdaa30a..7c6b3bbed9 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -199,9 +199,9 @@ do { \ * In some scenarios uint16_t (0~65535) is used to store the row len. * - Firstly, we use 65531(65535 - 4), as the SDataRow and SKVRow including 4 bits header. * - Secondly, if all cols are VarType except primary key, we need 4 bits to store the offset, thus - * the final value is 65531-8-4095*4 = 49143. + * the final value is 65531-(4096-1)*4 = 49151. */ -#define TSDB_MAX_BYTES_PER_ROW 49143 +#define TSDB_MAX_BYTES_PER_ROW 49151 #define TSDB_MAX_TAGS_LEN 16384 #define TSDB_MAX_TAGS 128 #define TSDB_MAX_TAG_CONDITIONS 1024 From 704fce3263d14ab748957e93d3ca0ed55e657de6 Mon Sep 17 00:00:00 2001 From: happyguoxy Date: Wed, 14 Jul 2021 10:22:02 +0800 Subject: [PATCH 31/67] [TD-5213]test --- .../TD-5213/insertSigcolumnsNum4096.py | 137 ++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100755 tests/pytest/tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.py diff --git a/tests/pytest/tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.py b/tests/pytest/tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.py new file mode 100755 index 0000000000..b710e34d8c --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.py @@ -0,0 +1,137 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + def run(self): + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + binPath = buildPath+ "/build/bin/" + + # insert: create one or mutiple tables per sql and insert multiple rows per sql + # test case for https://jira.taosdata.com:18080/browse/TD-5213 + os.system("%staosdemo -f tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.json -y " % binPath) + tdSql.execute("use db") + tdSql.query("select count (tbname) from stb_old") + tdSql.checkData(0, 0, 10) + + # tdSql.query("select * from stb_old") + # tdSql.checkRows(10) + # tdSql.checkCols(1024) + + # tdSql.query("select count (tbname) from stb_new") + # tdSql.checkData(0, 0, 10) + + # tdSql.query("select * from stb_new") + # tdSql.checkRows(10) + # tdSql.checkCols(4096) + + # tdLog.info("stop dnode to commit data to disk") + # tdDnodes.stop(1) + # tdDnodes.start(1) + + #regular table + sql = "create table tb(ts timestamp, " + for i in range(1022): + sql += "col%d binary(14), " % (i + 1) + sql += "col1023 binary(22))" + tdSql.execute(sql) + + for i in range(4): + sql = "insert into tb values(%d, " + for j in range(1022): + str = "'%s', " % self.get_random_string(14) + sql += str + sql += "'%s')" % self.get_random_string(22) + tdSql.execute(sql % (self.ts + i)) + + time.sleep(10) + tdSql.query("select count(*) from tb") + tdSql.checkData(0, 0, 4) + + tdDnodes.stop(1) + tdDnodes.start(1) + + time.sleep(1) + tdSql.query("select count(*) from tb") + tdSql.checkData(0, 0, 4) + + + sql = "create table tb1(ts timestamp, " + for i in range(4094): + sql += "col%d binary(14), " % (i + 1) + sql += "col4095 binary(22))" + tdSql.execute(sql) + + for i in range(4): + sql = "insert into tb1 values(%d, " + for j in range(4094): + str = "'%s', " % self.get_random_string(14) + sql += str + sql += "'%s')" % self.get_random_string(22) + tdSql.execute(sql % (self.ts + i)) + + time.sleep(10) + tdSql.query("select count(*) from tb1") + tdSql.checkData(0, 0, 4) + + tdDnodes.stop(1) + tdDnodes.start(1) + + time.sleep(1) + tdSql.query("select count(*) from tb1") + tdSql.checkData(0, 0, 4) + + + + #os.system("rm -rf tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.py.sql") + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From b81ca3465adf82e51d0d624b26986f76a978a8f4 Mon Sep 17 00:00:00 2001 From: happyguoxy Date: Wed, 14 Jul 2021 10:22:10 +0800 Subject: [PATCH 32/67] [TD-5213]test --- .../TD-5213/insertSigcolumnsNum4096.json | 112 ++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100755 tests/pytest/tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.json diff --git a/tests/pytest/tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.json b/tests/pytest/tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.json new file mode 100755 index 0000000000..e2b15b83a7 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.json @@ -0,0 +1,112 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 10, + "thread_count_create_tbl": 10, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 10, + "num_of_records_per_req": 1, + "max_sql_len": 102400000, + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb_old", + "child_table_exists":"no", + "childtable_count": 10, + "childtable_prefix": "stb_old_", + "auto_create_table": "no", + "batch_create_tbl_num": 5, + "data_source": "sample", + "insert_mode": "taosc", + "insert_rows": 100, + "childtable_limit": 0, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.csv", + "tags_file": "", + "columns": [{"type": "INT","count":4000}, {"type": "BINARY", "len": 16, "count":1}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}] + },{ + "name": "stb_new", + "child_table_exists":"no", + "childtable_count": 10, + "childtable_prefix": "stb_new_", + "auto_create_table": "no", + "batch_create_tbl_num": 5, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 100, + "childtable_limit": 0, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./tools/taosdemoAllTest/sample.csv", + "tags_file": "", + "columns": [{"type": "DOUBLE","count":1020}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}] + },{ + "name": "stb_int", + "child_table_exists":"no", + "childtable_count": 10, + "childtable_prefix": "stb_int_", + "auto_create_table": "no", + "batch_create_tbl_num": 5, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 100, + "childtable_limit": 0, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./tools/taosdemoAllTest/sample.csv", + "tags_file": "", + "columns": [{"type": "int","count":1020}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}] + }] + }] +} From 5ac5cf00515cdc9801c154f86de11ddff569e5d3 Mon Sep 17 00:00:00 2001 From: happyguoxy Date: Wed, 14 Jul 2021 10:22:24 +0800 Subject: [PATCH 33/67] [TD-5213]test --- .../TD-5213/insertSigcolumnsNum4096.csv | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100755 tests/pytest/tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.csv diff --git a/tests/pytest/tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.csv b/tests/pytest/tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.csv new file mode 100755 index 0000000000..078c3b93df --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.csv @@ -0,0 +1,10 @@ +0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +4,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +5,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +7,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +8,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +9,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, \ No newline at end of file From 0a063062438e283c3c717de13060d5cb128d4be2 Mon Sep 17 00:00:00 2001 From: Yurun Date: Wed, 14 Jul 2021 15:50:16 +0800 Subject: [PATCH 34/67] Fix the problem of parsing http protocol when the request header lacks spaces --- src/plugins/http/inc/httpParser.h | 1 + src/plugins/http/src/httpParser.c | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/plugins/http/inc/httpParser.h b/src/plugins/http/inc/httpParser.h index c7cf7a9608..cabd1693a0 100644 --- a/src/plugins/http/inc/httpParser.h +++ b/src/plugins/http/inc/httpParser.h @@ -100,6 +100,7 @@ typedef enum HTTP_PARSER_STATE { HTTP_PARSER_CHUNK, HTTP_PARSER_END, HTTP_PARSER_ERROR, + HTTP_PARSER_OPTIONAL_SP, } HTTP_PARSER_STATE; typedef enum HTTP_AUTH_TYPE { diff --git a/src/plugins/http/src/httpParser.c b/src/plugins/http/src/httpParser.c index a68a9bd7ed..ba88a2b9cd 100644 --- a/src/plugins/http/src/httpParser.c +++ b/src/plugins/http/src/httpParser.c @@ -744,6 +744,15 @@ static int32_t httpParserOnSp(HttpParser *parser, HTTP_PARSER_STATE state, const return ok; } +static int32_t httpParserOnOptionalSp(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { + int32_t ok = 0; + if (c != ' ') { + *again = 1; + httpPopStack(parser); + } + return ok; +} + static int32_t httpParserOnStatusCode(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; int32_t ok = 0; @@ -867,7 +876,7 @@ static int32_t httpParserOnHeader(HttpParser *parser, HTTP_PARSER_STATE state, c } httpPushStack(parser, HTTP_PARSER_CRLF); httpPushStack(parser, HTTP_PARSER_HEADER_VAL); - httpPushStack(parser, HTTP_PARSER_SP); + httpPushStack(parser, HTTP_PARSER_OPTIONAL_SP); httpPushStack(parser, HTTP_PARSER_HEADER_KEY); break; } @@ -1061,6 +1070,10 @@ static int32_t httpParseChar(HttpParser *parser, const char c, int32_t *again) { ok = httpParserOnSp(parser, state, c, again); break; } + if (state == HTTP_PARSER_OPTIONAL_SP) { + ok = httpParserOnOptionalSp(parser, state, c, again); + break; + } if (state == HTTP_PARSER_STATUS_CODE) { ok = httpParserOnStatusCode(parser, state, c, again); break; From fbb3eb1ac28fae7316e081f56918bc198b312602 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 14 Jul 2021 16:46:17 +0800 Subject: [PATCH 35/67] add macro memRowSetTKey --- src/common/inc/tdataformat.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index add2a656f5..1a842f05b6 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -547,6 +547,14 @@ typedef void *SMemRow; #define memRowTKey(r) (isDataRow(r) ? dataRowTKey(memRowDataBody(r)) : kvRowTKey(memRowKvBody(r))) #define memRowKey(r) (isDataRow(r) ? dataRowKey(memRowDataBody(r)) : kvRowKey(memRowKvBody(r))) +#define memRowSetTKey(r, k) \ + do { \ + if (isDataRow(r)) { \ + dataRowTKey(memRowDataBody(r)) = (k); \ + } else { \ + kvRowTKey(memRowKvBody(r)) = (k); \ + } \ + } while (0) #define memRowSetType(r, t) (memRowType(r) = (t)) #define memRowSetLen(r, l) (isDataRow(r) ? memRowDataLen(r) = (l) : memRowKvLen(r) = (l)) From c8916eed3404d9248ade917845bb7943d9127dec Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 14 Jul 2021 23:17:03 +0800 Subject: [PATCH 36/67] bounded column searching optimization --- src/client/src/tscParseInsert.c | 52 +++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 83c2c6e970..bfebed6676 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -788,7 +788,7 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i TDRowTLenT dataRowLen = pHelper->allNullLen; TDRowTLenT kvRowLen = TD_MEM_ROW_KV_VER_SIZE; TDRowTLenT payloadValOffset = 0; - TDRowLenT colValOffset = 0; + TDRowLenT colValOffset = 0; ASSERT(dataRowLen > 0); payloadSetNCols(payload, spd->numOfBound); @@ -907,8 +907,6 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i payloadSetType(payload, SMEM_ROW_DATA); } - ASSERT(colValOffset <= TSDB_MAX_BYTES_PER_ROW); - *len = (int32_t)(payloadValOffset + colValOffset); payloadSetTLen(payload, *len); @@ -1538,9 +1536,11 @@ static int32_t validateDataSource(SInsertStatementParam *pInsertParam, int32_t t static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDataColInfo* pColInfo, SSchema* pSchema, char* str, char **end) { + int32_t nCols = pColInfo->numOfCols; + pColInfo->numOfBound = 0; - memset(pColInfo->boundedColumns, 0, sizeof(int32_t) * pColInfo->numOfCols); - for(int32_t i = 0; i < pColInfo->numOfCols; ++i) { + memset(pColInfo->boundedColumns, 0, sizeof(int32_t) * nCols); + for (int32_t i = 0; i < nCols; ++i) { pColInfo->cols[i].hasVal = false; } @@ -1556,7 +1556,7 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat } bool isOrdered = true; - int32_t lastColIdx = -1; + int32_t lastColIdx = -1; // last column found while (1) { index = 0; sToken = tStrGetToken(str, &index, false); @@ -1577,7 +1577,8 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat bool findColumnIndex = false; // todo speedup by using hash list - for (int32_t t = 0; t < pColInfo->numOfCols; ++t) { + int32_t nScanned = 0, t = lastColIdx + 1; + while (t < nCols) { if (strncmp(sToken.z, pSchema[t].name, sToken.n) == 0 && strlen(pSchema[t].name) == sToken.n) { if (pColInfo->cols[t].hasVal == true) { code = tscInvalidOperationMsg(pInsertParam->msg, "duplicated column name", sToken.z); @@ -1586,18 +1587,39 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat pColInfo->cols[t].hasVal = true; pColInfo->boundedColumns[pColInfo->numOfBound] = t; - pColInfo->numOfBound += 1; + ++pColInfo->numOfBound; findColumnIndex = true; - - if (isOrdered) { - if (lastColIdx > t) { - isOrdered = false; - } else { - lastColIdx = t; - } + if (isOrdered && (lastColIdx > t)) { + isOrdered = false; } + lastColIdx = t; break; } + ++t; + ++nScanned; + } + if (!findColumnIndex) { + t = 0; + int32_t nRemain = nCols - nScanned; + while (t < nRemain) { + if (strncmp(sToken.z, pSchema[t].name, sToken.n) == 0 && strlen(pSchema[t].name) == sToken.n) { + if (pColInfo->cols[t].hasVal == true) { + code = tscInvalidOperationMsg(pInsertParam->msg, "duplicated column name", sToken.z); + goto _clean; + } + + pColInfo->cols[t].hasVal = true; + pColInfo->boundedColumns[pColInfo->numOfBound] = t; + ++pColInfo->numOfBound; + findColumnIndex = true; + if (isOrdered && (lastColIdx > t)) { + isOrdered = false; + } + lastColIdx = t; + break; + } + ++t; + } } if (!findColumnIndex) { From c65e0c313de8700551bae84fd635e5809f91abad Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 15 Jul 2021 12:01:52 +0800 Subject: [PATCH 37/67] expand int16_t to int32_t for query buffer offset --- src/client/src/tscSubquery.c | 6 +++--- src/query/inc/qExtbuffer.h | 4 ++-- src/query/inc/qUtil.h | 3 ++- src/query/src/qExecutor.c | 4 ++-- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index f10646e3a3..4a299e39e9 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -2430,9 +2430,9 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { tOrderDescriptor *pDesc = NULL; pRes->qId = 0x1; // hack the qhandle check - - const uint32_t nBufferSize = (1u << 16u); // 64KB - + + const uint32_t nBufferSize = (1u << 18u); // 256KB + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); SSubqueryState *pState = &pSql->subState; diff --git a/src/query/inc/qExtbuffer.h b/src/query/inc/qExtbuffer.h index b851fbb3e0..cf0e8ce31a 100644 --- a/src/query/inc/qExtbuffer.h +++ b/src/query/inc/qExtbuffer.h @@ -77,13 +77,13 @@ typedef struct tFilePagesItem { typedef struct SSchemaEx { struct SSchema field; - int16_t offset; + int32_t offset; } SSchemaEx; typedef struct SColumnModel { int32_t capacity; int32_t numOfCols; - int16_t rowSize; + int32_t rowSize; SSchemaEx *pFields; } SColumnModel; diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index c8741030c0..cab4ce58e6 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -66,7 +66,8 @@ static FORCE_INLINE SResultRow *getResultRow(SResultRowInfo *pResultRowInfo, int return pResultRowInfo->pResult[slot]; } -static FORCE_INLINE char *getPosInResultPage(SQueryAttr *pQueryAttr, tFilePage* page, int32_t rowOffset, int16_t offset) { +static FORCE_INLINE char* getPosInResultPage(SQueryAttr* pQueryAttr, tFilePage* page, int32_t rowOffset, + int32_t offset) { assert(rowOffset >= 0 && pQueryAttr != NULL); int32_t numOfRows = (int32_t)GET_ROW_PARAM_FOR_MULTIOUTPUT(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index fa2ddb05b8..c2e56fbc1d 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -3439,7 +3439,7 @@ void setResultRowOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pRe // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group tFilePage* bufPage = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pageId); - int16_t offset = 0; + int32_t offset = 0; for (int32_t i = 0; i < numOfOutput; ++i) { pCtx[i].resultInfo = getResultCell(pResult, i, rowCellInfoOffset); @@ -3749,7 +3749,7 @@ static int32_t doCopyToSDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pRow->pageId); - int16_t offset = 0; + int32_t offset = 0; for (int32_t j = 0; j < pBlock->info.numOfCols; ++j) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, j); int32_t bytes = pColInfoData->info.bytes; From 0063cb84e2dcb567e4aa2b6d0efe6fd2fff1f126 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 15 Jul 2021 14:38:30 +0800 Subject: [PATCH 38/67] comment the taosdemo test temporarily --- tests/pytest/fulltest.sh | 2 +- .../pytest/tools/taosdemoAllTest/taosdemoTestInsertWithJson.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 1f45cab13a..7d31714896 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -351,7 +351,7 @@ python3 ./test.py -f alter/alter_debugFlag.py python3 ./test.py -f query/queryBetweenAnd.py python3 ./test.py -f tag_lite/alter_tag.py -python3 test.py -f tools/taosdemoAllTest/taosdemoTestInsertWithJson.py +# python3 test.py -f tools/taosdemoAllTest/taosdemoTestInsertWithJson.py python3 test.py -f tools/taosdemoAllTest/taosdemoTestQueryWithJson.py python3 test.py -f tools/taosdemoAllTest/TD-4985/query-limit-offset.py python3 ./test.py -f tag_lite/drop_auto_create.py diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoTestInsertWithJson.py b/tests/pytest/tools/taosdemoAllTest/taosdemoTestInsertWithJson.py index 128c63413b..01e46eaaa0 100644 --- a/tests/pytest/tools/taosdemoAllTest/taosdemoTestInsertWithJson.py +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoTestInsertWithJson.py @@ -198,7 +198,7 @@ class TDTestCase: tdSql.checkRows(1) tdSql.query("select count(*) from db.stb1") tdSql.checkRows(1) - # tdSql.error("select * from db.stb3") + tdSql.error("select * from db.stb3") tdSql.error("select * from db.stb2") tdSql.execute("drop database if exists db") os.system("%staosdemo -f tools/taosdemoAllTest/insertNumOfrecordPerReq0.json -y " % binPath) From 2f271fa684fa0eaf2de6294cb3c40efbb55af706 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 15 Jul 2021 21:09:30 +0800 Subject: [PATCH 39/67] code optimization and give the same hint when exceed max columns --- src/client/inc/tsclient.h | 4 +--- src/client/src/tscParseInsert.c | 9 ++------- src/client/src/tscPrepare.c | 10 ++-------- src/client/src/tscSQLParser.c | 6 +++++- src/inc/taosdef.h | 2 +- 5 files changed, 11 insertions(+), 20 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index c23ca2bc98..d60c6756d6 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -41,8 +41,6 @@ extern "C" { // forward declaration struct SSqlInfo; -#define KvRowNColsThresh 128 // default 128 - typedef void (*__async_cb_func_t)(void *param, TAOS_RES *tres, int32_t numOfRows); typedef struct SNewVgroupInfo { @@ -111,7 +109,7 @@ typedef struct SParsedDataColInfo { int8_t orderStatus; // bounded columns: } SParsedDataColInfo; -#define IS_DATA_COL_ORDERED(s) (((s)->orderStatus) == (int8_t)ORDER_STATUS_ORDERED) +#define IS_DATA_COL_ORDERED(s) ((s) == (int8_t)ORDER_STATUS_ORDERED) typedef struct { SSchema * pSchema; diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index bfebed6676..b6062b5ad2 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -406,7 +406,6 @@ static FORCE_INLINE TDRowLenT tsSetPayloadColValue(char *payloadStart, char *pay payloadColSetId(payload, columnId); payloadColSetType(payload, columnType); memcpy(POINTER_SHIFT(payloadStart,tOffset), value, valueLen); - // payloadSetTLen(payloadStart, payloadTLen(payloadStart) + valueLen); return valueLen; } @@ -867,7 +866,7 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i TDRowLenT kvRowColLen = 0; TDRowLenT colValAppended = 0; - if (!IS_DATA_COL_ORDERED(spd)) { + if (!IS_DATA_COL_ORDERED(spd->orderStatus)) { ASSERT(spd->colIdxInfo != NULL); if(!isPrimaryKey) { kvStart = POINTER_SHIFT(kvPrimaryKeyStart, spd->colIdxInfo[i].finalIdx * PAYLOAD_COL_HEAD_LEN); @@ -891,7 +890,7 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i payloadColSetOffset(kvPrimaryKeyStart, colValOffset); } else { payloadColSetOffset(kvStart, colValOffset); - if (spd->orderStatus == ORDER_STATUS_ORDERED) { + if (IS_DATA_COL_ORDERED(spd->orderStatus)) { kvStart += PAYLOAD_COL_HEAD_LEN; // move to next column } } @@ -910,10 +909,6 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i *len = (int32_t)(payloadValOffset + colValOffset); payloadSetTLen(payload, *len); - // TSKEY tsKey = payloadKey(payload); - // ASSERT((tsKey < 1627747200000000 && tsKey > 1498838400000000) || (tsKey < 1627747200000 && tsKey > 1498838400000) || - // (tsKey < 1627747200 && tsKey > 1498838400)); - return TSDB_CODE_SUCCESS; } diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 0eb1eaa12f..36c10f6cd5 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -370,7 +370,7 @@ static int refactorPayload(STableDataBlocks* pBlock, int32_t rowNum) { bool isPrimaryKey = (colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX); // the primary key locates in 1st column - if (!IS_DATA_COL_ORDERED(spd)) { + if (!IS_DATA_COL_ORDERED(spd->orderStatus)) { ASSERT(spd->colIdxInfo != NULL); if (!isPrimaryKey) { kvStart = POINTER_SHIFT(kvPrimaryKeyStart, spd->colIdxInfo[i].finalIdx * PAYLOAD_COL_HEAD_LEN); @@ -406,7 +406,7 @@ static int refactorPayload(STableDataBlocks* pBlock, int32_t rowNum) { kvRowLen += TYPE_BYTES[pSchema->type]; } - if (IS_DATA_COL_ORDERED(spd)) { + if (IS_DATA_COL_ORDERED(spd->orderStatus)) { kvStart += PAYLOAD_COL_HEAD_LEN; // move to next column } } @@ -425,12 +425,6 @@ static int refactorPayload(STableDataBlocks* pBlock, int32_t rowNum) { TDRowTLenT len = payloadValOffset + colValOffset; payloadSetTLen(destPayload, len); -#if 0 - TSKEY tsKey = payloadKey(destPayload); - ASSERT((tsKey < 1627747200000000 && tsKey > 1498838400000000) || (tsKey < 1627747200000 && tsKey > 1498838400000) || - (tsKey < 1627747200 && tsKey > 1498838400)); -#endif - // next loop srcPayload += pBlock->rowSize; destPayload += len; diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index d9ad830214..13fd9a8092 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -1245,12 +1245,16 @@ static bool validateTableColumnInfo(SArray* pFieldList, SSqlCmd* pCmd) { const char* msg4 = "invalid data type"; const char* msg5 = "invalid binary/nchar column length"; const char* msg6 = "invalid column name"; + const char* msg7 = "too many columns"; // number of fields no less than 2 size_t numOfCols = taosArrayGetSize(pFieldList); - if (numOfCols <= 1 || numOfCols > TSDB_MAX_COLUMNS) { + if (numOfCols <= 1 ) { invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); return false; + } else if (numOfCols > TSDB_MAX_COLUMNS) { + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); + return false; } // first column must be timestamp diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 7c6b3bbed9..f5be879878 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -198,7 +198,7 @@ do { \ /** * In some scenarios uint16_t (0~65535) is used to store the row len. * - Firstly, we use 65531(65535 - 4), as the SDataRow and SKVRow including 4 bits header. - * - Secondly, if all cols are VarType except primary key, we need 4 bits to store the offset, thus + * - Secondly, if all cols are VarDataT type except primary key, we need 4 bits to store the offset, thus * the final value is 65531-(4096-1)*4 = 49151. */ #define TSDB_MAX_BYTES_PER_ROW 49151 From c6c8772261f5ff477bbd2b406235261164480a3a Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Fri, 16 Jul 2021 09:50:59 +0800 Subject: [PATCH 40/67] code optimization --- src/client/src/tscParseInsert.c | 2 +- src/common/inc/tdataformat.h | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index b6062b5ad2..ee662fee14 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -1146,7 +1146,7 @@ int tscSortRemoveDataBlockDupRows(STableDataBlocks *dataBuf, SBlockKeyInfo *pBlk TDRowTLenT payloadTLen = 0; int n = 0; while (n < nRows) { - pBlkKeyTuple->skey = payloadKey(pBlockData); + pBlkKeyTuple->skey = payloadTSKey(pBlockData); pBlkKeyTuple->payloadAddr = pBlockData; payloadTLen = payloadTLen(pBlockData); #if 0 diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index 1a842f05b6..43639e5c5f 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -617,10 +617,11 @@ static FORCE_INLINE void *tdGetMemRowDataOfCol(void *row, int8_t type, int32_t o #define payloadColSetType(c, t) (payloadColType(c) = (t)) #define payloadColSetOffset(c, o) (payloadColOffset(c) = (o)) -#define payloadKeyOffset(r) (*(uint16_t *)POINTER_SHIFT(r, PAYLOAD_HEADER_LEN + PAYLOAD_ID_TYPE_LEN)) -#define payloadTKey(r) (*(TKEY *)POINTER_SHIFT(r, payloadValuesOffset(r) + payloadKeyOffset(r))) +#define payloadTSKey(r) (*(TSKEY *)POINTER_SHIFT(r, payloadValuesOffset(r))) +#define payloadTKey(r) (*(TKEY *)POINTER_SHIFT(r, payloadValuesOffset(r))) #define payloadKey(r) tdGetKey(payloadTKey(r)) + static FORCE_INLINE char *payloadNextCol(char *pCol) { return (char *)POINTER_SHIFT(pCol, PAYLOAD_COL_HEAD_LEN); } #ifdef __cplusplus From fb92d7b988678c977deb03b3d8bb77f4addc9b49 Mon Sep 17 00:00:00 2001 From: Yurun Date: Fri, 16 Jul 2021 16:31:45 +0800 Subject: [PATCH 41/67] remove the last comma --- src/plugins/http/inc/httpParser.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/http/inc/httpParser.h b/src/plugins/http/inc/httpParser.h index cabd1693a0..865d275874 100644 --- a/src/plugins/http/inc/httpParser.h +++ b/src/plugins/http/inc/httpParser.h @@ -100,7 +100,7 @@ typedef enum HTTP_PARSER_STATE { HTTP_PARSER_CHUNK, HTTP_PARSER_END, HTTP_PARSER_ERROR, - HTTP_PARSER_OPTIONAL_SP, + HTTP_PARSER_OPTIONAL_SP } HTTP_PARSER_STATE; typedef enum HTTP_AUTH_TYPE { From 236f50ea0249ecb8e58f5f6f51a4001bfcbbe44f Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 16 Jul 2021 18:52:08 +0800 Subject: [PATCH 42/67] [TD-5055] handle incorret result --- src/client/src/tscSQLParser.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 77c37239da..ecc0d5ad86 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -8262,7 +8262,9 @@ static int32_t doValidateSubquery(SSqlNode* pSqlNode, int32_t index, SSqlObj* pS // union all is not support currently SSqlNode* p = taosArrayGetP(subInfo->pSubquery, 0); - + if (taosArrayGetSize(subInfo->pSubquery) >= 2) { + return invalidOperationMsg(msgBuf, "not support union in subquery"); + } SQueryInfo* pSub = calloc(1, sizeof(SQueryInfo)); tscInitQueryInfo(pSub); From e6a8f2f7fc9e3307671e92316380ed439e61b0f1 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Fri, 16 Jul 2021 18:52:19 +0800 Subject: [PATCH 43/67] use getNullValue() in ttypes.c --- src/client/src/tscParseInsert.c | 28 ++++++++++---------- src/client/src/tscUtil.c | 4 +-- src/common/inc/tdataformat.h | 4 +-- src/common/src/tdataformat.c | 46 ++++----------------------------- 4 files changed, 22 insertions(+), 60 deletions(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index ee662fee14..c3bf3605b2 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -425,7 +425,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay case TSDB_DATA_TYPE_BOOL: { // bool if (isNullStr(pToken)) { *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - tdGetNullVal(TSDB_DATA_TYPE_BOOL), TYPE_BYTES[TSDB_DATA_TYPE_BOOL], tOffset); + getNullValue(TSDB_DATA_TYPE_BOOL), TYPE_BYTES[TSDB_DATA_TYPE_BOOL], tOffset); } else { if ((pToken->type == TK_BOOL || pToken->type == TK_STRING) && (pToken->n != 0)) { if (strncmp(pToken->z, "true", pToken->n) == 0) { @@ -459,7 +459,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay case TSDB_DATA_TYPE_TINYINT: if (isNullStr(pToken)) { *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - tdGetNullVal(TSDB_DATA_TYPE_TINYINT), TYPE_BYTES[TSDB_DATA_TYPE_TINYINT], tOffset); + getNullValue(TSDB_DATA_TYPE_TINYINT), TYPE_BYTES[TSDB_DATA_TYPE_TINYINT], tOffset); } else { ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); if (ret != TSDB_CODE_SUCCESS) { @@ -479,7 +479,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay case TSDB_DATA_TYPE_UTINYINT: if (isNullStr(pToken)) { *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - tdGetNullVal(TSDB_DATA_TYPE_UTINYINT), TYPE_BYTES[TSDB_DATA_TYPE_UTINYINT], tOffset); + getNullValue(TSDB_DATA_TYPE_UTINYINT), TYPE_BYTES[TSDB_DATA_TYPE_UTINYINT], tOffset); } else { ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); if (ret != TSDB_CODE_SUCCESS) { @@ -499,7 +499,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay case TSDB_DATA_TYPE_SMALLINT: if (isNullStr(pToken)) { *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - tdGetNullVal(TSDB_DATA_TYPE_SMALLINT), TYPE_BYTES[TSDB_DATA_TYPE_SMALLINT], tOffset); + getNullValue(TSDB_DATA_TYPE_SMALLINT), TYPE_BYTES[TSDB_DATA_TYPE_SMALLINT], tOffset); } else { ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); if (ret != TSDB_CODE_SUCCESS) { @@ -520,7 +520,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay if (isNullStr(pToken)) { *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - tdGetNullVal(TSDB_DATA_TYPE_USMALLINT), TYPE_BYTES[TSDB_DATA_TYPE_USMALLINT], tOffset); + getNullValue(TSDB_DATA_TYPE_USMALLINT), TYPE_BYTES[TSDB_DATA_TYPE_USMALLINT], tOffset); } else { ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); if (ret != TSDB_CODE_SUCCESS) { @@ -540,7 +540,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay case TSDB_DATA_TYPE_INT: if (isNullStr(pToken)) { *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - tdGetNullVal(TSDB_DATA_TYPE_INT), TYPE_BYTES[TSDB_DATA_TYPE_INT], tOffset); + getNullValue(TSDB_DATA_TYPE_INT), TYPE_BYTES[TSDB_DATA_TYPE_INT], tOffset); } else { ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); if (ret != TSDB_CODE_SUCCESS) { @@ -560,7 +560,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay case TSDB_DATA_TYPE_UINT: if (isNullStr(pToken)) { *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - tdGetNullVal(TSDB_DATA_TYPE_UINT), TYPE_BYTES[TSDB_DATA_TYPE_UINT], tOffset); + getNullValue(TSDB_DATA_TYPE_UINT), TYPE_BYTES[TSDB_DATA_TYPE_UINT], tOffset); } else { ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); if (ret != TSDB_CODE_SUCCESS) { @@ -580,7 +580,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay case TSDB_DATA_TYPE_BIGINT: if (isNullStr(pToken)) { *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - tdGetNullVal(TSDB_DATA_TYPE_BIGINT), TYPE_BYTES[TSDB_DATA_TYPE_BIGINT], tOffset); + getNullValue(TSDB_DATA_TYPE_BIGINT), TYPE_BYTES[TSDB_DATA_TYPE_BIGINT], tOffset); } else { ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); if (ret != TSDB_CODE_SUCCESS) { @@ -598,7 +598,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay case TSDB_DATA_TYPE_UBIGINT: if (isNullStr(pToken)) { *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - tdGetNullVal(TSDB_DATA_TYPE_UBIGINT), TYPE_BYTES[TSDB_DATA_TYPE_UBIGINT], tOffset); + getNullValue(TSDB_DATA_TYPE_UBIGINT), TYPE_BYTES[TSDB_DATA_TYPE_UBIGINT], tOffset); } else { ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); if (ret != TSDB_CODE_SUCCESS) { @@ -617,7 +617,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay case TSDB_DATA_TYPE_FLOAT: if (isNullStr(pToken)) { *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - tdGetNullVal(TSDB_DATA_TYPE_FLOAT), TYPE_BYTES[TSDB_DATA_TYPE_FLOAT], tOffset); + getNullValue(TSDB_DATA_TYPE_FLOAT), TYPE_BYTES[TSDB_DATA_TYPE_FLOAT], tOffset); } else { double dv; if (TK_ILLEGAL == tscToDouble(pToken, &dv, &endptr)) { @@ -639,7 +639,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay case TSDB_DATA_TYPE_DOUBLE: if (isNullStr(pToken)) { *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - tdGetNullVal(TSDB_DATA_TYPE_DOUBLE), TYPE_BYTES[TSDB_DATA_TYPE_DOUBLE], tOffset); + getNullValue(TSDB_DATA_TYPE_DOUBLE), TYPE_BYTES[TSDB_DATA_TYPE_DOUBLE], tOffset); } else { double dv; if (TK_ILLEGAL == tscToDouble(pToken, &dv, &endptr)) { @@ -661,7 +661,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay if (pToken->type == TK_NULL) { payloadColSetId(payload, pSchema->colId); payloadColSetType(payload, pSchema->type); - memcpy(POINTER_SHIFT(payloadStart, tOffset), tdGetNullVal(TSDB_DATA_TYPE_BINARY), VARSTR_HEADER_SIZE + CHAR_BYTES); + memcpy(POINTER_SHIFT(payloadStart, tOffset), getNullValue(TSDB_DATA_TYPE_BINARY), VARSTR_HEADER_SIZE + CHAR_BYTES); *sizeAppend = (TDRowLenT)(VARSTR_HEADER_SIZE + CHAR_BYTES); } else { // too long values will return invalid sql, not be truncated automatically if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { // todo refactor @@ -684,7 +684,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay if (pToken->type == TK_NULL) { payloadColSetId(payload, pSchema->colId); payloadColSetType(payload, pSchema->type); - memcpy(POINTER_SHIFT(payloadStart,tOffset), tdGetNullVal(TSDB_DATA_TYPE_NCHAR), VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE); + memcpy(POINTER_SHIFT(payloadStart,tOffset), getNullValue(TSDB_DATA_TYPE_NCHAR), VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE); *sizeAppend = (TDRowLenT)(VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE); } else { // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' @@ -716,7 +716,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP]); } else { *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - tdGetNullVal(TSDB_DATA_TYPE_TIMESTAMP), + getNullValue(TSDB_DATA_TYPE_TIMESTAMP), TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], tOffset); } } else { diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 6de9632f96..2e0d5c7d8a 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1734,14 +1734,14 @@ static SMemRow tdGenMemRowFromBuilder(SMemRowBuilder* pBuilder) { p = payloadNextCol(p); ++i; } else { - tdAppendColVal(trow, tdGetNullVal(pSchema[j].type), pSchema[j].type, toffset); + tdAppendColVal(trow, getNullValue(pSchema[j].type), pSchema[j].type, toffset); toffset += TYPE_BYTES[pSchema[j].type]; ++j; } } while (j < nCols) { - tdAppendColVal(trow, tdGetNullVal(pSchema[j].type), pSchema[j].type, toffset); + tdAppendColVal(trow, getNullValue(pSchema[j].type), pSchema[j].type, toffset); toffset += TYPE_BYTES[pSchema[j].type]; ++j; } diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index 43639e5c5f..cb7afa9680 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -274,12 +274,10 @@ void dataColSetOffset(SDataCol *pCol, int nEle); bool isNEleNull(SDataCol *pCol, int nEle); void dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints); -const void *tdGetNullVal(int8_t type); - // Get the data pointer from a column-wised data static FORCE_INLINE const void *tdGetColDataOfRow(SDataCol *pCol, int row) { if (isAllRowsNull(pCol)) { - return tdGetNullVal(pCol->type); + return getNullValue(pCol->type); } if (IS_VAR_DATA_TYPE(pCol->type)) { return POINTER_SHIFT(pCol->pData, pCol->dataOff[row]); diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index f26f7511c1..188109c6c6 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -453,7 +453,7 @@ static void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols SDataCol *pDataCol = &(pCols->cols[dcol]); if (rcol >= schemaNCols(pSchema)) { // dataColSetNullAt(pDataCol, pCols->numOfRows); - dataColAppendVal(pDataCol, tdGetNullVal(pDataCol->type), pCols->numOfRows, pCols->maxPoints); + dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints); dcol++; continue; } @@ -468,7 +468,7 @@ static void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols rcol++; } else { // dataColSetNullAt(pDataCol, pCols->numOfRows); - dataColAppendVal(pDataCol, tdGetNullVal(pDataCol->type), pCols->numOfRows, pCols->maxPoints); + dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints); dcol++; } } @@ -498,7 +498,7 @@ static void tdAppendKvRowToDataCol(SKVRow row, STSchema *pSchema, SDataCols *pCo SDataCol *pDataCol = &(pCols->cols[dcol]); if (rcol >= nRowCols || rcol >= schemaNCols(pSchema)) { // dataColSetNullAt(pDataCol, pCols->numOfRows); - dataColAppendVal(pDataCol, tdGetNullVal(pDataCol->type), pCols->numOfRows, pCols->maxPoints); + dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints); ++dcol; continue; } @@ -514,7 +514,7 @@ static void tdAppendKvRowToDataCol(SKVRow row, STSchema *pSchema, SDataCols *pCo ++rcol; } else { // dataColSetNullAt(pDataCol, pCols->numOfRows); - dataColAppendVal(pDataCol, tdGetNullVal(pDataCol->type), pCols->numOfRows, pCols->maxPoints); + dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints); ++dcol; } } @@ -799,40 +799,4 @@ SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder) { memcpy(kvRowValues(row), pBuilder->buf, pBuilder->size); return row; -} - -const void *tdGetNullVal(int8_t type) { - switch (type) { - case TSDB_DATA_TYPE_BOOL: - return &BoolNull; - case TSDB_DATA_TYPE_TINYINT: - return &TinyintNull; - case TSDB_DATA_TYPE_SMALLINT: - return &SmallintNull; - case TSDB_DATA_TYPE_INT: - return &IntNull; - case TSDB_DATA_TYPE_BIGINT: - return &BigintNull; - case TSDB_DATA_TYPE_FLOAT: - return &FloatNull; - case TSDB_DATA_TYPE_DOUBLE: - return &DoubleNull; - case TSDB_DATA_TYPE_BINARY: - return &BinaryNull; - case TSDB_DATA_TYPE_TIMESTAMP: - return &TimestampNull; - case TSDB_DATA_TYPE_NCHAR: - return &NcharNull; - case TSDB_DATA_TYPE_UTINYINT: - return &UTinyintNull; - case TSDB_DATA_TYPE_USMALLINT: - return &USmallintNull; - case TSDB_DATA_TYPE_UINT: - return &UIntNull; - case TSDB_DATA_TYPE_UBIGINT: - return &UBigintNull; - default: - ASSERT(0); - return NULL; - } -} +} \ No newline at end of file From 58f2621fc84e1c5985981ac3b9faf02798c03285 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Fri, 16 Jul 2021 19:17:35 +0800 Subject: [PATCH 44/67] code optimization --- src/client/inc/tsclient.h | 2 +- src/client/src/tscSQLParser.c | 6 +++--- src/client/src/tscServer.c | 6 +++--- src/client/src/tscUtil.c | 4 ++-- src/common/src/ttypes.c | 2 +- src/cq/src/cqMain.c | 2 +- src/inc/taosdef.h | 5 +++-- src/inc/ttype.h | 2 +- src/tsdb/src/tsdbMeta.c | 2 +- 9 files changed, 16 insertions(+), 15 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index d60c6756d6..6b6b8a23c6 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -184,6 +184,7 @@ typedef struct { uint32_t allocSize; char * payload; int32_t payloadLen; + void * pBuf; // table meta buffer SHashObj *pTableMetaMap; // local buffer to keep the queried table meta, before validating the AST SQueryInfo *pQueryInfo; @@ -273,7 +274,6 @@ typedef struct SSqlObj { void * pStream; void * pSubscription; char * sqlstr; - void * pBuf; // tableMeta buffer char parseRetry; char retry; char maxRetry; diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 13fd9a8092..18d17e9016 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -7697,8 +7697,8 @@ int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo) { char name[TSDB_TABLE_FNAME_LEN] = {0}; assert(maxSize < 80 * TSDB_MAX_COLUMNS); - if (!pSql->pBuf) { - if (NULL == (pSql->pBuf = tcalloc(1, 80 * TSDB_MAX_COLUMNS))) { + if (!pSql->cmd.pBuf) { + if (NULL == (pSql->cmd.pBuf = tcalloc(1, 80 * TSDB_MAX_COLUMNS))) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } } @@ -7718,7 +7718,7 @@ int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo) { if (pTableMeta->id.uid > 0) { if (pTableMeta->tableType == TSDB_CHILD_TABLE) { - code = tscCreateTableMetaFromSTableMeta(pTableMeta, name, pSql->pBuf); + code = tscCreateTableMetaFromSTableMeta(pTableMeta, name, pSql->cmd.pBuf); // create the child table meta from super table failed, try load it from mnode if (code != TSDB_CODE_SUCCESS) { diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 223c03b108..2f469a3828 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -2596,8 +2596,8 @@ int32_t tscGetTableMetaImpl(SSqlObj* pSql, STableMetaInfo *pTableMetaInfo, bool // TODO resize the tableMeta assert(size < 80 * TSDB_MAX_COLUMNS); - if (!pSql->pBuf) { - if (NULL == (pSql->pBuf = tcalloc(1, 80 * TSDB_MAX_COLUMNS))) { + if (!pSql->cmd.pBuf) { + if (NULL == (pSql->cmd.pBuf = tcalloc(1, 80 * TSDB_MAX_COLUMNS))) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } } @@ -2606,7 +2606,7 @@ int32_t tscGetTableMetaImpl(SSqlObj* pSql, STableMetaInfo *pTableMetaInfo, bool if (pMeta->id.uid > 0) { // in case of child table, here only get the if (pMeta->tableType == TSDB_CHILD_TABLE) { - int32_t code = tscCreateTableMetaFromSTableMeta(pTableMetaInfo->pTableMeta, name, pSql->pBuf); + int32_t code = tscCreateTableMetaFromSTableMeta(pTableMetaInfo->pTableMeta, name, pSql->cmd.pBuf); if (code != TSDB_CODE_SUCCESS) { return getTableMetaFromMnode(pSql, pTableMetaInfo, autocreate); } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 2e0d5c7d8a..bf997f3cbb 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1450,7 +1450,6 @@ void tscFreeSqlObj(SSqlObj* pSql) { pSql->signature = NULL; pSql->fp = NULL; tfree(pSql->sqlstr); - tfree(pSql->pBuf); tfree(pSql->pSubs); pSql->subState.numOfSub = 0; @@ -1461,8 +1460,9 @@ void tscFreeSqlObj(SSqlObj* pSql) { memset(pCmd->payload, 0, (size_t)pCmd->allocSize); tfree(pCmd->payload); + tfree(pCmd->pBuf); pCmd->allocSize = 0; - + tsem_destroy(&pSql->rspSem); memset(pSql, 0, sizeof(*pSql)); free(pSql); diff --git a/src/common/src/ttypes.c b/src/common/src/ttypes.c index 34dda32401..24789d01ad 100644 --- a/src/common/src/ttypes.c +++ b/src/common/src/ttypes.c @@ -515,7 +515,7 @@ static void *nullValues[] = { &nullTinyIntu, &nullSmallIntu, &nullIntu, &nullBigIntu, }; -void *getNullValue(int32_t type) { +const void *getNullValue(int32_t type) { assert(type >= TSDB_DATA_TYPE_BOOL && type <= TSDB_DATA_TYPE_UBIGINT); return nullValues[type - 1]; } diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c index cd762f3110..b45234901b 100644 --- a/src/cq/src/cqMain.c +++ b/src/cq/src/cqMain.c @@ -492,7 +492,7 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) { STColumn *c = pSchema->columns + i; void* val = row[i]; if (val == NULL) { - val = getNullValue(c->type); + val = (void *)getNullValue(c->type); } else if (c->type == TSDB_DATA_TYPE_BINARY) { val = ((char*)val) - sizeof(VarDataLenT); } else if (c->type == TSDB_DATA_TYPE_NCHAR) { diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index f5be879878..0caf204baf 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -329,8 +329,9 @@ do { \ #define TSDB_MAX_JOIN_TABLE_NUM 10 #define TSDB_MAX_UNION_CLAUSE 5 -#define TSDB_MAX_BINARY_LEN (16384-TSDB_KEYSIZE) // keep 16384 -#define TSDB_MAX_NCHAR_LEN (16384-TSDB_KEYSIZE) // keep 16384 +#define TSDB_MAX_FIELD_LEN 16384 +#define TSDB_MAX_BINARY_LEN (TSDB_MAX_FIELD_LEN-TSDB_KEYSIZE) // keep 16384 +#define TSDB_MAX_NCHAR_LEN (TSDB_MAX_FIELD_LEN-TSDB_KEYSIZE) // keep 16384 #define PRIMARYKEY_TIMESTAMP_COL_INDEX 0 #define TSDB_MAX_RPC_THREADS 5 diff --git a/src/inc/ttype.h b/src/inc/ttype.h index 2581589563..d0266d6a5c 100644 --- a/src/inc/ttype.h +++ b/src/inc/ttype.h @@ -178,7 +178,7 @@ bool isValidDataType(int32_t type); void setVardataNull(char* val, int32_t type); void setNull(char *val, int32_t type, int32_t bytes); void setNullN(char *val, int32_t type, int32_t bytes, int32_t numOfElems); -void *getNullValue(int32_t type); +const void *getNullValue(int32_t type); void assignVal(char *val, const char *src, int32_t len, int32_t type); void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf); diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 621be04e21..b599e38e3a 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -787,7 +787,7 @@ static char *getTagIndexKey(const void *pData) { void * res = tdGetKVRowValOfCol(pTable->tagVal, pCol->colId); if (res == NULL) { // treat the column as NULL if we cannot find it - res = getNullValue(pCol->type); + res = (void *)getNullValue(pCol->type); } return res; } From 58ba4d008144d2b63327918fa6e0cb3a35413386 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 16 Jul 2021 23:38:11 +0800 Subject: [PATCH 45/67] [TD-5055] handle incorret resul --- tests/pytest/query/nestquery_last_row.py | 75 ++++++++++++------------ 1 file changed, 38 insertions(+), 37 deletions(-) diff --git a/tests/pytest/query/nestquery_last_row.py b/tests/pytest/query/nestquery_last_row.py index a04cb173af..3c4ada5174 100644 --- a/tests/pytest/query/nestquery_last_row.py +++ b/tests/pytest/query/nestquery_last_row.py @@ -160,22 +160,23 @@ class TDTestCase: tdSql.checkData(0,9,9.000000000) tdSql.checkData(0,10,'2020-09-13 20:26:40.009') + # incorrect result, not support nest > 2 sql = '''select last_row(*) from ((select * from table_0) union all (select * from table_1) union all (select * from table_2));''' - tdSql.query(sql) - tdSql.checkRows(1) - tdSql.checkData(0,1,self.num-1) - tdSql.checkData(0,2,self.num-1) - tdSql.checkData(0,3,self.num-1) - tdSql.checkData(0,4,self.num-1) - tdSql.checkData(0,5,'False') - tdSql.checkData(0,6,'binary.9') - tdSql.checkData(0,7,'nchar.9') - tdSql.checkData(0,8,9.00000) - tdSql.checkData(0,9,9.000000000) - tdSql.checkData(0,10,'2020-09-13 20:26:40.009') + tdSql.error(sql) + #tdSql.checkRows(1) + #tdSql.checkData(0,1,self.num-1) + #tdSql.checkData(0,2,self.num-1) + #tdSql.checkData(0,3,self.num-1) + #tdSql.checkData(0,4,self.num-1) + #tdSql.checkData(0,5,'False') + #tdSql.checkData(0,6,'binary.9') + #tdSql.checkData(0,7,'nchar.9') + #tdSql.checkData(0,8,9.00000) + #tdSql.checkData(0,9,9.000000000) + #tdSql.checkData(0,10,'2020-09-13 20:26:40.009') # bug 5055 # sql = '''select last_row(*) from @@ -189,18 +190,18 @@ class TDTestCase: ((select last_row(*) from table_0) union all (select last_row(*) from table_1) union all (select last_row(*) from table_2));''' - tdSql.query(sql) - tdSql.checkRows(1) - tdSql.checkData(0,1,self.num-1) - tdSql.checkData(0,2,self.num-1) - tdSql.checkData(0,3,self.num-1) - tdSql.checkData(0,4,self.num-1) - tdSql.checkData(0,5,'False') - tdSql.checkData(0,6,'binary.9') - tdSql.checkData(0,7,'nchar.9') - tdSql.checkData(0,8,9.00000) - tdSql.checkData(0,9,9.000000000) - tdSql.checkData(0,10,'2020-09-13 20:26:40.009') + tdSql.error(sql) + #tdSql.checkRows(1) + #tdSql.checkData(0,1,self.num-1) + #tdSql.checkData(0,2,self.num-1) + #tdSql.checkData(0,3,self.num-1) + #tdSql.checkData(0,4,self.num-1) + #tdSql.checkData(0,5,'False') + #tdSql.checkData(0,6,'binary.9') + #tdSql.checkData(0,7,'nchar.9') + #tdSql.checkData(0,8,9.00000) + #tdSql.checkData(0,9,9.000000000) + #tdSql.checkData(0,10,'2020-09-13 20:26:40.009') # bug 5055 # sql = '''select last_row(*) from @@ -214,18 +215,18 @@ class TDTestCase: ((select * from table_0 limit 5 offset 5) union all (select * from table_1 limit 5 offset 5) union all (select * from regular_table_1 limit 5 offset 5));''' - tdSql.query(sql) - tdSql.checkRows(1) - tdSql.checkData(0,1,self.num-1) - tdSql.checkData(0,2,self.num-1) - tdSql.checkData(0,3,self.num-1) - tdSql.checkData(0,4,self.num-1) - tdSql.checkData(0,5,'False') - tdSql.checkData(0,6,'binary.9') - tdSql.checkData(0,7,'nchar.9') - tdSql.checkData(0,8,9.00000) - tdSql.checkData(0,9,9.000000000) - tdSql.checkData(0,10,'2020-09-13 20:26:40.009') + tdSql.error(sql) + #tdSql.checkRows(1) + #tdSql.checkData(0,1,self.num-1) + #tdSql.checkData(0,2,self.num-1) + #tdSql.checkData(0,3,self.num-1) + #tdSql.checkData(0,4,self.num-1) + #tdSql.checkData(0,5,'False') + #tdSql.checkData(0,6,'binary.9') + #tdSql.checkData(0,7,'nchar.9') + #tdSql.checkData(0,8,9.00000) + #tdSql.checkData(0,9,9.000000000) + #tdSql.checkData(0,10,'2020-09-13 20:26:40.009') sql = '''select last_row(*) from @@ -260,4 +261,4 @@ class TDTestCase: tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file +tdCases.addLinux(__file__, TDTestCase()) From 567ecdc40197bd076aa45d597f42d5a4aa38ad75 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sat, 17 Jul 2021 10:47:26 +0800 Subject: [PATCH 46/67] restore codes as error imported from previous commit --- src/client/inc/tsclient.h | 2 +- src/client/src/tscParseInsert.c | 26 +++++++++---------- src/client/src/tscSQLParser.c | 6 ++--- src/client/src/tscServer.c | 6 ++--- src/client/src/tscUtil.c | 6 ++--- src/common/inc/tdataformat.h | 4 ++- src/common/src/tdataformat.c | 44 ++++++++++++++++++++++++++++++--- src/common/src/ttypes.c | 2 +- src/cq/src/cqMain.c | 2 +- src/inc/ttype.h | 2 +- src/tsdb/src/tsdbMeta.c | 2 +- 11 files changed, 70 insertions(+), 32 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 30a6951431..37a6b4b051 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -185,7 +185,6 @@ typedef struct { uint32_t allocSize; char * payload; int32_t payloadLen; - void * pBuf; // table meta buffer SHashObj *pTableMetaMap; // local buffer to keep the queried table meta, before validating the AST SQueryInfo *pQueryInfo; @@ -275,6 +274,7 @@ typedef struct SSqlObj { void * pStream; void * pSubscription; char * sqlstr; + void * pBuf; // table meta buffer char parseRetry; char retry; char maxRetry; diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index c3bf3605b2..18fb7ab1be 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -425,7 +425,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay case TSDB_DATA_TYPE_BOOL: { // bool if (isNullStr(pToken)) { *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - getNullValue(TSDB_DATA_TYPE_BOOL), TYPE_BYTES[TSDB_DATA_TYPE_BOOL], tOffset); + tdGetNullVal(TSDB_DATA_TYPE_BOOL), TYPE_BYTES[TSDB_DATA_TYPE_BOOL], tOffset); } else { if ((pToken->type == TK_BOOL || pToken->type == TK_STRING) && (pToken->n != 0)) { if (strncmp(pToken->z, "true", pToken->n) == 0) { @@ -459,7 +459,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay case TSDB_DATA_TYPE_TINYINT: if (isNullStr(pToken)) { *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - getNullValue(TSDB_DATA_TYPE_TINYINT), TYPE_BYTES[TSDB_DATA_TYPE_TINYINT], tOffset); + tdGetNullVal(TSDB_DATA_TYPE_TINYINT), TYPE_BYTES[TSDB_DATA_TYPE_TINYINT], tOffset); } else { ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); if (ret != TSDB_CODE_SUCCESS) { @@ -479,7 +479,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay case TSDB_DATA_TYPE_UTINYINT: if (isNullStr(pToken)) { *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - getNullValue(TSDB_DATA_TYPE_UTINYINT), TYPE_BYTES[TSDB_DATA_TYPE_UTINYINT], tOffset); + tdGetNullVal(TSDB_DATA_TYPE_UTINYINT), TYPE_BYTES[TSDB_DATA_TYPE_UTINYINT], tOffset); } else { ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); if (ret != TSDB_CODE_SUCCESS) { @@ -499,7 +499,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay case TSDB_DATA_TYPE_SMALLINT: if (isNullStr(pToken)) { *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - getNullValue(TSDB_DATA_TYPE_SMALLINT), TYPE_BYTES[TSDB_DATA_TYPE_SMALLINT], tOffset); + tdGetNullVal(TSDB_DATA_TYPE_SMALLINT), TYPE_BYTES[TSDB_DATA_TYPE_SMALLINT], tOffset); } else { ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); if (ret != TSDB_CODE_SUCCESS) { @@ -520,7 +520,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay if (isNullStr(pToken)) { *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - getNullValue(TSDB_DATA_TYPE_USMALLINT), TYPE_BYTES[TSDB_DATA_TYPE_USMALLINT], tOffset); + tdGetNullVal(TSDB_DATA_TYPE_USMALLINT), TYPE_BYTES[TSDB_DATA_TYPE_USMALLINT], tOffset); } else { ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); if (ret != TSDB_CODE_SUCCESS) { @@ -540,7 +540,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay case TSDB_DATA_TYPE_INT: if (isNullStr(pToken)) { *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - getNullValue(TSDB_DATA_TYPE_INT), TYPE_BYTES[TSDB_DATA_TYPE_INT], tOffset); + tdGetNullVal(TSDB_DATA_TYPE_INT), TYPE_BYTES[TSDB_DATA_TYPE_INT], tOffset); } else { ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); if (ret != TSDB_CODE_SUCCESS) { @@ -560,7 +560,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay case TSDB_DATA_TYPE_UINT: if (isNullStr(pToken)) { *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - getNullValue(TSDB_DATA_TYPE_UINT), TYPE_BYTES[TSDB_DATA_TYPE_UINT], tOffset); + tdGetNullVal(TSDB_DATA_TYPE_UINT), TYPE_BYTES[TSDB_DATA_TYPE_UINT], tOffset); } else { ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); if (ret != TSDB_CODE_SUCCESS) { @@ -580,7 +580,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay case TSDB_DATA_TYPE_BIGINT: if (isNullStr(pToken)) { *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - getNullValue(TSDB_DATA_TYPE_BIGINT), TYPE_BYTES[TSDB_DATA_TYPE_BIGINT], tOffset); + tdGetNullVal(TSDB_DATA_TYPE_BIGINT), TYPE_BYTES[TSDB_DATA_TYPE_BIGINT], tOffset); } else { ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); if (ret != TSDB_CODE_SUCCESS) { @@ -598,7 +598,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay case TSDB_DATA_TYPE_UBIGINT: if (isNullStr(pToken)) { *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - getNullValue(TSDB_DATA_TYPE_UBIGINT), TYPE_BYTES[TSDB_DATA_TYPE_UBIGINT], tOffset); + tdGetNullVal(TSDB_DATA_TYPE_UBIGINT), TYPE_BYTES[TSDB_DATA_TYPE_UBIGINT], tOffset); } else { ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); if (ret != TSDB_CODE_SUCCESS) { @@ -639,7 +639,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay case TSDB_DATA_TYPE_DOUBLE: if (isNullStr(pToken)) { *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - getNullValue(TSDB_DATA_TYPE_DOUBLE), TYPE_BYTES[TSDB_DATA_TYPE_DOUBLE], tOffset); + tdGetNullVal(TSDB_DATA_TYPE_DOUBLE), TYPE_BYTES[TSDB_DATA_TYPE_DOUBLE], tOffset); } else { double dv; if (TK_ILLEGAL == tscToDouble(pToken, &dv, &endptr)) { @@ -661,7 +661,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay if (pToken->type == TK_NULL) { payloadColSetId(payload, pSchema->colId); payloadColSetType(payload, pSchema->type); - memcpy(POINTER_SHIFT(payloadStart, tOffset), getNullValue(TSDB_DATA_TYPE_BINARY), VARSTR_HEADER_SIZE + CHAR_BYTES); + memcpy(POINTER_SHIFT(payloadStart, tOffset), tdGetNullVal(TSDB_DATA_TYPE_BINARY), VARSTR_HEADER_SIZE + CHAR_BYTES); *sizeAppend = (TDRowLenT)(VARSTR_HEADER_SIZE + CHAR_BYTES); } else { // too long values will return invalid sql, not be truncated automatically if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { // todo refactor @@ -684,7 +684,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay if (pToken->type == TK_NULL) { payloadColSetId(payload, pSchema->colId); payloadColSetType(payload, pSchema->type); - memcpy(POINTER_SHIFT(payloadStart,tOffset), getNullValue(TSDB_DATA_TYPE_NCHAR), VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE); + memcpy(POINTER_SHIFT(payloadStart,tOffset), tdGetNullVal(TSDB_DATA_TYPE_NCHAR), VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE); *sizeAppend = (TDRowLenT)(VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE); } else { // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' @@ -716,7 +716,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP]); } else { *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - getNullValue(TSDB_DATA_TYPE_TIMESTAMP), + tdGetNullVal(TSDB_DATA_TYPE_TIMESTAMP), TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], tOffset); } } else { diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 724f5d39a2..6fb838876f 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -8053,8 +8053,8 @@ int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo) { char name[TSDB_TABLE_FNAME_LEN] = {0}; assert(maxSize < 80 * TSDB_MAX_COLUMNS); - if (!pSql->cmd.pBuf) { - if (NULL == (pSql->cmd.pBuf = tcalloc(1, 80 * TSDB_MAX_COLUMNS))) { + if (!pSql->pBuf) { + if (NULL == (pSql->pBuf = tcalloc(1, 80 * TSDB_MAX_COLUMNS))) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } } @@ -8074,7 +8074,7 @@ int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo) { if (pTableMeta->id.uid > 0) { if (pTableMeta->tableType == TSDB_CHILD_TABLE) { - code = tscCreateTableMetaFromSTableMeta(pTableMeta, name, pSql->cmd.pBuf); + code = tscCreateTableMetaFromSTableMeta(pTableMeta, name, pSql->pBuf); // create the child table meta from super table failed, try load it from mnode if (code != TSDB_CODE_SUCCESS) { diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 00b73a297c..3f9eae37aa 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -2795,8 +2795,8 @@ int32_t tscGetTableMetaImpl(SSqlObj* pSql, STableMetaInfo *pTableMetaInfo, bool // TODO resize the tableMeta assert(size < 80 * TSDB_MAX_COLUMNS); - if (!pSql->cmd.pBuf) { - if (NULL == (pSql->cmd.pBuf = tcalloc(1, 80 * TSDB_MAX_COLUMNS))) { + if (!pSql->pBuf) { + if (NULL == (pSql->pBuf = tcalloc(1, 80 * TSDB_MAX_COLUMNS))) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } } @@ -2805,7 +2805,7 @@ int32_t tscGetTableMetaImpl(SSqlObj* pSql, STableMetaInfo *pTableMetaInfo, bool if (pMeta->id.uid > 0) { // in case of child table, here only get the if (pMeta->tableType == TSDB_CHILD_TABLE) { - int32_t code = tscCreateTableMetaFromSTableMeta(pTableMetaInfo->pTableMeta, name, pSql->cmd.pBuf); + int32_t code = tscCreateTableMetaFromSTableMeta(pTableMetaInfo->pTableMeta, name, pSql->pBuf); if (code != TSDB_CODE_SUCCESS) { return getTableMetaFromMnode(pSql, pTableMetaInfo, autocreate); } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 8fdc0d6005..59c6583661 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1488,6 +1488,7 @@ void tscFreeSqlObj(SSqlObj* pSql) { pSql->signature = NULL; pSql->fp = NULL; tfree(pSql->sqlstr); + tfree(pSql->pBuf); tfree(pSql->pSubs); pSql->subState.numOfSub = 0; @@ -1498,7 +1499,6 @@ void tscFreeSqlObj(SSqlObj* pSql) { memset(pCmd->payload, 0, (size_t)pCmd->allocSize); tfree(pCmd->payload); - tfree(pCmd->pBuf); pCmd->allocSize = 0; tsem_destroy(&pSql->rspSem); @@ -1813,14 +1813,14 @@ static SMemRow tdGenMemRowFromBuilder(SMemRowBuilder* pBuilder) { p = payloadNextCol(p); ++i; } else { - tdAppendColVal(trow, getNullValue(pSchema[j].type), pSchema[j].type, toffset); + tdAppendColVal(trow, tdGetNullVal(pSchema[j].type), pSchema[j].type, toffset); toffset += TYPE_BYTES[pSchema[j].type]; ++j; } } while (j < nCols) { - tdAppendColVal(trow, getNullValue(pSchema[j].type), pSchema[j].type, toffset); + tdAppendColVal(trow, tdGetNullVal(pSchema[j].type), pSchema[j].type, toffset); toffset += TYPE_BYTES[pSchema[j].type]; ++j; } diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index fde5f59e10..22f721d9e0 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -51,6 +51,8 @@ extern const uint64_t DoubleNull; extern const SBinaryNullT BinaryNull; extern const SNCharNullT NcharNull; +const void *tdGetNullVal(int8_t type); + #define STR_TO_VARSTR(x, str) \ do { \ VarDataLenT __len = (VarDataLenT)strlen(str); \ @@ -285,7 +287,7 @@ void dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints); // Get the data pointer from a column-wised data static FORCE_INLINE const void *tdGetColDataOfRow(SDataCol *pCol, int row) { if (isAllRowsNull(pCol)) { - return getNullValue(pCol->type); + return tdGetNullVal(pCol->type); } if (IS_VAR_DATA_TYPE(pCol->type)) { return POINTER_SHIFT(pCol->pData, pCol->dataOff[row]); diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index 188109c6c6..04d70e7aa0 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -453,7 +453,7 @@ static void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols SDataCol *pDataCol = &(pCols->cols[dcol]); if (rcol >= schemaNCols(pSchema)) { // dataColSetNullAt(pDataCol, pCols->numOfRows); - dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints); + dataColAppendVal(pDataCol, tdGetNullVal(pDataCol->type), pCols->numOfRows, pCols->maxPoints); dcol++; continue; } @@ -468,7 +468,7 @@ static void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols rcol++; } else { // dataColSetNullAt(pDataCol, pCols->numOfRows); - dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints); + dataColAppendVal(pDataCol, tdGetNullVal(pDataCol->type), pCols->numOfRows, pCols->maxPoints); dcol++; } } @@ -498,7 +498,7 @@ static void tdAppendKvRowToDataCol(SKVRow row, STSchema *pSchema, SDataCols *pCo SDataCol *pDataCol = &(pCols->cols[dcol]); if (rcol >= nRowCols || rcol >= schemaNCols(pSchema)) { // dataColSetNullAt(pDataCol, pCols->numOfRows); - dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints); + dataColAppendVal(pDataCol, tdGetNullVal(pDataCol->type), pCols->numOfRows, pCols->maxPoints); ++dcol; continue; } @@ -514,7 +514,7 @@ static void tdAppendKvRowToDataCol(SKVRow row, STSchema *pSchema, SDataCols *pCo ++rcol; } else { // dataColSetNullAt(pDataCol, pCols->numOfRows); - dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints); + dataColAppendVal(pDataCol, tdGetNullVal(pDataCol->type), pCols->numOfRows, pCols->maxPoints); ++dcol; } } @@ -799,4 +799,40 @@ SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder) { memcpy(kvRowValues(row), pBuilder->buf, pBuilder->size); return row; +} + +const void *tdGetNullVal(int8_t type) { + switch (type) { + case TSDB_DATA_TYPE_BOOL: + return &BoolNull; + case TSDB_DATA_TYPE_TINYINT: + return &TinyintNull; + case TSDB_DATA_TYPE_SMALLINT: + return &SmallintNull; + case TSDB_DATA_TYPE_INT: + return &IntNull; + case TSDB_DATA_TYPE_BIGINT: + return &BigintNull; + case TSDB_DATA_TYPE_FLOAT: + return &FloatNull; + case TSDB_DATA_TYPE_DOUBLE: + return &DoubleNull; + case TSDB_DATA_TYPE_BINARY: + return &BinaryNull; + case TSDB_DATA_TYPE_TIMESTAMP: + return &TimestampNull; + case TSDB_DATA_TYPE_NCHAR: + return &NcharNull; + case TSDB_DATA_TYPE_UTINYINT: + return &UTinyintNull; + case TSDB_DATA_TYPE_USMALLINT: + return &USmallintNull; + case TSDB_DATA_TYPE_UINT: + return &UIntNull; + case TSDB_DATA_TYPE_UBIGINT: + return &UBigintNull; + default: + ASSERT(0); + return NULL; + } } \ No newline at end of file diff --git a/src/common/src/ttypes.c b/src/common/src/ttypes.c index 24789d01ad..34dda32401 100644 --- a/src/common/src/ttypes.c +++ b/src/common/src/ttypes.c @@ -515,7 +515,7 @@ static void *nullValues[] = { &nullTinyIntu, &nullSmallIntu, &nullIntu, &nullBigIntu, }; -const void *getNullValue(int32_t type) { +void *getNullValue(int32_t type) { assert(type >= TSDB_DATA_TYPE_BOOL && type <= TSDB_DATA_TYPE_UBIGINT); return nullValues[type - 1]; } diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c index b45234901b..cd762f3110 100644 --- a/src/cq/src/cqMain.c +++ b/src/cq/src/cqMain.c @@ -492,7 +492,7 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) { STColumn *c = pSchema->columns + i; void* val = row[i]; if (val == NULL) { - val = (void *)getNullValue(c->type); + val = getNullValue(c->type); } else if (c->type == TSDB_DATA_TYPE_BINARY) { val = ((char*)val) - sizeof(VarDataLenT); } else if (c->type == TSDB_DATA_TYPE_NCHAR) { diff --git a/src/inc/ttype.h b/src/inc/ttype.h index 404e38ee3a..8def1cefe2 100644 --- a/src/inc/ttype.h +++ b/src/inc/ttype.h @@ -182,7 +182,7 @@ bool isValidDataType(int32_t type); void setVardataNull(char* val, int32_t type); void setNull(char *val, int32_t type, int32_t bytes); void setNullN(char *val, int32_t type, int32_t bytes, int32_t numOfElems); -const void *getNullValue(int32_t type); +void *getNullValue(int32_t type); void assignVal(char *val, const char *src, int32_t len, int32_t type); void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf); diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index b599e38e3a..621be04e21 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -787,7 +787,7 @@ static char *getTagIndexKey(const void *pData) { void * res = tdGetKVRowValOfCol(pTable->tagVal, pCol->colId); if (res == NULL) { // treat the column as NULL if we cannot find it - res = (void *)getNullValue(pCol->type); + res = getNullValue(pCol->type); } return res; } From 3756ae02d6ac3b0af30ccf300d190f8c664d13eb Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sat, 17 Jul 2021 18:25:21 +0800 Subject: [PATCH 47/67] : check max columns when add tags of stable --- src/client/src/tscSQLParser.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 3166b47ae8..1047838404 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -1536,13 +1536,20 @@ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { const char* msg4 = "invalid tag name"; const char* msg5 = "invalid binary/nchar tag length"; const char* msg6 = "invalid data type in tags"; + const char* msg7 = "too many columns"; STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; int32_t numOfTags = tscGetNumOfTags(pTableMeta); int32_t numOfCols = tscGetNumOfColumns(pTableMeta); - + + // no more max columns + if (numOfTags + numOfCols >= TSDB_MAX_COLUMNS) { + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); + return false; + } + // no more than 6 tags if (numOfTags == TSDB_MAX_TAGS) { char msg[128] = {0}; From 9a5402b01004af93c5d0dd996951e3231d43a804 Mon Sep 17 00:00:00 2001 From: lichuang Date: Mon, 19 Jul 2021 10:42:19 +0800 Subject: [PATCH 48/67] [TD-5324]:check max row bytes when alter tag/column width --- src/client/src/tscSQLParser.c | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 6d33aed823..fe93049eca 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -6006,6 +6006,16 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { return invalidOperationMsg(pMsg, msg22); } + SSchema* pSchema = (SSchema*) pTableMetaInfo->pTableMeta->schema; + int16_t numOfColumns = pTableMetaInfo->pTableMeta->tableInfo.numOfColumns; + int16_t i; + uint32_t nLen = 0; + for (i = 0; i < numOfColumns; ++i) { + nLen += pSchema[i].colId != columnIndex.columnIndex ? pSchema[i].bytes : pItem->bytes; + } + if (nLen >= TSDB_MAX_BYTES_PER_ROW) { + return invalidOperationMsg(pMsg, msg24); + } TAOS_FIELD f = tscCreateField(pColSchema->type, name.z, pItem->bytes); tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); }else if (pAlterSQL->type == TSDB_ALTER_TABLE_MODIFY_TAG_COLUMN) { @@ -6047,6 +6057,17 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { return invalidOperationMsg(pMsg, msg22); } + SSchema* pSchema = (SSchema*) pTableMetaInfo->pTableMeta->schema; + int16_t numOfColumns = pTableMetaInfo->pTableMeta->tableInfo.numOfColumns; + int16_t i; + uint32_t nLen = 0; + for (i = 0; i < numOfColumns; ++i) { + nLen += pSchema[i].colId != columnIndex.columnIndex ? pSchema[i].bytes : pItem->bytes; + } + if (nLen >= TSDB_MAX_BYTES_PER_ROW) { + return invalidOperationMsg(pMsg, msg24); + } + TAOS_FIELD f = tscCreateField(pColSchema->type, name.z, pItem->bytes); tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); } From 02eca3c0e44bfafa67707a126074289dd36b7dc1 Mon Sep 17 00:00:00 2001 From: lichuang Date: Mon, 19 Jul 2021 10:55:39 +0800 Subject: [PATCH 49/67] [TD-5324]:check max row bytes when alter tag/column width --- src/inc/taoserror.h | 1 + src/mnode/src/mnodeTable.c | 24 ++++++++++++++++++++++++ src/util/src/terror.c | 1 + 3 files changed, 26 insertions(+) diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index fbb29014e0..eefa0ed92a 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -174,6 +174,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_FIELD_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x036C) //"Field does not exist") #define TSDB_CODE_MND_INVALID_STABLE_NAME TAOS_DEF_ERROR_CODE(0, 0x036D) //"Super table does not exist") #define TSDB_CODE_MND_INVALID_CREATE_TABLE_MSG TAOS_DEF_ERROR_CODE(0, 0x036E) //"Invalid create table message") +#define TSDB_CODE_MND_EXCEED_MAX_ROW_BYTES TAOS_DEF_ERROR_CODE(0, 0x036F) //"Exceed max row bytes") #define TSDB_CODE_MND_INVALID_FUNC_NAME TAOS_DEF_ERROR_CODE(0, 0x0370) //"Invalid func name") #define TSDB_CODE_MND_INVALID_FUNC_LEN TAOS_DEF_ERROR_CODE(0, 0x0371) //"Invalid func length") diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 3631b1e005..aeff5c578e 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -1503,6 +1503,18 @@ static int32_t mnodeChangeSuperTableColumn(SMnodeMsg *pMsg) { return TSDB_CODE_MND_FIELD_NOT_EXIST; } + // check exceed max row bytes + int32_t i; + uint32_t nLen = 0; + for (i = 0; i < pStable->numOfColumns; ++i) { + nLen += (pStable->schema[i].colId == col) ? pAlter->schema[0].bytes : pStable->schema[i].bytes; + } + if (nLen > TSDB_MAX_BYTES_PER_ROW) { + mError("msg:%p, app:%p stable:%s, change column, name:%s exceed max row bytes", pMsg, pMsg->rpcMsg.ahandle, + pStable->info.tableId, name); + return TSDB_CODE_MND_EXCEED_MAX_ROW_BYTES; + } + // update SSchema *schema = (SSchema *) (pStable->schema + col); ASSERT(schema->type == TSDB_DATA_TYPE_BINARY || schema->type == TSDB_DATA_TYPE_NCHAR); @@ -1533,6 +1545,18 @@ static int32_t mnodeChangeSuperTableTag(SMnodeMsg *pMsg) { return TSDB_CODE_MND_FIELD_NOT_EXIST; } + // check exceed max row bytes + int32_t i; + uint32_t nLen = 0; + for (i = 0; i < pStable->numOfColumns; ++i) { + nLen += (pStable->schema[i].colId == col) ? pAlter->schema[0].bytes : pStable->schema[i].bytes; + } + if (nLen > TSDB_MAX_BYTES_PER_ROW) { + mError("msg:%p, app:%p stable:%s, change column, name:%s exceed max row bytes", pMsg, pMsg->rpcMsg.ahandle, + pStable->info.tableId, name); + return TSDB_CODE_MND_EXCEED_MAX_ROW_BYTES; + } + // update SSchema *schema = (SSchema *) (pStable->schema + col + pStable->numOfColumns); ASSERT(schema->type == TSDB_DATA_TYPE_BINARY || schema->type == TSDB_DATA_TYPE_NCHAR); diff --git a/src/util/src/terror.c b/src/util/src/terror.c index e36ab94844..46a33569b2 100644 --- a/src/util/src/terror.c +++ b/src/util/src/terror.c @@ -183,6 +183,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_FIELD_ALREAY_EXIST, "Field already exists" TAOS_DEFINE_ERROR(TSDB_CODE_MND_FIELD_NOT_EXIST, "Field does not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STABLE_NAME, "Super table does not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CREATE_TABLE_MSG, "Invalid create table message") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_EXCEED_MAX_ROW_BYTES, "Exceed max row bytes") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_NAME, "Invalid func name") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_LEN, "Invalid func length") From 1b90edcc55d91c47b23d4bd87f9ad19204d2465e Mon Sep 17 00:00:00 2001 From: lichuang Date: Mon, 19 Jul 2021 11:09:35 +0800 Subject: [PATCH 50/67] [TD-5324]:check max row bytes when alter tag/column width --- src/mnode/src/mnodeTable.c | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index aeff5c578e..6e5cf14b96 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -1545,18 +1545,6 @@ static int32_t mnodeChangeSuperTableTag(SMnodeMsg *pMsg) { return TSDB_CODE_MND_FIELD_NOT_EXIST; } - // check exceed max row bytes - int32_t i; - uint32_t nLen = 0; - for (i = 0; i < pStable->numOfColumns; ++i) { - nLen += (pStable->schema[i].colId == col) ? pAlter->schema[0].bytes : pStable->schema[i].bytes; - } - if (nLen > TSDB_MAX_BYTES_PER_ROW) { - mError("msg:%p, app:%p stable:%s, change column, name:%s exceed max row bytes", pMsg, pMsg->rpcMsg.ahandle, - pStable->info.tableId, name); - return TSDB_CODE_MND_EXCEED_MAX_ROW_BYTES; - } - // update SSchema *schema = (SSchema *) (pStable->schema + col + pStable->numOfColumns); ASSERT(schema->type == TSDB_DATA_TYPE_BINARY || schema->type == TSDB_DATA_TYPE_NCHAR); From 0953a60676e8902174e47a638c2a39029972f389 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 19 Jul 2021 12:03:18 +0800 Subject: [PATCH 51/67] Feature/sangshuduo/td 5136 taosdemo rework (#6911) * [TD-5136]: taosdemo simulate real senario. * update test case according to taosdemo change * adjust range of semi-random data. * make demo mode use different tag name and value. * change malloc to calloc for pid allocation. * fix typo. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index fb3a33101c..c91af8f337 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -2815,7 +2815,7 @@ static int createSuperTable( if (strcasecmp(dataType, "BINARY") == 0) { if ((g_args.demo_mode) && (tagIndex == 1)) { len += snprintf(tags + len, STRING_LEN - len, - "loction BINARY(%d), ", + "location BINARY(%d), ", superTbl->tags[tagIndex].dataLen); } else { len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", From ccb10efb736154f89ff21f9a046484b5882f60a6 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 19 Jul 2021 12:04:06 +0800 Subject: [PATCH 52/67] eliminate redundant codes --- src/client/src/tscParseInsert.c | 26 ++++++------ src/client/src/tscUtil.c | 4 +- src/common/inc/tdataformat.h | 31 +------------- src/common/src/tdataformat.c | 72 +++++++++------------------------ src/common/src/ttypes.c | 36 +++++++++-------- src/cq/src/cqMain.c | 4 +- src/inc/ttype.h | 14 ++++++- src/tsdb/src/tsdbMeta.c | 2 +- 8 files changed, 69 insertions(+), 120 deletions(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 18fb7ab1be..c3bf3605b2 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -425,7 +425,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay case TSDB_DATA_TYPE_BOOL: { // bool if (isNullStr(pToken)) { *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - tdGetNullVal(TSDB_DATA_TYPE_BOOL), TYPE_BYTES[TSDB_DATA_TYPE_BOOL], tOffset); + getNullValue(TSDB_DATA_TYPE_BOOL), TYPE_BYTES[TSDB_DATA_TYPE_BOOL], tOffset); } else { if ((pToken->type == TK_BOOL || pToken->type == TK_STRING) && (pToken->n != 0)) { if (strncmp(pToken->z, "true", pToken->n) == 0) { @@ -459,7 +459,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay case TSDB_DATA_TYPE_TINYINT: if (isNullStr(pToken)) { *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - tdGetNullVal(TSDB_DATA_TYPE_TINYINT), TYPE_BYTES[TSDB_DATA_TYPE_TINYINT], tOffset); + getNullValue(TSDB_DATA_TYPE_TINYINT), TYPE_BYTES[TSDB_DATA_TYPE_TINYINT], tOffset); } else { ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); if (ret != TSDB_CODE_SUCCESS) { @@ -479,7 +479,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay case TSDB_DATA_TYPE_UTINYINT: if (isNullStr(pToken)) { *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - tdGetNullVal(TSDB_DATA_TYPE_UTINYINT), TYPE_BYTES[TSDB_DATA_TYPE_UTINYINT], tOffset); + getNullValue(TSDB_DATA_TYPE_UTINYINT), TYPE_BYTES[TSDB_DATA_TYPE_UTINYINT], tOffset); } else { ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); if (ret != TSDB_CODE_SUCCESS) { @@ -499,7 +499,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay case TSDB_DATA_TYPE_SMALLINT: if (isNullStr(pToken)) { *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - tdGetNullVal(TSDB_DATA_TYPE_SMALLINT), TYPE_BYTES[TSDB_DATA_TYPE_SMALLINT], tOffset); + getNullValue(TSDB_DATA_TYPE_SMALLINT), TYPE_BYTES[TSDB_DATA_TYPE_SMALLINT], tOffset); } else { ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); if (ret != TSDB_CODE_SUCCESS) { @@ -520,7 +520,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay if (isNullStr(pToken)) { *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - tdGetNullVal(TSDB_DATA_TYPE_USMALLINT), TYPE_BYTES[TSDB_DATA_TYPE_USMALLINT], tOffset); + getNullValue(TSDB_DATA_TYPE_USMALLINT), TYPE_BYTES[TSDB_DATA_TYPE_USMALLINT], tOffset); } else { ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); if (ret != TSDB_CODE_SUCCESS) { @@ -540,7 +540,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay case TSDB_DATA_TYPE_INT: if (isNullStr(pToken)) { *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - tdGetNullVal(TSDB_DATA_TYPE_INT), TYPE_BYTES[TSDB_DATA_TYPE_INT], tOffset); + getNullValue(TSDB_DATA_TYPE_INT), TYPE_BYTES[TSDB_DATA_TYPE_INT], tOffset); } else { ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); if (ret != TSDB_CODE_SUCCESS) { @@ -560,7 +560,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay case TSDB_DATA_TYPE_UINT: if (isNullStr(pToken)) { *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - tdGetNullVal(TSDB_DATA_TYPE_UINT), TYPE_BYTES[TSDB_DATA_TYPE_UINT], tOffset); + getNullValue(TSDB_DATA_TYPE_UINT), TYPE_BYTES[TSDB_DATA_TYPE_UINT], tOffset); } else { ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); if (ret != TSDB_CODE_SUCCESS) { @@ -580,7 +580,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay case TSDB_DATA_TYPE_BIGINT: if (isNullStr(pToken)) { *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - tdGetNullVal(TSDB_DATA_TYPE_BIGINT), TYPE_BYTES[TSDB_DATA_TYPE_BIGINT], tOffset); + getNullValue(TSDB_DATA_TYPE_BIGINT), TYPE_BYTES[TSDB_DATA_TYPE_BIGINT], tOffset); } else { ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); if (ret != TSDB_CODE_SUCCESS) { @@ -598,7 +598,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay case TSDB_DATA_TYPE_UBIGINT: if (isNullStr(pToken)) { *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - tdGetNullVal(TSDB_DATA_TYPE_UBIGINT), TYPE_BYTES[TSDB_DATA_TYPE_UBIGINT], tOffset); + getNullValue(TSDB_DATA_TYPE_UBIGINT), TYPE_BYTES[TSDB_DATA_TYPE_UBIGINT], tOffset); } else { ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); if (ret != TSDB_CODE_SUCCESS) { @@ -639,7 +639,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay case TSDB_DATA_TYPE_DOUBLE: if (isNullStr(pToken)) { *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - tdGetNullVal(TSDB_DATA_TYPE_DOUBLE), TYPE_BYTES[TSDB_DATA_TYPE_DOUBLE], tOffset); + getNullValue(TSDB_DATA_TYPE_DOUBLE), TYPE_BYTES[TSDB_DATA_TYPE_DOUBLE], tOffset); } else { double dv; if (TK_ILLEGAL == tscToDouble(pToken, &dv, &endptr)) { @@ -661,7 +661,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay if (pToken->type == TK_NULL) { payloadColSetId(payload, pSchema->colId); payloadColSetType(payload, pSchema->type); - memcpy(POINTER_SHIFT(payloadStart, tOffset), tdGetNullVal(TSDB_DATA_TYPE_BINARY), VARSTR_HEADER_SIZE + CHAR_BYTES); + memcpy(POINTER_SHIFT(payloadStart, tOffset), getNullValue(TSDB_DATA_TYPE_BINARY), VARSTR_HEADER_SIZE + CHAR_BYTES); *sizeAppend = (TDRowLenT)(VARSTR_HEADER_SIZE + CHAR_BYTES); } else { // too long values will return invalid sql, not be truncated automatically if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { // todo refactor @@ -684,7 +684,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay if (pToken->type == TK_NULL) { payloadColSetId(payload, pSchema->colId); payloadColSetType(payload, pSchema->type); - memcpy(POINTER_SHIFT(payloadStart,tOffset), tdGetNullVal(TSDB_DATA_TYPE_NCHAR), VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE); + memcpy(POINTER_SHIFT(payloadStart,tOffset), getNullValue(TSDB_DATA_TYPE_NCHAR), VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE); *sizeAppend = (TDRowLenT)(VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE); } else { // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' @@ -716,7 +716,7 @@ static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *pay *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP]); } else { *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - tdGetNullVal(TSDB_DATA_TYPE_TIMESTAMP), + getNullValue(TSDB_DATA_TYPE_TIMESTAMP), TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], tOffset); } } else { diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 6cb8e9a0de..2c82f767bd 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1813,14 +1813,14 @@ static SMemRow tdGenMemRowFromBuilder(SMemRowBuilder* pBuilder) { p = payloadNextCol(p); ++i; } else { - tdAppendColVal(trow, tdGetNullVal(pSchema[j].type), pSchema[j].type, toffset); + tdAppendColVal(trow, getNullValue(pSchema[j].type), pSchema[j].type, toffset); toffset += TYPE_BYTES[pSchema[j].type]; ++j; } } while (j < nCols) { - tdAppendColVal(trow, tdGetNullVal(pSchema[j].type), pSchema[j].type, toffset); + tdAppendColVal(trow, getNullValue(pSchema[j].type), pSchema[j].type, toffset); toffset += TYPE_BYTES[pSchema[j].type]; ++j; } diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index 22f721d9e0..829e771c70 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -24,35 +24,6 @@ extern "C" { #endif -#pragma pack(push, 1) -typedef struct { - VarDataLenT len; - uint8_t data; -} SBinaryNullT; - -typedef struct { - VarDataLenT len; - uint32_t data; -} SNCharNullT; -#pragma pack(pop) - -extern const uint8_t BoolNull; -extern const uint8_t TinyintNull; -extern const uint16_t SmallintNull; -extern const uint32_t IntNull; -extern const uint64_t BigintNull; -extern const uint64_t TimestampNull; -extern const uint8_t UTinyintNull; -extern const uint16_t USmallintNull; -extern const uint32_t UIntNull; -extern const uint64_t UBigintNull; -extern const uint32_t FloatNull; -extern const uint64_t DoubleNull; -extern const SBinaryNullT BinaryNull; -extern const SNCharNullT NcharNull; - -const void *tdGetNullVal(int8_t type); - #define STR_TO_VARSTR(x, str) \ do { \ VarDataLenT __len = (VarDataLenT)strlen(str); \ @@ -287,7 +258,7 @@ void dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints); // Get the data pointer from a column-wised data static FORCE_INLINE const void *tdGetColDataOfRow(SDataCol *pCol, int row) { if (isAllRowsNull(pCol)) { - return tdGetNullVal(pCol->type); + return getNullValue(pCol->type); } if (IS_VAR_DATA_TYPE(pCol->type)) { return POINTER_SHIFT(pCol->pData, pCol->dataOff[row]); diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index 04d70e7aa0..df727c02db 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -18,20 +18,20 @@ #include "tcoding.h" #include "wchar.h" -const uint8_t BoolNull = TSDB_DATA_BOOL_NULL; -const uint8_t TinyintNull = TSDB_DATA_TINYINT_NULL; -const uint16_t SmallintNull = TSDB_DATA_SMALLINT_NULL; -const uint32_t IntNull = TSDB_DATA_INT_NULL; -const uint64_t BigintNull = TSDB_DATA_BIGINT_NULL; -const uint64_t TimestampNull = TSDB_DATA_BIGINT_NULL; -const uint8_t UTinyintNull = TSDB_DATA_UTINYINT_NULL; -const uint16_t USmallintNull = TSDB_DATA_USMALLINT_NULL; -const uint32_t UIntNull = TSDB_DATA_UINT_NULL; -const uint64_t UBigintNull = TSDB_DATA_UBIGINT_NULL; -const uint32_t FloatNull = TSDB_DATA_FLOAT_NULL; -const uint64_t DoubleNull = TSDB_DATA_DOUBLE_NULL; -const SBinaryNullT BinaryNull = {1, TSDB_DATA_BINARY_NULL}; -const SNCharNullT NcharNull = {4, TSDB_DATA_NCHAR_NULL}; +// const uint8_t BoolNull = TSDB_DATA_BOOL_NULL; +// const uint8_t TinyintNull = TSDB_DATA_TINYINT_NULL; +// const uint16_t SmallintNull = TSDB_DATA_SMALLINT_NULL; +// const uint32_t IntNull = TSDB_DATA_INT_NULL; +// const uint64_t BigintNull = TSDB_DATA_BIGINT_NULL; +// const uint64_t TimestampNull = TSDB_DATA_BIGINT_NULL; +// const uint8_t UTinyintNull = TSDB_DATA_UTINYINT_NULL; +// const uint16_t USmallintNull = TSDB_DATA_USMALLINT_NULL; +// const uint32_t UIntNull = TSDB_DATA_UINT_NULL; +// const uint64_t UBigintNull = TSDB_DATA_UBIGINT_NULL; +// const uint32_t FloatNull = TSDB_DATA_FLOAT_NULL; +// const uint64_t DoubleNull = TSDB_DATA_DOUBLE_NULL; +// const SBinaryNullT BinaryNull = {1, TSDB_DATA_BINARY_NULL}; +// const SNCharNullT NcharNull = {4, TSDB_DATA_NCHAR_NULL}; static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, int limit2, int tRows); @@ -453,7 +453,7 @@ static void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols SDataCol *pDataCol = &(pCols->cols[dcol]); if (rcol >= schemaNCols(pSchema)) { // dataColSetNullAt(pDataCol, pCols->numOfRows); - dataColAppendVal(pDataCol, tdGetNullVal(pDataCol->type), pCols->numOfRows, pCols->maxPoints); + dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints); dcol++; continue; } @@ -468,7 +468,7 @@ static void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols rcol++; } else { // dataColSetNullAt(pDataCol, pCols->numOfRows); - dataColAppendVal(pDataCol, tdGetNullVal(pDataCol->type), pCols->numOfRows, pCols->maxPoints); + dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints); dcol++; } } @@ -498,7 +498,7 @@ static void tdAppendKvRowToDataCol(SKVRow row, STSchema *pSchema, SDataCols *pCo SDataCol *pDataCol = &(pCols->cols[dcol]); if (rcol >= nRowCols || rcol >= schemaNCols(pSchema)) { // dataColSetNullAt(pDataCol, pCols->numOfRows); - dataColAppendVal(pDataCol, tdGetNullVal(pDataCol->type), pCols->numOfRows, pCols->maxPoints); + dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints); ++dcol; continue; } @@ -514,7 +514,7 @@ static void tdAppendKvRowToDataCol(SKVRow row, STSchema *pSchema, SDataCols *pCo ++rcol; } else { // dataColSetNullAt(pDataCol, pCols->numOfRows); - dataColAppendVal(pDataCol, tdGetNullVal(pDataCol->type), pCols->numOfRows, pCols->maxPoints); + dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints); ++dcol; } } @@ -799,40 +799,4 @@ SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder) { memcpy(kvRowValues(row), pBuilder->buf, pBuilder->size); return row; -} - -const void *tdGetNullVal(int8_t type) { - switch (type) { - case TSDB_DATA_TYPE_BOOL: - return &BoolNull; - case TSDB_DATA_TYPE_TINYINT: - return &TinyintNull; - case TSDB_DATA_TYPE_SMALLINT: - return &SmallintNull; - case TSDB_DATA_TYPE_INT: - return &IntNull; - case TSDB_DATA_TYPE_BIGINT: - return &BigintNull; - case TSDB_DATA_TYPE_FLOAT: - return &FloatNull; - case TSDB_DATA_TYPE_DOUBLE: - return &DoubleNull; - case TSDB_DATA_TYPE_BINARY: - return &BinaryNull; - case TSDB_DATA_TYPE_TIMESTAMP: - return &TimestampNull; - case TSDB_DATA_TYPE_NCHAR: - return &NcharNull; - case TSDB_DATA_TYPE_UTINYINT: - return &UTinyintNull; - case TSDB_DATA_TYPE_USMALLINT: - return &USmallintNull; - case TSDB_DATA_TYPE_UINT: - return &UIntNull; - case TSDB_DATA_TYPE_UBIGINT: - return &UBigintNull; - default: - ASSERT(0); - return NULL; - } } \ No newline at end of file diff --git a/src/common/src/ttypes.c b/src/common/src/ttypes.c index 34dda32401..6021a8f579 100644 --- a/src/common/src/ttypes.c +++ b/src/common/src/ttypes.c @@ -492,30 +492,32 @@ void setNullN(char *val, int32_t type, int32_t bytes, int32_t numOfElems) { } } -static uint8_t nullBool = TSDB_DATA_BOOL_NULL; -static uint8_t nullTinyInt = TSDB_DATA_TINYINT_NULL; -static uint16_t nullSmallInt = TSDB_DATA_SMALLINT_NULL; -static uint32_t nullInt = TSDB_DATA_INT_NULL; -static uint64_t nullBigInt = TSDB_DATA_BIGINT_NULL; -static uint32_t nullFloat = TSDB_DATA_FLOAT_NULL; -static uint64_t nullDouble = TSDB_DATA_DOUBLE_NULL; -static uint8_t nullTinyIntu = TSDB_DATA_UTINYINT_NULL; -static uint16_t nullSmallIntu = TSDB_DATA_USMALLINT_NULL; -static uint32_t nullIntu = TSDB_DATA_UINT_NULL; -static uint64_t nullBigIntu = TSDB_DATA_UBIGINT_NULL; +static uint8_t nullBool = TSDB_DATA_BOOL_NULL; +static uint8_t nullTinyInt = TSDB_DATA_TINYINT_NULL; +static uint16_t nullSmallInt = TSDB_DATA_SMALLINT_NULL; +static uint32_t nullInt = TSDB_DATA_INT_NULL; +static uint64_t nullBigInt = TSDB_DATA_BIGINT_NULL; +static uint32_t nullFloat = TSDB_DATA_FLOAT_NULL; +static uint64_t nullDouble = TSDB_DATA_DOUBLE_NULL; +static uint8_t nullTinyIntu = TSDB_DATA_UTINYINT_NULL; +static uint16_t nullSmallIntu = TSDB_DATA_USMALLINT_NULL; +static uint32_t nullIntu = TSDB_DATA_UINT_NULL; +static uint64_t nullBigIntu = TSDB_DATA_UBIGINT_NULL; +static SBinaryNullT nullBinary = {1, TSDB_DATA_BINARY_NULL}; +static SNCharNullT nullNchar = {4, TSDB_DATA_NCHAR_NULL}; -static union { - tstr str; - char pad[sizeof(tstr) + 4]; -} nullBinary = {.str = {.len = 1}}, nullNchar = {.str = {.len = 4}}; +// static union { +// tstr str; +// char pad[sizeof(tstr) + 4]; +// } nullBinary = {.str = {.len = 1}}, nullNchar = {.str = {.len = 4}}; -static void *nullValues[] = { +static const void *nullValues[] = { &nullBool, &nullTinyInt, &nullSmallInt, &nullInt, &nullBigInt, &nullFloat, &nullDouble, &nullBinary, &nullBigInt, &nullNchar, &nullTinyIntu, &nullSmallIntu, &nullIntu, &nullBigIntu, }; -void *getNullValue(int32_t type) { +const void *getNullValue(int32_t type) { assert(type >= TSDB_DATA_TYPE_BOOL && type <= TSDB_DATA_TYPE_UBIGINT); return nullValues[type - 1]; } diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c index cd762f3110..aac5a1c665 100644 --- a/src/cq/src/cqMain.c +++ b/src/cq/src/cqMain.c @@ -490,9 +490,9 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) { for (int32_t i = 0; i < pSchema->numOfCols; i++) { STColumn *c = pSchema->columns + i; - void* val = row[i]; + void *val = row[i]; if (val == NULL) { - val = getNullValue(c->type); + val = (void *)getNullValue(c->type); } else if (c->type == TSDB_DATA_TYPE_BINARY) { val = ((char*)val) - sizeof(VarDataLenT); } else if (c->type == TSDB_DATA_TYPE_NCHAR) { diff --git a/src/inc/ttype.h b/src/inc/ttype.h index 8def1cefe2..6e436bd23d 100644 --- a/src/inc/ttype.h +++ b/src/inc/ttype.h @@ -20,6 +20,18 @@ typedef struct tstr { char data[]; } tstr; +#pragma pack(push, 1) +typedef struct { + VarDataLenT len; + uint8_t data; +} SBinaryNullT; + +typedef struct { + VarDataLenT len; + uint32_t data; +} SNCharNullT; +#pragma pack(pop) + #define VARSTR_HEADER_SIZE sizeof(VarDataLenT) #define varDataLen(v) ((VarDataLenT *)(v))[0] @@ -182,7 +194,7 @@ bool isValidDataType(int32_t type); void setVardataNull(char* val, int32_t type); void setNull(char *val, int32_t type, int32_t bytes); void setNullN(char *val, int32_t type, int32_t bytes, int32_t numOfElems); -void *getNullValue(int32_t type); +const void *getNullValue(int32_t type); void assignVal(char *val, const char *src, int32_t len, int32_t type); void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf); diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 621be04e21..f233500ee9 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -787,7 +787,7 @@ static char *getTagIndexKey(const void *pData) { void * res = tdGetKVRowValOfCol(pTable->tagVal, pCol->colId); if (res == NULL) { // treat the column as NULL if we cannot find it - res = getNullValue(pCol->type); + res = (char*)getNullValue(pCol->type); } return res; } From 65d31bea78ba25b8ad7ed04920eaf4365db5998d Mon Sep 17 00:00:00 2001 From: bryanchang0603 Date: Mon, 19 Jul 2021 12:40:52 +0800 Subject: [PATCH 53/67] [TD-5143] adding test case about subquery --- tests/pytest/fulltest.sh | 2 +- .../query/nestedQuery/queryWithSpread.py | 46 +++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 tests/pytest/query/nestedQuery/queryWithSpread.py diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index ac22d3ee4a..c034afd836 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -241,7 +241,7 @@ python3 ./test.py -f query/queryStateWindow.py python3 ./test.py -f query/nestedQuery/queryWithOrderLimit.py python3 ./test.py -f query/nestquery_last_row.py python3 ./test.py -f query/queryCnameDisplay.py - +python3 test.py -f query/nestedQuery/queryWithSpread.py #stream python3 ./test.py -f stream/metric_1.py diff --git a/tests/pytest/query/nestedQuery/queryWithSpread.py b/tests/pytest/query/nestedQuery/queryWithSpread.py new file mode 100644 index 0000000000..4e09b1b826 --- /dev/null +++ b/tests/pytest/query/nestedQuery/queryWithSpread.py @@ -0,0 +1,46 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + + tdSql.execute('create table stb (ts timestamp, speed int) tags(t1 int);') + + for i in range(10): + tdSql.execute(f'create table tb_{i} using stb tags({i});') + tdSql.execute(f'insert into tb_{i} values(now, 0)') + tdSql.execute(f'insert into tb_{i} values(now+10s, {i})') + + tdSql.query('select max(col3) from (select spread(speed) as col3 from stb group by tbname);') + tdSql.checkData(0,0,'9.0') + tdSql.error('select max(col3) from (select spread(col3) as col3 from stb group by tbname);') + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From bd38430097b7795aa684209993591059fe8fefa6 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 19 Jul 2021 13:57:45 +0800 Subject: [PATCH 54/67] removed unused codes --- src/common/src/tdataformat.c | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index df727c02db..9f7432c90d 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -18,21 +18,6 @@ #include "tcoding.h" #include "wchar.h" -// const uint8_t BoolNull = TSDB_DATA_BOOL_NULL; -// const uint8_t TinyintNull = TSDB_DATA_TINYINT_NULL; -// const uint16_t SmallintNull = TSDB_DATA_SMALLINT_NULL; -// const uint32_t IntNull = TSDB_DATA_INT_NULL; -// const uint64_t BigintNull = TSDB_DATA_BIGINT_NULL; -// const uint64_t TimestampNull = TSDB_DATA_BIGINT_NULL; -// const uint8_t UTinyintNull = TSDB_DATA_UTINYINT_NULL; -// const uint16_t USmallintNull = TSDB_DATA_USMALLINT_NULL; -// const uint32_t UIntNull = TSDB_DATA_UINT_NULL; -// const uint64_t UBigintNull = TSDB_DATA_UBIGINT_NULL; -// const uint32_t FloatNull = TSDB_DATA_FLOAT_NULL; -// const uint64_t DoubleNull = TSDB_DATA_DOUBLE_NULL; -// const SBinaryNullT BinaryNull = {1, TSDB_DATA_BINARY_NULL}; -// const SNCharNullT NcharNull = {4, TSDB_DATA_NCHAR_NULL}; - static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, int limit2, int tRows); From b0bdd7c2c6cbae086dedfaee0d255f2f74121a94 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 19 Jul 2021 14:40:43 +0800 Subject: [PATCH 55/67] Hotfix/sangshuduo/td 5300 taosdemo stmt print (#6918) * [TD-5300]: taosdemo stmt debug print. * fix default iface is unknown. --- src/kit/taosdemo/taosdemo.c | 764 ++++++++++++++++++------------------ 1 file changed, 385 insertions(+), 379 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index c91af8f337..964f1a1d0f 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -5225,63 +5225,69 @@ static int32_t generateStbDataTail( int64_t remainderBufLen, int64_t insertRows, uint64_t recordFrom, int64_t startTime, int64_t *pSamplePos, int64_t *dataLen) { - uint64_t len = 0; + uint64_t len = 0; - char *pstr = buffer; + char *pstr = buffer; - bool tsRand; - if (0 == strncasecmp(superTblInfo->dataSource, "rand", strlen("rand"))) { - tsRand = true; - } else { - tsRand = false; - } - verbosePrint("%s() LN%d batch=%u buflen=%"PRId64"\n", - __func__, __LINE__, batch, remainderBufLen); - - int32_t k; - for (k = 0; k < batch;) { - char data[MAX_DATA_SIZE]; - memset(data, 0, MAX_DATA_SIZE); - - int64_t lenOfRow = 0; - - if (tsRand) { - lenOfRow = generateStbRowData(superTblInfo, data, - startTime + getTSRandTail( - superTblInfo->timeStampStep, k, - superTblInfo->disorderRatio, - superTblInfo->disorderRange) - ); + bool tsRand; + if (0 == strncasecmp(superTblInfo->dataSource, "rand", strlen("rand"))) { + tsRand = true; } else { - lenOfRow = getRowDataFromSample( - data, - (remainderBufLen < MAX_DATA_SIZE)?remainderBufLen:MAX_DATA_SIZE, - startTime + superTblInfo->timeStampStep * k, - superTblInfo, - pSamplePos); + tsRand = false; + } + verbosePrint("%s() LN%d batch=%u buflen=%"PRId64"\n", + __func__, __LINE__, batch, remainderBufLen); + + int32_t k; + for (k = 0; k < batch;) { + char data[MAX_DATA_SIZE]; + memset(data, 0, MAX_DATA_SIZE); + + int64_t lenOfRow = 0; + + if (tsRand) { + if (superTblInfo->disorderRatio > 0) { + lenOfRow = generateStbRowData(superTblInfo, data, + startTime + getTSRandTail( + superTblInfo->timeStampStep, k, + superTblInfo->disorderRatio, + superTblInfo->disorderRange) + ); + } else { + lenOfRow = generateStbRowData(superTblInfo, data, + startTime + superTblInfo->timeStampStep * k + ); + } + } else { + lenOfRow = getRowDataFromSample( + data, + (remainderBufLen < MAX_DATA_SIZE)?remainderBufLen:MAX_DATA_SIZE, + startTime + superTblInfo->timeStampStep * k, + superTblInfo, + pSamplePos); + } + + if ((lenOfRow + 1) > remainderBufLen) { + break; + } + + pstr += snprintf(pstr , lenOfRow + 1, "%s", data); + k++; + len += lenOfRow; + remainderBufLen -= lenOfRow; + + verbosePrint("%s() LN%d len=%"PRIu64" k=%u \nbuffer=%s\n", + __func__, __LINE__, len, k, buffer); + + recordFrom ++; + + if (recordFrom >= insertRows) { + break; + } } - if ((lenOfRow + 1) > remainderBufLen) { - break; - } - - pstr += snprintf(pstr , lenOfRow + 1, "%s", data); - k++; - len += lenOfRow; - remainderBufLen -= lenOfRow; - - verbosePrint("%s() LN%d len=%"PRIu64" k=%u \nbuffer=%s\n", - __func__, __LINE__, len, k, buffer); - - recordFrom ++; - - if (recordFrom >= insertRows) { - break; - } - } - - *dataLen = len; - return k; + *dataLen = len; + return k; } @@ -5378,52 +5384,52 @@ static int32_t generateStbInterlaceData( int64_t startTime, uint64_t *pRemainderBufLen) { - assert(buffer); - char *pstr = buffer; + assert(buffer); + char *pstr = buffer; - int headLen = generateStbSQLHead( - superTblInfo, - tableName, tableSeq, pThreadInfo->db_name, - pstr, *pRemainderBufLen); + int headLen = generateStbSQLHead( + superTblInfo, + tableName, tableSeq, pThreadInfo->db_name, + pstr, *pRemainderBufLen); - if (headLen <= 0) { - return 0; - } - // generate data buffer - verbosePrint("[%d] %s() LN%d i=%"PRIu64" buffer:\n%s\n", + if (headLen <= 0) { + return 0; + } + // generate data buffer + verbosePrint("[%d] %s() LN%d i=%"PRIu64" buffer:\n%s\n", pThreadInfo->threadID, __func__, __LINE__, i, buffer); - pstr += headLen; - *pRemainderBufLen -= headLen; + pstr += headLen; + *pRemainderBufLen -= headLen; - int64_t dataLen = 0; + int64_t dataLen = 0; - verbosePrint("[%d] %s() LN%d i=%"PRIu64" batchPerTblTimes=%u batchPerTbl = %u\n", + verbosePrint("[%d] %s() LN%d i=%"PRIu64" batchPerTblTimes=%u batchPerTbl = %u\n", pThreadInfo->threadID, __func__, __LINE__, i, batchPerTblTimes, batchPerTbl); - if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) { - startTime = taosGetTimestamp(pThreadInfo->time_precision); - } + if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) { + startTime = taosGetTimestamp(pThreadInfo->time_precision); + } - int32_t k = generateStbDataTail( + int32_t k = generateStbDataTail( superTblInfo, batchPerTbl, pstr, *pRemainderBufLen, insertRows, 0, startTime, &(pThreadInfo->samplePos), &dataLen); - if (k == batchPerTbl) { - pstr += dataLen; - *pRemainderBufLen -= dataLen; - } else { - debugPrint("%s() LN%d, generated data tail: %u, not equal batch per table: %u\n", - __func__, __LINE__, k, batchPerTbl); - pstr -= headLen; - pstr[0] = '\0'; - k = 0; - } + if (k == batchPerTbl) { + pstr += dataLen; + *pRemainderBufLen -= dataLen; + } else { + debugPrint("%s() LN%d, generated data tail: %u, not equal batch per table: %u\n", + __func__, __LINE__, k, batchPerTbl); + pstr -= headLen; + pstr[0] = '\0'; + k = 0; + } - return k; + return k; } static int64_t generateInterlaceDataWithoutStb( @@ -5871,7 +5877,7 @@ static int32_t prepareStbStmtInterlace( stbInfo, stmt, tableName, - g_args.num_of_RPR, + batch, insertRows, 0, startTime, pSamplePos); } @@ -5905,29 +5911,29 @@ static int32_t generateStbProgressiveData( uint64_t recordFrom, int64_t startTime, int64_t *pSamplePos, int64_t *pRemainderBufLen) { - assert(buffer != NULL); - char *pstr = buffer; + assert(buffer != NULL); + char *pstr = buffer; - memset(buffer, 0, *pRemainderBufLen); + memset(buffer, 0, *pRemainderBufLen); - int64_t headLen = generateStbSQLHead( - superTblInfo, - tableName, tableSeq, dbName, - buffer, *pRemainderBufLen); + int64_t headLen = generateStbSQLHead( + superTblInfo, + tableName, tableSeq, dbName, + buffer, *pRemainderBufLen); - if (headLen <= 0) { - return 0; - } - pstr += headLen; - *pRemainderBufLen -= headLen; + if (headLen <= 0) { + return 0; + } + pstr += headLen; + *pRemainderBufLen -= headLen; - int64_t dataLen; + int64_t dataLen; - return generateStbDataTail(superTblInfo, - g_args.num_of_RPR, pstr, *pRemainderBufLen, - insertRows, recordFrom, - startTime, - pSamplePos, &dataLen); + return generateStbDataTail(superTblInfo, + g_args.num_of_RPR, pstr, *pRemainderBufLen, + insertRows, recordFrom, + startTime, + pSamplePos, &dataLen); } static int32_t generateProgressiveDataWithoutStb( @@ -5974,283 +5980,283 @@ static void printStatPerThread(threadInfo *pThreadInfo) // sync write interlace data static void* syncWriteInterlace(threadInfo *pThreadInfo) { - debugPrint("[%d] %s() LN%d: ### interlace write\n", - pThreadInfo->threadID, __func__, __LINE__); - - int64_t insertRows; - uint32_t interlaceRows; - uint64_t maxSqlLen; - int64_t nTimeStampStep; - uint64_t insert_interval; - - SSuperTable* superTblInfo = pThreadInfo->superTblInfo; - - if (superTblInfo) { - insertRows = superTblInfo->insertRows; - - if ((superTblInfo->interlaceRows == 0) - && (g_args.interlace_rows > 0)) { - interlaceRows = g_args.interlace_rows; - } else { - interlaceRows = superTblInfo->interlaceRows; - } - maxSqlLen = superTblInfo->maxSqlLen; - nTimeStampStep = superTblInfo->timeStampStep; - insert_interval = superTblInfo->insertInterval; - } else { - insertRows = g_args.num_of_DPT; - interlaceRows = g_args.interlace_rows; - maxSqlLen = g_args.max_sql_len; - nTimeStampStep = DEFAULT_TIMESTAMP_STEP; - insert_interval = g_args.insert_interval; - } - - debugPrint("[%d] %s() LN%d: start_table_from=%"PRIu64" ntables=%"PRId64" insertRows=%"PRIu64"\n", - pThreadInfo->threadID, __func__, __LINE__, - pThreadInfo->start_table_from, - pThreadInfo->ntables, insertRows); - - if (interlaceRows > insertRows) - interlaceRows = insertRows; - - if (interlaceRows > g_args.num_of_RPR) - interlaceRows = g_args.num_of_RPR; - - uint32_t batchPerTbl = interlaceRows; - uint32_t batchPerTblTimes; - - if ((interlaceRows > 0) && (pThreadInfo->ntables > 1)) { - batchPerTblTimes = - g_args.num_of_RPR / interlaceRows; - } else { - batchPerTblTimes = 1; - } - - pThreadInfo->buffer = calloc(maxSqlLen, 1); - if (NULL == pThreadInfo->buffer) { - errorPrint( "%s() LN%d, Failed to alloc %"PRIu64" Bytes, reason:%s\n", - __func__, __LINE__, maxSqlLen, strerror(errno)); - return NULL; - } - - pThreadInfo->totalInsertRows = 0; - pThreadInfo->totalAffectedRows = 0; - - uint64_t st = 0; - uint64_t et = UINT64_MAX; - - uint64_t lastPrintTime = taosGetTimestampMs(); - uint64_t startTs = taosGetTimestampMs(); - uint64_t endTs; - - uint64_t tableSeq = pThreadInfo->start_table_from; - int64_t startTime = pThreadInfo->start_time; - - uint64_t generatedRecPerTbl = 0; - bool flagSleep = true; - uint64_t sleepTimeTotal = 0; - - while(pThreadInfo->totalInsertRows < pThreadInfo->ntables * insertRows) { - if ((flagSleep) && (insert_interval)) { - st = taosGetTimestampMs(); - flagSleep = false; - } - // generate data - memset(pThreadInfo->buffer, 0, maxSqlLen); - uint64_t remainderBufLen = maxSqlLen; - - char *pstr = pThreadInfo->buffer; - - int len = snprintf(pstr, - strlen(STR_INSERT_INTO) + 1, "%s", STR_INSERT_INTO); - pstr += len; - remainderBufLen -= len; - - uint32_t recOfBatch = 0; - - for (uint64_t i = 0; i < batchPerTblTimes; i ++) { - char tableName[TSDB_TABLE_NAME_LEN]; - - getTableName(tableName, pThreadInfo, tableSeq); - if (0 == strlen(tableName)) { - errorPrint("[%d] %s() LN%d, getTableName return null\n", + debugPrint("[%d] %s() LN%d: ### interlace write\n", pThreadInfo->threadID, __func__, __LINE__); - free(pThreadInfo->buffer); - return NULL; - } - uint64_t oldRemainderLen = remainderBufLen; + int64_t insertRows; + uint32_t interlaceRows; + uint64_t maxSqlLen; + int64_t nTimeStampStep; + uint64_t insert_interval; - int32_t generated; - if (superTblInfo) { - if (superTblInfo->iface == STMT_IFACE) { -#if STMT_IFACE_ENABLED == 1 - generated = prepareStbStmtInterlace( - superTblInfo, - pThreadInfo->stmt, - tableName, - batchPerTbl, - insertRows, i, - startTime, - &(pThreadInfo->samplePos)); -#else - generated = -1; -#endif - } else { - generated = generateStbInterlaceData( - superTblInfo, - tableName, batchPerTbl, i, - batchPerTblTimes, - tableSeq, - pThreadInfo, pstr, - insertRows, - startTime, - &remainderBufLen); - } - } else { - if (g_args.iface == STMT_IFACE) { - debugPrint("[%d] %s() LN%d, tableName:%s, batch:%d startTime:%"PRId64"\n", - pThreadInfo->threadID, - __func__, __LINE__, - tableName, batchPerTbl, startTime); -#if STMT_IFACE_ENABLED == 1 - generated = prepareStmtWithoutStb( - pThreadInfo->stmt, tableName, - batchPerTbl, - insertRows, i, - startTime); -#else - generated = -1; -#endif - } else { - generated = generateInterlaceDataWithoutStb( - tableName, batchPerTbl, - tableSeq, - pThreadInfo->db_name, pstr, - insertRows, - startTime, - &remainderBufLen); - } - } + SSuperTable* superTblInfo = pThreadInfo->superTblInfo; - debugPrint("[%d] %s() LN%d, generated records is %d\n", - pThreadInfo->threadID, __func__, __LINE__, generated); - if (generated < 0) { - errorPrint("[%d] %s() LN%d, generated records is %d\n", - pThreadInfo->threadID, __func__, __LINE__, generated); - goto free_of_interlace; - } else if (generated == 0) { - break; - } + if (superTblInfo) { + insertRows = superTblInfo->insertRows; - tableSeq ++; - recOfBatch += batchPerTbl; - - pstr += (oldRemainderLen - remainderBufLen); - pThreadInfo->totalInsertRows += batchPerTbl; - - verbosePrint("[%d] %s() LN%d batchPerTbl=%d recOfBatch=%d\n", - pThreadInfo->threadID, __func__, __LINE__, - batchPerTbl, recOfBatch); - - if (tableSeq == pThreadInfo->start_table_from + pThreadInfo->ntables) { - // turn to first table - tableSeq = pThreadInfo->start_table_from; - generatedRecPerTbl += batchPerTbl; - - startTime = pThreadInfo->start_time - + generatedRecPerTbl * nTimeStampStep; - - flagSleep = true; - if (generatedRecPerTbl >= insertRows) - break; - - int64_t remainRows = insertRows - generatedRecPerTbl; - if ((remainRows > 0) && (batchPerTbl > remainRows)) - batchPerTbl = remainRows; - - if (pThreadInfo->ntables * batchPerTbl < g_args.num_of_RPR) - break; - } - - verbosePrint("[%d] %s() LN%d generatedRecPerTbl=%"PRId64" insertRows=%"PRId64"\n", - pThreadInfo->threadID, __func__, __LINE__, - generatedRecPerTbl, insertRows); - - if ((g_args.num_of_RPR - recOfBatch) < batchPerTbl) - break; - } - - verbosePrint("[%d] %s() LN%d recOfBatch=%d totalInsertRows=%"PRIu64"\n", - pThreadInfo->threadID, __func__, __LINE__, recOfBatch, - pThreadInfo->totalInsertRows); - verbosePrint("[%d] %s() LN%d, buffer=%s\n", - pThreadInfo->threadID, __func__, __LINE__, pThreadInfo->buffer); - - startTs = taosGetTimestampMs(); - - if (recOfBatch == 0) { - errorPrint("[%d] %s() LN%d Failed to insert records of batch %d\n", - pThreadInfo->threadID, __func__, __LINE__, - batchPerTbl); - if (batchPerTbl > 0) { - errorPrint("\tIf the batch is %d, the length of the SQL to insert a row must be less then %"PRId64"\n", - batchPerTbl, maxSqlLen / batchPerTbl); + if ((superTblInfo->interlaceRows == 0) + && (g_args.interlace_rows > 0)) { + interlaceRows = g_args.interlace_rows; + } else { + interlaceRows = superTblInfo->interlaceRows; } - errorPrint("\tPlease check if the buffer length(%"PRId64") or batch(%d) is set with proper value!\n", - maxSqlLen, batchPerTbl); - goto free_of_interlace; - } - int64_t affectedRows = execInsert(pThreadInfo, recOfBatch); - - endTs = taosGetTimestampMs(); - uint64_t delay = endTs - startTs; - performancePrint("%s() LN%d, insert execution time is %"PRIu64"ms\n", - __func__, __LINE__, delay); - verbosePrint("[%d] %s() LN%d affectedRows=%"PRId64"\n", - pThreadInfo->threadID, - __func__, __LINE__, affectedRows); - - if (delay > pThreadInfo->maxDelay) pThreadInfo->maxDelay = delay; - if (delay < pThreadInfo->minDelay) pThreadInfo->minDelay = delay; - pThreadInfo->cntDelay++; - pThreadInfo->totalDelay += delay; - - if (recOfBatch != affectedRows) { - errorPrint("[%d] %s() LN%d execInsert insert %d, affected rows: %"PRId64"\n%s\n", - pThreadInfo->threadID, __func__, __LINE__, - recOfBatch, affectedRows, pThreadInfo->buffer); - goto free_of_interlace; + maxSqlLen = superTblInfo->maxSqlLen; + nTimeStampStep = superTblInfo->timeStampStep; + insert_interval = superTblInfo->insertInterval; + } else { + insertRows = g_args.num_of_DPT; + interlaceRows = g_args.interlace_rows; + maxSqlLen = g_args.max_sql_len; + nTimeStampStep = DEFAULT_TIMESTAMP_STEP; + insert_interval = g_args.insert_interval; } - pThreadInfo->totalAffectedRows += affectedRows; + debugPrint("[%d] %s() LN%d: start_table_from=%"PRIu64" ntables=%"PRId64" insertRows=%"PRIu64"\n", + pThreadInfo->threadID, __func__, __LINE__, + pThreadInfo->start_table_from, + pThreadInfo->ntables, insertRows); - int64_t currentPrintTime = taosGetTimestampMs(); - if (currentPrintTime - lastPrintTime > 30*1000) { - printf("thread[%d] has currently inserted rows: %"PRIu64 ", affected rows: %"PRIu64 "\n", + if (interlaceRows > insertRows) + interlaceRows = insertRows; + + if (interlaceRows > g_args.num_of_RPR) + interlaceRows = g_args.num_of_RPR; + + uint32_t batchPerTbl = interlaceRows; + uint32_t batchPerTblTimes; + + if ((interlaceRows > 0) && (pThreadInfo->ntables > 1)) { + batchPerTblTimes = + g_args.num_of_RPR / interlaceRows; + } else { + batchPerTblTimes = 1; + } + + pThreadInfo->buffer = calloc(maxSqlLen, 1); + if (NULL == pThreadInfo->buffer) { + errorPrint( "%s() LN%d, Failed to alloc %"PRIu64" Bytes, reason:%s\n", + __func__, __LINE__, maxSqlLen, strerror(errno)); + return NULL; + } + + pThreadInfo->totalInsertRows = 0; + pThreadInfo->totalAffectedRows = 0; + + uint64_t st = 0; + uint64_t et = UINT64_MAX; + + uint64_t lastPrintTime = taosGetTimestampMs(); + uint64_t startTs = taosGetTimestampMs(); + uint64_t endTs; + + uint64_t tableSeq = pThreadInfo->start_table_from; + int64_t startTime = pThreadInfo->start_time; + + uint64_t generatedRecPerTbl = 0; + bool flagSleep = true; + uint64_t sleepTimeTotal = 0; + + while(pThreadInfo->totalInsertRows < pThreadInfo->ntables * insertRows) { + if ((flagSleep) && (insert_interval)) { + st = taosGetTimestampMs(); + flagSleep = false; + } + // generate data + memset(pThreadInfo->buffer, 0, maxSqlLen); + uint64_t remainderBufLen = maxSqlLen; + + char *pstr = pThreadInfo->buffer; + + int len = snprintf(pstr, + strlen(STR_INSERT_INTO) + 1, "%s", STR_INSERT_INTO); + pstr += len; + remainderBufLen -= len; + + uint32_t recOfBatch = 0; + + for (uint64_t i = 0; i < batchPerTblTimes; i ++) { + char tableName[TSDB_TABLE_NAME_LEN]; + + getTableName(tableName, pThreadInfo, tableSeq); + if (0 == strlen(tableName)) { + errorPrint("[%d] %s() LN%d, getTableName return null\n", + pThreadInfo->threadID, __func__, __LINE__); + free(pThreadInfo->buffer); + return NULL; + } + + uint64_t oldRemainderLen = remainderBufLen; + + int32_t generated; + if (superTblInfo) { + if (superTblInfo->iface == STMT_IFACE) { +#if STMT_IFACE_ENABLED == 1 + generated = prepareStbStmtInterlace( + superTblInfo, + pThreadInfo->stmt, + tableName, + batchPerTbl, + insertRows, i, + startTime, + &(pThreadInfo->samplePos)); +#else + generated = -1; +#endif + } else { + generated = generateStbInterlaceData( + superTblInfo, + tableName, batchPerTbl, i, + batchPerTblTimes, + tableSeq, + pThreadInfo, pstr, + insertRows, + startTime, + &remainderBufLen); + } + } else { + if (g_args.iface == STMT_IFACE) { + debugPrint("[%d] %s() LN%d, tableName:%s, batch:%d startTime:%"PRId64"\n", + pThreadInfo->threadID, + __func__, __LINE__, + tableName, batchPerTbl, startTime); +#if STMT_IFACE_ENABLED == 1 + generated = prepareStmtWithoutStb( + pThreadInfo->stmt, tableName, + batchPerTbl, + insertRows, i, + startTime); +#else + generated = -1; +#endif + } else { + generated = generateInterlaceDataWithoutStb( + tableName, batchPerTbl, + tableSeq, + pThreadInfo->db_name, pstr, + insertRows, + startTime, + &remainderBufLen); + } + } + + debugPrint("[%d] %s() LN%d, generated records is %d\n", + pThreadInfo->threadID, __func__, __LINE__, generated); + if (generated < 0) { + errorPrint("[%d] %s() LN%d, generated records is %d\n", + pThreadInfo->threadID, __func__, __LINE__, generated); + goto free_of_interlace; + } else if (generated == 0) { + break; + } + + tableSeq ++; + recOfBatch += batchPerTbl; + + pstr += (oldRemainderLen - remainderBufLen); + pThreadInfo->totalInsertRows += batchPerTbl; + + verbosePrint("[%d] %s() LN%d batchPerTbl=%d recOfBatch=%d\n", + pThreadInfo->threadID, __func__, __LINE__, + batchPerTbl, recOfBatch); + + if (tableSeq == pThreadInfo->start_table_from + pThreadInfo->ntables) { + // turn to first table + tableSeq = pThreadInfo->start_table_from; + generatedRecPerTbl += batchPerTbl; + + startTime = pThreadInfo->start_time + + generatedRecPerTbl * nTimeStampStep; + + flagSleep = true; + if (generatedRecPerTbl >= insertRows) + break; + + int64_t remainRows = insertRows - generatedRecPerTbl; + if ((remainRows > 0) && (batchPerTbl > remainRows)) + batchPerTbl = remainRows; + + if (pThreadInfo->ntables * batchPerTbl < g_args.num_of_RPR) + break; + } + + verbosePrint("[%d] %s() LN%d generatedRecPerTbl=%"PRId64" insertRows=%"PRId64"\n", + pThreadInfo->threadID, __func__, __LINE__, + generatedRecPerTbl, insertRows); + + if ((g_args.num_of_RPR - recOfBatch) < batchPerTbl) + break; + } + + verbosePrint("[%d] %s() LN%d recOfBatch=%d totalInsertRows=%"PRIu64"\n", + pThreadInfo->threadID, __func__, __LINE__, recOfBatch, + pThreadInfo->totalInsertRows); + verbosePrint("[%d] %s() LN%d, buffer=%s\n", + pThreadInfo->threadID, __func__, __LINE__, pThreadInfo->buffer); + + startTs = taosGetTimestampMs(); + + if (recOfBatch == 0) { + errorPrint("[%d] %s() LN%d Failed to insert records of batch %d\n", + pThreadInfo->threadID, __func__, __LINE__, + batchPerTbl); + if (batchPerTbl > 0) { + errorPrint("\tIf the batch is %d, the length of the SQL to insert a row must be less then %"PRId64"\n", + batchPerTbl, maxSqlLen / batchPerTbl); + } + errorPrint("\tPlease check if the buffer length(%"PRId64") or batch(%d) is set with proper value!\n", + maxSqlLen, batchPerTbl); + goto free_of_interlace; + } + int64_t affectedRows = execInsert(pThreadInfo, recOfBatch); + + endTs = taosGetTimestampMs(); + uint64_t delay = endTs - startTs; + performancePrint("%s() LN%d, insert execution time is %"PRIu64"ms\n", + __func__, __LINE__, delay); + verbosePrint("[%d] %s() LN%d affectedRows=%"PRId64"\n", + pThreadInfo->threadID, + __func__, __LINE__, affectedRows); + + if (delay > pThreadInfo->maxDelay) pThreadInfo->maxDelay = delay; + if (delay < pThreadInfo->minDelay) pThreadInfo->minDelay = delay; + pThreadInfo->cntDelay++; + pThreadInfo->totalDelay += delay; + + if (recOfBatch != affectedRows) { + errorPrint("[%d] %s() LN%d execInsert insert %d, affected rows: %"PRId64"\n%s\n", + pThreadInfo->threadID, __func__, __LINE__, + recOfBatch, affectedRows, pThreadInfo->buffer); + goto free_of_interlace; + } + + pThreadInfo->totalAffectedRows += affectedRows; + + int64_t currentPrintTime = taosGetTimestampMs(); + if (currentPrintTime - lastPrintTime > 30*1000) { + printf("thread[%d] has currently inserted rows: %"PRIu64 ", affected rows: %"PRIu64 "\n", pThreadInfo->threadID, pThreadInfo->totalInsertRows, pThreadInfo->totalAffectedRows); - lastPrintTime = currentPrintTime; - } + lastPrintTime = currentPrintTime; + } - if ((insert_interval) && flagSleep) { - et = taosGetTimestampMs(); + if ((insert_interval) && flagSleep) { + et = taosGetTimestampMs(); - if (insert_interval > (et - st) ) { - uint64_t sleepTime = insert_interval - (et -st); - performancePrint("%s() LN%d sleep: %"PRId64" ms for insert interval\n", - __func__, __LINE__, sleepTime); - taosMsleep(sleepTime); // ms - sleepTimeTotal += insert_interval; - } + if (insert_interval > (et - st) ) { + uint64_t sleepTime = insert_interval - (et -st); + performancePrint("%s() LN%d sleep: %"PRId64" ms for insert interval\n", + __func__, __LINE__, sleepTime); + taosMsleep(sleepTime); // ms + sleepTimeTotal += insert_interval; + } + } } - } free_of_interlace: - tmfree(pThreadInfo->buffer); - printStatPerThread(pThreadInfo); - return NULL; + tmfree(pThreadInfo->buffer); + printStatPerThread(pThreadInfo); + return NULL; } // sync insertion progressive data @@ -6417,29 +6423,29 @@ free_of_progressive: static void* syncWrite(void *sarg) { - threadInfo *pThreadInfo = (threadInfo *)sarg; - SSuperTable* superTblInfo = pThreadInfo->superTblInfo; + threadInfo *pThreadInfo = (threadInfo *)sarg; + SSuperTable* superTblInfo = pThreadInfo->superTblInfo; - uint32_t interlaceRows; + uint32_t interlaceRows; - if (superTblInfo) { - if ((superTblInfo->interlaceRows == 0) - && (g_args.interlace_rows > 0)) { - interlaceRows = g_args.interlace_rows; + if (superTblInfo) { + if ((superTblInfo->interlaceRows == 0) + && (g_args.interlace_rows > 0)) { + interlaceRows = g_args.interlace_rows; + } else { + interlaceRows = superTblInfo->interlaceRows; + } } else { - interlaceRows = superTblInfo->interlaceRows; + interlaceRows = g_args.interlace_rows; } - } else { - interlaceRows = g_args.interlace_rows; - } - if (interlaceRows > 0) { - // interlace mode - return syncWriteInterlace(pThreadInfo); - } else { - // progressive mode - return syncWriteProgressive(pThreadInfo); - } + if (interlaceRows > 0) { + // interlace mode + return syncWriteInterlace(pThreadInfo); + } else { + // progressive mode + return syncWriteProgressive(pThreadInfo); + } } static void callBack(void *param, TAOS_RES *res, int code) { From f3fd5d1a66cc63bf03fa0e1f6119fca9ad18975e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 19 Jul 2021 14:49:11 +0800 Subject: [PATCH 56/67] [td-5376]: improve last_row query performance. --- src/tsdb/src/tsdbRead.c | 22 +++-- src/util/CMakeLists.txt | 5 +- src/util/inc/tarray.h | 9 ++ src/util/src/tarray.c | 41 ++++++++++ src/util/src/tskiplist.c | 140 -------------------------------- src/util/tests/cacheTest.cpp | 8 +- src/util/tests/skiplistTest.cpp | 27 +++--- src/util/tests/trefTest.c | 16 ++-- 8 files changed, 90 insertions(+), 178 deletions(-) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 1eafb5e233..924b5e24a8 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -3002,7 +3002,7 @@ int32_t checkForCachedLastRow(STsdbQueryHandle* pQueryHandle, STableGroupInfo *g } // update the tsdb query time range - if (pQueryHandle->cachelastrow) { + if (pQueryHandle->cachelastrow != TSDB_CACHED_TYPE_NONE) { pQueryHandle->window = TSWINDOW_INITIALIZER; pQueryHandle->checkFiles = false; pQueryHandle->activeIndex = -1; // start from -1 @@ -3034,6 +3034,7 @@ STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList) { STimeWindow window = {INT64_MAX, INT64_MIN}; int32_t totalNumOfTable = 0; + SArray* emptyGroup = taosArrayInit(16, sizeof(int32_t)); // NOTE: starts from the buffer in case of descending timestamp order check data blocks size_t numOfGroups = taosArrayGetSize(groupList->pGroupList); @@ -3076,18 +3077,18 @@ STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList) { } } - taosArrayClear(pGroup); - // more than one table in each group, only one table left for each group if (keyInfo.pTable != NULL) { totalNumOfTable++; - taosArrayPush(pGroup, &keyInfo); - } else { + if (taosArrayGetSize(pGroup) == 1) { + // do nothing + } else { + taosArrayClear(pGroup); + taosArrayPush(pGroup, &keyInfo); + } + } else { // mark all the empty groups, and remove it later taosArrayDestroy(pGroup); - - taosArrayRemove(groupList->pGroupList, j); - numOfGroups -= 1; - j -= 1; + taosArrayPush(emptyGroup, &j); } } @@ -3097,6 +3098,9 @@ STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList) { assert(totalNumOfTable == 0 && taosArrayGetSize(groupList->pGroupList) == 0); } + taosArrayRemoveBatch(groupList->pGroupList, TARRAY_GET_START(emptyGroup), taosArrayGetSize(emptyGroup)); + taosArrayDestroy(emptyGroup); + groupList->numOfTables = totalNumOfTable; return window; } diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index 85b15c0a4f..cb08944cbc 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -4,13 +4,14 @@ PROJECT(TDengine) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/rpc/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/sync/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/rmonotonic/inc) + AUX_SOURCE_DIRECTORY(src SRC) -ADD_LIBRARY(tutil ${SRC}) +ADD_LIBRARY(tutil ${SRC} tests/arrayTest.cpp) TARGET_LINK_LIBRARIES(tutil pthread os lz4 z rmonotonic) IF (TD_LINUX) TARGET_LINK_LIBRARIES(tutil m rt) - # ADD_SUBDIRECTORY(tests) + ADD_SUBDIRECTORY(tests) FIND_PATH(ICONV_INCLUDE_EXIST iconv.h /usr/include/ /usr/local/include/) IF (ICONV_INCLUDE_EXIST) diff --git a/src/util/inc/tarray.h b/src/util/inc/tarray.h index 63cadf39a3..97aeaa61d4 100644 --- a/src/util/inc/tarray.h +++ b/src/util/inc/tarray.h @@ -52,6 +52,15 @@ void* taosArrayInit(size_t size, size_t elemSize); */ void *taosArrayAddBatch(SArray *pArray, const void *pData, int nEles); +/** + * + * @param pArray + * @param pData position array list + * @param numOfElems the number of removed position + */ +void taosArrayRemoveBatch(SArray *pArray, const int32_t* pData, int32_t numOfElems); + + /** * add all element from the source array list into the destination * @param pArray diff --git a/src/util/src/tarray.c b/src/util/src/tarray.c index fe529edaac..d25d547be1 100644 --- a/src/util/src/tarray.c +++ b/src/util/src/tarray.c @@ -83,6 +83,47 @@ void* taosArrayAddBatch(SArray* pArray, const void* pData, int nEles) { return dst; } +void taosArrayRemoveBatch(SArray *pArray, const int32_t* pData, int32_t numOfElems) { + assert(pArray != NULL && pData != NULL); + if (numOfElems <= 0) { + return; + } + + size_t size = taosArrayGetSize(pArray); + if (numOfElems >= size) { + taosArrayClear(pArray); + return; + } + + int32_t i = pData[0] + 1, j = 0; + while(i < size) { + if (j == numOfElems - 1) { + break; + } + + char* p = TARRAY_GET_ELEM(pArray, i); + if (i > pData[j] && i < pData[j + 1]) { + char* dst = TARRAY_GET_ELEM(pArray, i - (j + 1)); + memmove(dst, p, pArray->elemSize); + } else if (i == pData[j + 1]) { + j += 1; + } + + i += 1; + } + + assert(i == pData[numOfElems - 1] + 1); + + int32_t dstIndex = pData[numOfElems - 1] - numOfElems + 1; + int32_t srcIndex = pData[numOfElems - 1] + 1; + + char* dst = TARRAY_GET_ELEM(pArray, dstIndex); + char* src = TARRAY_GET_ELEM(pArray, srcIndex); + memmove(dst, src, pArray->elemSize * (pArray->size - numOfElems)); + + pArray->size -= numOfElems; +} + void* taosArrayAddAll(SArray* pArray, const SArray* pInput) { return taosArrayAddBatch(pArray, pInput->pData, (int32_t) taosArrayGetSize(pInput)); } diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index a88fee51d7..082b454bb5 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -681,143 +681,3 @@ static SSkipListNode *tSkipListPutImpl(SSkipList *pSkipList, void *pData, SSkipL return pNode; } - -// static int32_t tSkipListEndParQuery(SSkipList *pSkipList, SSkipListNode *pStartNode, SSkipListKey *pEndKey, -// int32_t cond, SSkipListNode ***pRes) { -// pthread_rwlock_rdlock(&pSkipList->lock); -// SSkipListNode *p = pStartNode; -// int32_t numOfRes = 0; -// -// __compar_fn_t filterComparFn = getComparFunc(pSkipList, pEndKey->nType); -// while (p != NULL) { -// int32_t ret = filterComparFn(&p->key, pEndKey); -// if (ret > 0) { -// break; -// } -// -// if (ret < 0) { -// numOfRes++; -// p = p->pForward[0]; -// } else if (ret == 0) { -// if (cond == TSDB_RELATION_LESS_EQUAL) { -// numOfRes++; -// p = p->pForward[0]; -// } else { -// break; -// } -// } -// } -// -// (*pRes) = (SSkipListNode **)malloc(POINTER_BYTES * numOfRes); -// for (int32_t i = 0; i < numOfRes; ++i) { -// (*pRes)[i] = pStartNode; -// pStartNode = pStartNode->pForward[0]; -// } -// pthread_rwlock_unlock(&pSkipList->lock); -// -// return numOfRes; -//} -// -///* -// * maybe return the copy of SSkipListNode would be better -// */ -// int32_t tSkipListGets(SSkipList *pSkipList, SSkipListKey *pKey, SSkipListNode ***pRes) { -// (*pRes) = NULL; -// -// SSkipListNode *pNode = tSkipListGet(pSkipList, pKey); -// if (pNode == NULL) { -// return 0; -// } -// -// __compar_fn_t filterComparFn = getComparFunc(pSkipList, pKey->nType); -// -// // backward check if previous nodes are with the same value. -// SSkipListNode *pPrev = pNode->pBackward[0]; -// while ((pPrev != &pSkipList->pHead) && filterComparFn(&pPrev->key, pKey) == 0) { -// pPrev = pPrev->pBackward[0]; -// } -// -// return tSkipListEndParQuery(pSkipList, pPrev->pForward[0], &pNode->key, TSDB_RELATION_LESS_EQUAL, pRes); -//} -// -// static SSkipListNode *tSkipListParQuery(SSkipList *pSkipList, SSkipListKey *pKey, int32_t cond) { -// int32_t sLevel = pSkipList->level - 1; -// int32_t ret = -1; -// -// SSkipListNode *x = &pSkipList->pHead; -// __compar_fn_t filterComparFn = getComparFunc(pSkipList, pKey->nType); -// -// pthread_rwlock_rdlock(&pSkipList->lock); -// -// if (cond == TSDB_RELATION_GREATER_EQUAL || cond == TSDB_RELATION_GREATER) { -// for (int32_t i = sLevel; i >= 0; --i) { -// while (x->pForward[i] != NULL && (ret = filterComparFn(&x->pForward[i]->key, pKey)) < 0) { -// x = x->pForward[i]; -// } -// } -// -// // backward check if previous nodes are with the same value. -// if (cond == TSDB_RELATION_GREATER_EQUAL && ret == 0) { -// SSkipListNode *pNode = x->pForward[0]; -// while ((pNode->pBackward[0] != &pSkipList->pHead) && (filterComparFn(&pNode->pBackward[0]->key, pKey) == 0)) { -// pNode = pNode->pBackward[0]; -// } -// pthread_rwlock_unlock(&pSkipList->lock); -// return pNode; -// } -// -// if (ret > 0 || cond == TSDB_RELATION_GREATER_EQUAL) { -// pthread_rwlock_unlock(&pSkipList->lock); -// return x->pForward[0]; -// } else { // cond == TSDB_RELATION_GREATER && ret == 0 -// SSkipListNode *pn = x->pForward[0]; -// while (pn != NULL && filterComparFn(&pn->key, pKey) == 0) { -// pn = pn->pForward[0]; -// } -// pthread_rwlock_unlock(&pSkipList->lock); -// return pn; -// } -// } -// -// pthread_rwlock_unlock(&pSkipList->lock); -// return NULL; -//} -// -// -// static bool removeSupport(SSkipList *pSkipList, SSkipListNode **forward, SSkipListKey *pKey) { -// __compar_fn_t filterComparFn = getComparFunc(pSkipList, pKey->nType); -// -// if (filterComparFn(&forward[0]->pForward[0]->key, pKey) == 0) { -// SSkipListNode *p = forward[0]->pForward[0]; -// doRemove(pSkipList, p, forward); -// } else { // failed to find the node of specified value,abort -// return false; -// } -// -// // compress the minimum level of skip list -// while (pSkipList->level > 0 && SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, pSkipList->level - 1) == NULL) { -// pSkipList->level -= 1; -// } -// -// return true; -//} -// -// bool tSkipListRemove(SSkipList *pSkipList, SSkipListKey *pKey) { -// SSkipListNode *forward[MAX_SKIP_LIST_LEVEL] = {0}; -// __compar_fn_t filterComparFn = getComparFunc(pSkipList, pKey->nType); -// -// pthread_rwlock_rdlock(&pSkipList->lock); -// -// SSkipListNode *x = &pSkipList->pHead; -// for (int32_t i = pSkipList->level - 1; i >= 0; --i) { -// while (x->pForward[i] != NULL && (filterComparFn(&x->pForward[i]->key, pKey) < 0)) { -// x = x->pForward[i]; -// } -// forward[i] = x; -// } -// -// bool ret = removeSupport(pSkipList, forward, pKey); -// pthread_rwlock_unlock(&pSkipList->lock); -// -// return ret; -//} diff --git a/src/util/tests/cacheTest.cpp b/src/util/tests/cacheTest.cpp index 0a4791f6a9..970f1c23a9 100644 --- a/src/util/tests/cacheTest.cpp +++ b/src/util/tests/cacheTest.cpp @@ -5,10 +5,6 @@ #include "taos.h" #include "tcache.h" -namespace { -int32_t tsMaxMgmtConnections = 10000; -int32_t tsMaxMeterConnections = 200; -} // test cache TEST(testCase, client_cache_test) { const int32_t REFRESH_TIME_IN_SEC = 2; @@ -43,7 +39,7 @@ TEST(testCase, client_cache_test) { sleep(3); char* d = (char*) taosCacheAcquireByKey(tscMetaCache, key3, strlen(key3)); -// assert(d == NULL); + assert(d == NULL); char key5[] = "test5"; char data5[] = "data5kkkkk"; @@ -102,7 +98,7 @@ TEST(testCase, cache_resize_test) { char key[256] = {0}; char data[1024] = "abcdefghijk"; - int32_t len = strlen(data); +// int32_t len = strlen(data); uint64_t startTime = taosGetTimestampUs(); int32_t num = 10000; diff --git a/src/util/tests/skiplistTest.cpp b/src/util/tests/skiplistTest.cpp index 2203ae8e4f..dfbe0f6716 100644 --- a/src/util/tests/skiplistTest.cpp +++ b/src/util/tests/skiplistTest.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include "os.h" @@ -8,12 +9,14 @@ #include "tskiplist.h" #include "tutil.h" +#if 0 namespace { char* getkey(const void* data) { return (char*)(data); } void doubleSkipListTest() { - SSkipList* pSkipList = tSkipListCreate(10, TSDB_DATA_TYPE_DOUBLE, sizeof(double), 0, false, true, getkey); + SSkipList* pSkipList = tSkipListCreate(10, TSDB_DATA_TYPE_DOUBLE, sizeof(double), + getKeyComparFunc(TSDB_DATA_TYPE_DOUBLE), false, getkey); double doubleVal[1000] = {0}; int32_t size = 20000; @@ -25,18 +28,15 @@ void doubleSkipListTest() { doubleVal[i] = i * 0.997; } - int32_t level = 0; - int32_t size = 0; +// int32_t level = 0; + size = 0; - tSkipListNewNodeInfo(pSkipList, &level, &size); - auto d = (SSkipListNode*)calloc(1, size + sizeof(double) * 2); - d->level = level; + // tSkipListNewNodeInfo(pSkipList, &level, &size); + // auto d = (SSkipListNode*)calloc(1, size + sizeof(double) * 2); + // d->level = level; - double* key = (double*)SL_GET_NODE_KEY(pSkipList, d); - key[0] = i * 0.997; - key[1] = i * 0.997; - - tSkipListPut(pSkipList, d); + double key = 0.997; + tSkipListPut(pSkipList, &key); } printf("the first level of skip list is:\n"); @@ -70,7 +70,8 @@ void doubleSkipListTest() { } void randKeyTest() { - SSkipList* pSkipList = tSkipListCreate(10, TSDB_DATA_TYPE_INT, sizeof(int32_t), 0, false, true, getkey); + SSkipList* pSkipList = tSkipListCreate(10, TSDB_DATA_TYPE_INT, sizeof(int32_t), getKeyComparFunc(TSDB_DATA_TYPE_INT), + false, getkey); int32_t size = 200000; srand(time(NULL)); @@ -375,3 +376,5 @@ TEST(testCase, skiplist_test) { free(pKeys);*/ } + +#endif \ No newline at end of file diff --git a/src/util/tests/trefTest.c b/src/util/tests/trefTest.c index 454860410b..e01da070af 100644 --- a/src/util/tests/trefTest.c +++ b/src/util/tests/trefTest.c @@ -14,18 +14,18 @@ typedef struct { int refNum; int steps; int rsetId; - int64_t rid; + int64_t*rid; void **p; } SRefSpace; void iterateRefs(int rsetId) { int count = 0; - void *p = taosIterateRef(rsetId, NULL); + void *p = taosIterateRef(rsetId, 0); while (p) { // process P count++; - p = taosIterateRef(rsetId, p); + p = taosIterateRef(rsetId, (int64_t) p); } printf(" %d ", count); @@ -34,7 +34,6 @@ void iterateRefs(int rsetId) { void *addRef(void *param) { SRefSpace *pSpace = (SRefSpace *)param; int id; - int64_t rid; for (int i=0; i < pSpace->steps; ++i) { printf("a"); @@ -51,8 +50,7 @@ void *addRef(void *param) { void *removeRef(void *param) { SRefSpace *pSpace = (SRefSpace *)param; - int id; - int64_t rid; + int id, code; for (int i=0; i < pSpace->steps; ++i) { printf("d"); @@ -71,16 +69,15 @@ void *removeRef(void *param) { void *acquireRelease(void *param) { SRefSpace *pSpace = (SRefSpace *)param; int id; - int64_t rid; for (int i=0; i < pSpace->steps; ++i) { printf("a"); id = random() % pSpace->refNum; - void *p = taosAcquireRef(pSpace->rsetId, pSpace->p[id]); + void *p = taosAcquireRef(pSpace->rsetId, (int64_t) pSpace->p[id]); if (p) { usleep(id % 5 + 1); - taosReleaseRef(pSpace->rsetId, pSpace->p[id]); + taosReleaseRef(pSpace->rsetId, (int64_t) pSpace->p[id]); } } @@ -103,6 +100,7 @@ void *openRefSpace(void *param) { } pSpace->p = (void **) calloc(sizeof(void *), pSpace->refNum); + pSpace->rid = calloc(pSpace->refNum, sizeof(int64_t)); pthread_attr_t thattr; pthread_attr_init(&thattr); From 83dc3b31cfd73309225d1b2ada263f6a394f231e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 19 Jul 2021 15:13:57 +0800 Subject: [PATCH 57/67] [td-225]add test files. --- src/tsdb/src/tsdbRead.c | 2 +- src/util/tests/arrayTest.cpp | 52 ++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 src/util/tests/arrayTest.cpp diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 924b5e24a8..6c16c72c2a 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -3095,7 +3095,7 @@ STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList) { // window does not being updated, so set the original if (window.skey == INT64_MAX && window.ekey == INT64_MIN) { window = TSWINDOW_INITIALIZER; - assert(totalNumOfTable == 0 && taosArrayGetSize(groupList->pGroupList) == 0); + assert(totalNumOfTable == 0 && taosArrayGetSize(groupList->pGroupList) == numOfGroups); } taosArrayRemoveBatch(groupList->pGroupList, TARRAY_GET_START(emptyGroup), taosArrayGetSize(emptyGroup)); diff --git a/src/util/tests/arrayTest.cpp b/src/util/tests/arrayTest.cpp new file mode 100644 index 0000000000..08983b6c06 --- /dev/null +++ b/src/util/tests/arrayTest.cpp @@ -0,0 +1,52 @@ +#include +#include +#include +#include + +#include "tarray.h" + +namespace { + +static void remove_batch_test() { + SArray *pa = (SArray*) taosArrayInit(4, sizeof(int32_t)); + + for(int32_t i = 0; i < 20; ++i) { + int32_t a = i; + taosArrayPush(pa, &a); + } + + SArray* delList = (SArray*)taosArrayInit(4, sizeof(int32_t)); + taosArrayRemoveBatch(pa, (const int32_t*) TARRAY_GET_START(delList), taosArrayGetSize(delList)); + EXPECT_EQ(taosArrayGetSize(pa), 20); + + int32_t a = 5; + taosArrayPush(delList, &a); + + taosArrayRemoveBatch(pa, (const int32_t*) TARRAY_GET_START(delList), taosArrayGetSize(delList)); + EXPECT_EQ(taosArrayGetSize(pa), 19); + EXPECT_EQ(*(int*)taosArrayGet(pa, 5), 6); + + taosArrayInsert(pa, 5, &a); + EXPECT_EQ(taosArrayGetSize(pa), 20); + EXPECT_EQ(*(int*)taosArrayGet(pa, 5), 5); + + taosArrayClear(delList); + + a = 6; + taosArrayPush(delList, &a); + + a = 9; + taosArrayPush(delList, &a); + + a = 14; + taosArrayPush(delList, &a); + taosArrayRemoveBatch(pa, (const int32_t*) TARRAY_GET_START(delList), taosArrayGetSize(delList)); + EXPECT_EQ(taosArrayGetSize(pa), 17); +} +} // namespace + +TEST(arrayTest, array_list_test) { + + remove_batch_test(); + +} From b9a321e8c55249e74eabf1bfc211a303f1d86c79 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 19 Jul 2021 15:18:27 +0800 Subject: [PATCH 58/67] [td-225]disable utiltest --- src/util/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index cb08944cbc..bcbadc26d5 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -11,7 +11,7 @@ TARGET_LINK_LIBRARIES(tutil pthread os lz4 z rmonotonic) IF (TD_LINUX) TARGET_LINK_LIBRARIES(tutil m rt) - ADD_SUBDIRECTORY(tests) +# ADD_SUBDIRECTORY(tests) FIND_PATH(ICONV_INCLUDE_EXIST iconv.h /usr/include/ /usr/local/include/) IF (ICONV_INCLUDE_EXIST) From b4154d8ad6a1535b57987fbd5b5facdecc117560 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 19 Jul 2021 15:45:48 +0800 Subject: [PATCH 59/67] [td-255]update cmake files. --- src/util/CMakeLists.txt | 2 +- src/util/tests/arrayTest.cpp | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index bcbadc26d5..27d9e11053 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -6,7 +6,7 @@ INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/sync/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/rmonotonic/inc) AUX_SOURCE_DIRECTORY(src SRC) -ADD_LIBRARY(tutil ${SRC} tests/arrayTest.cpp) +ADD_LIBRARY(tutil ${SRC}) TARGET_LINK_LIBRARIES(tutil pthread os lz4 z rmonotonic) IF (TD_LINUX) diff --git a/src/util/tests/arrayTest.cpp b/src/util/tests/arrayTest.cpp index 08983b6c06..94b08ca6d7 100644 --- a/src/util/tests/arrayTest.cpp +++ b/src/util/tests/arrayTest.cpp @@ -46,7 +46,5 @@ static void remove_batch_test() { } // namespace TEST(arrayTest, array_list_test) { - remove_batch_test(); - } From 182f97676dea9caa842d15c27031976433e1262f Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Mon, 19 Jul 2021 16:23:23 +0800 Subject: [PATCH 60/67] [TD-5255] : fix table names typo. --- documentation20/cn/02.getting-started/docs.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/documentation20/cn/02.getting-started/docs.md b/documentation20/cn/02.getting-started/docs.md index fa36481646..c67b9e61b9 100644 --- a/documentation20/cn/02.getting-started/docs.md +++ b/documentation20/cn/02.getting-started/docs.md @@ -126,7 +126,7 @@ taos> source ; $ taosdemo ``` -该命令将在数据库 test 下面自动创建一张超级表 meters，该超级表下有 1 万张表，表名为 "t0" 到 "t9999"，每张表有 1 万条记录，每条记录有 (ts, current, voltage, phase) 四个字段，时间戳从 "2017-07-14 10:40:00 000" 到 "2017-07-14 10:40:09 999"，每张表带有标签 location 和 groupdId，groupdId 被设置为 1 到 10， location 被设置为 "beijing" 或者 "shanghai"。 +该命令将在数据库 test 下面自动创建一张超级表 meters，该超级表下有 1 万张表，表名为 "d0" 到 "d9999"，每张表有 1 万条记录，每条记录有 (ts, current, voltage, phase) 四个字段，时间戳从 "2017-07-14 10:40:00 000" 到 "2017-07-14 10:40:09 999"，每张表带有标签 location 和 groupdId，groupdId 被设置为 1 到 10， location 被设置为 "beijing" 或者 "shanghai"。 执行这条命令大概需要几分钟，最后共插入 1 亿条记录。 @@ -156,10 +156,10 @@ taos> select count(*) from test.meters where location="beijing"; taos> select avg(current), max(voltage), min(phase) from test.meters where groupdId=10; ``` -- 对表 t10 按 10s 进行平均值、最大值和最小值聚合统计： +- 对表 d10 按 10s 进行平均值、最大值和最小值聚合统计： ```mysql -taos> select avg(current), max(voltage), min(phase) from test.t10 interval(10s); +taos> select avg(current), max(voltage), min(phase) from test.d10 interval(10s); ``` **Note:** taosdemo 命令本身带有很多选项，配置表的数目、记录条数等等，请执行 `taosdemo --help` 详细列出。您可以设置不同参数进行体验。 From 1800c2e846e290ede088dc313a64d2dd95f99636 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 19 Jul 2021 16:50:25 +0800 Subject: [PATCH 61/67] support payload type for both prepare and non-prepare mode --- src/client/inc/tscUtil.h | 1 + src/client/inc/tsclient.h | 9 ++ src/client/src/tscParseInsert.c | 4 +- src/client/src/tscPrepare.c | 227 ++------------------------------ src/client/src/tscUtil.c | 88 +++++++++---- 5 files changed, 81 insertions(+), 248 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index abc3d47e0b..9557d3afda 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -108,6 +108,7 @@ int32_t converToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *le int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOffset, SName* name, STableMeta* pTableMeta, STableDataBlocks** dataBlocks); void tscDestroyDataBlock(STableDataBlocks* pDataBlock, bool removeMeta); +void tscSortRemoveDataBlockDupRowsRaw(STableDataBlocks* dataBuf); int tscSortRemoveDataBlockDupRows(STableDataBlocks* dataBuf, SBlockKeyInfo* pBlkKeyInfo); void tscDestroyBoundColumnInfo(SParsedDataColInfo* pColInfo); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 37a6b4b051..eeb7cc68aa 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -159,6 +159,7 @@ typedef struct SInsertStatementParam { SHashObj *pTableBlockHashList; // data block for each table SArray *pDataBlocks; // SArray. Merged submit block for each vgroup int8_t schemaAttached; // denote if submit block is built with table schema or not + uint8_t payloadType; // EPayloadType. 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert STagData tagData; // NOTE: pTagData->data is used as a variant length array int32_t batchSize; // for parameter ('?') binding and batch processing @@ -170,6 +171,14 @@ typedef struct SInsertStatementParam { char *sql; // current sql statement position } SInsertStatementParam; +typedef enum { + PAYLOAD_TYPE_KV = 0, + PAYLOAD_TYPE_RAW = 1, +} EPayloadType; + +#define IS_RAW_PAYLOAD(t) \ + (((int)(t)) == PAYLOAD_TYPE_RAW) // 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert + // TODO extract sql parser supporter typedef struct { int command; diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index c3bf3605b2..adc24dfbab 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -1069,9 +1069,8 @@ static int32_t tsSetBlockInfo(SSubmitBlk *pBlocks, const STableMeta *pTableMeta, } } -#if 0 // data block is disordered, sort it in ascending order -static void tscSortRemoveDataBlockDupRowsOld(STableDataBlocks *dataBuf) { +void tscSortRemoveDataBlockDupRowsRaw(STableDataBlocks *dataBuf) { SSubmitBlk *pBlocks = (SSubmitBlk *)dataBuf->pData; // size is less than the total size, since duplicated rows may be removed yet. @@ -1114,7 +1113,6 @@ static void tscSortRemoveDataBlockDupRowsOld(STableDataBlocks *dataBuf) { dataBuf->prevTS = INT64_MIN; } -#endif // data block is disordered, sort it in ascending order int tscSortRemoveDataBlockDupRows(STableDataBlocks *dataBuf, SBlockKeyInfo *pBlkKeyInfo) { diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 36c10f6cd5..d39f9ebe24 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -291,7 +291,6 @@ static char* normalStmtBuildSql(STscStmt* stmt) { return taosStringBuilderGetResult(&sb, NULL); } -#if 0 static int fillColumnsNull(STableDataBlocks* pBlock, int32_t rowNum) { SParsedDataColInfo* spd = &pBlock->boundColumnInfo; int32_t offset = 0; @@ -319,129 +318,8 @@ static int fillColumnsNull(STableDataBlocks* pBlock, int32_t rowNum) { return TSDB_CODE_SUCCESS; } -#endif - -/** - * input: - * - schema: - * - payload: - * - spd: - * output: - * - pBlock with data block replaced by K-V format - */ -static int refactorPayload(STableDataBlocks* pBlock, int32_t rowNum) { - SParsedDataColInfo* spd = &pBlock->boundColumnInfo; - SSchema* schema = (SSchema*)pBlock->pTableMeta->schema; - SMemRowHelper* pHelper = &pBlock->rowHelper; - STableMeta* pTableMeta = pBlock->pTableMeta; - STableComInfo tinfo = tscGetTableInfo(pTableMeta); - int code = TSDB_CODE_SUCCESS; - int32_t extendedRowSize = getExtendedRowSize(&tinfo); - TDRowTLenT destPayloadSize = sizeof(SSubmitBlk); - - ASSERT(pHelper->allNullLen >= 8); - - TDRowTLenT destAllocSize = sizeof(SSubmitBlk) + rowNum * extendedRowSize; - SSubmitBlk* pDestBlock = tcalloc(destAllocSize, 1); - if (pDestBlock == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - memcpy(pDestBlock, pBlock->pData, sizeof(SSubmitBlk)); - char* destPayload = (char*)pDestBlock + sizeof(SSubmitBlk); - - char* srcPayload = (char*)pBlock->pData + sizeof(SSubmitBlk); - - for (int n = 0; n < rowNum; ++n) { - payloadSetNCols(destPayload, spd->numOfBound); - - TDRowTLenT dataRowLen = pHelper->allNullLen; - TDRowTLenT kvRowLen = TD_MEM_ROW_KV_VER_SIZE + sizeof(SColIdx) * spd->numOfBound; - TDRowTLenT payloadValOffset = payloadValuesOffset(destPayload); // rely on payloadNCols - TDRowLenT colValOffset = 0; - - char* kvPrimaryKeyStart = destPayload + PAYLOAD_HEADER_LEN; // primaryKey in 1st column tuple - char* kvStart = kvPrimaryKeyStart + PAYLOAD_COL_HEAD_LEN; // the column tuple behind the primaryKey - - for (int32_t i = 0; i < spd->numOfBound; ++i) { - int32_t colIndex = spd->boundedColumns[i]; - ASSERT(spd->cols[colIndex].hasVal); - char* start = srcPayload + spd->cols[colIndex].offset; - SSchema* pSchema = &schema[colIndex]; // get colId here - bool isPrimaryKey = (colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX); - - // the primary key locates in 1st column - if (!IS_DATA_COL_ORDERED(spd->orderStatus)) { - ASSERT(spd->colIdxInfo != NULL); - if (!isPrimaryKey) { - kvStart = POINTER_SHIFT(kvPrimaryKeyStart, spd->colIdxInfo[i].finalIdx * PAYLOAD_COL_HEAD_LEN); - } else { - ASSERT(spd->colIdxInfo[i].finalIdx == 0); - } - } - if (isPrimaryKey) { - payloadColSetId(kvPrimaryKeyStart, pSchema->colId); - payloadColSetType(kvPrimaryKeyStart, pSchema->type); - payloadColSetOffset(kvPrimaryKeyStart, colValOffset); - memcpy(POINTER_SHIFT(destPayload, payloadValOffset + colValOffset), start, TYPE_BYTES[pSchema->type]); - colValOffset += TYPE_BYTES[pSchema->type]; - kvRowLen += TYPE_BYTES[pSchema->type]; - } else { - payloadColSetId(kvStart, pSchema->colId); - payloadColSetType(kvStart, pSchema->type); - payloadColSetOffset(kvStart, colValOffset); - if (IS_VAR_DATA_TYPE(pSchema->type)) { - varDataCopy(POINTER_SHIFT(destPayload, payloadValOffset + colValOffset), start); - colValOffset += varDataTLen(start); - kvRowLen += varDataTLen(start); - if (pSchema->type == TSDB_DATA_TYPE_BINARY) { - dataRowLen += (varDataLen(start) - CHAR_BYTES); - } else if (pSchema->type == TSDB_DATA_TYPE_NCHAR) { - dataRowLen += (varDataLen(start) - TSDB_NCHAR_SIZE); - } else { - ASSERT(0); - } - } else { - memcpy(POINTER_SHIFT(destPayload, payloadValOffset + colValOffset), start, TYPE_BYTES[pSchema->type]); - colValOffset += TYPE_BYTES[pSchema->type]; - kvRowLen += TYPE_BYTES[pSchema->type]; - } - - if (IS_DATA_COL_ORDERED(spd->orderStatus)) { - kvStart += PAYLOAD_COL_HEAD_LEN; // move to next column - } - } - } // end of column - - if (kvRowLen < dataRowLen) { - payloadSetType(destPayload, SMEM_ROW_KV); - } else { - payloadSetType(destPayload, SMEM_ROW_DATA); - } - - ASSERT(colValOffset <= TSDB_MAX_BYTES_PER_ROW); - - TDRowTLenT len = payloadValOffset + colValOffset; - payloadSetTLen(destPayload, len); - - // next loop - srcPayload += pBlock->rowSize; - destPayload += len; - - destPayloadSize += len; - } // end of row - - ASSERT(destPayloadSize <= destAllocSize); - - tfree(pBlock->pData); - pBlock->pData = (char*)pDestBlock; - pBlock->nAllocSize = destAllocSize; - pBlock->size = destPayloadSize; - - return code; -} -#if 0 int32_t fillTablesColumnsNull(SSqlObj* pSql) { SSqlCmd* pCmd = &pSql->cmd; @@ -464,98 +342,17 @@ int32_t fillTablesColumnsNull(SSqlObj* pSql) { return TSDB_CODE_SUCCESS; } -#endif -/** - * check and sort - */ -static int initPayloadEnv(STableDataBlocks* pBlock, int32_t rowNum) { - SParsedDataColInfo* spd = &pBlock->boundColumnInfo; - if (spd->orderStatus != ORDER_STATUS_UNKNOWN) { + + +//////////////////////////////////////////////////////////////////////////////// +// functions for insertion statement preparation +static int doBindParam(STableDataBlocks* pBlock, char* data, SParamInfo* param, TAOS_BIND* bind, int32_t colNum) { + if (bind->is_null != NULL && *(bind->is_null)) { + setNull(data + param->offset, param->type, param->bytes); return TSDB_CODE_SUCCESS; } - bool isOrdered = true; - int32_t lastColIdx = -1; - for (int32_t i = 0; i < spd->numOfBound; ++i) { - ASSERT(spd->cols[i].hasVal); - int32_t colIdx = spd->boundedColumns[i]; - if (isOrdered) { - if (lastColIdx > colIdx) { - isOrdered = false; - break; - } else { - lastColIdx = colIdx; - } - } - } - - spd->orderStatus = isOrdered ? ORDER_STATUS_ORDERED : ORDER_STATUS_DISORDERED; - - if (isOrdered) { - spd->colIdxInfo = NULL; - } else { - spd->colIdxInfo = calloc(spd->numOfBound, sizeof(SBoundIdxInfo)); - if (spd->colIdxInfo == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - SBoundIdxInfo* pColIdx = spd->colIdxInfo; - for (uint16_t i = 0; i < spd->numOfBound; ++i) { - pColIdx[i].schemaColIdx = (uint16_t)spd->boundedColumns[i]; - pColIdx[i].boundIdx = i; - } - qsort(pColIdx, spd->numOfBound, sizeof(SBoundIdxInfo), schemaIdxCompar); - for (uint16_t i = 0; i < spd->numOfBound; ++i) { - pColIdx[i].finalIdx = i; - } - qsort(pColIdx, spd->numOfBound, sizeof(SBoundIdxInfo), boundIdxCompar); - } - - return TSDB_CODE_SUCCESS; -} - -/** - * Refactor the raw payload structure to K-V format as the in tsParseOneRow() - */ -int32_t fillTablesPayload(SSqlObj* pSql) { - SSqlCmd* pCmd = &pSql->cmd; - int code = TSDB_CODE_SUCCESS; - - STableDataBlocks** p = taosHashIterate(pCmd->insertParam.pTableBlockHashList, NULL); - - STableDataBlocks* pOneTableBlock = *p; - while (pOneTableBlock) { - SSubmitBlk* pBlocks = (SSubmitBlk*)pOneTableBlock->pData; - - if (pBlocks->numOfRows > 0) { - initSMemRowHelper(&pOneTableBlock->rowHelper, tscGetTableSchema(pOneTableBlock->pTableMeta), - tscGetNumOfColumns(pOneTableBlock->pTableMeta), 0); - if ((code = initPayloadEnv(pOneTableBlock, pBlocks->numOfRows)) != TSDB_CODE_SUCCESS) { - return code; - } - if ((code = refactorPayload(pOneTableBlock, pBlocks->numOfRows)) != TSDB_CODE_SUCCESS) { - return code; - }; - } - - p = taosHashIterate(pCmd->insertParam.pTableBlockHashList, p); - if (p == NULL) { - break; - } - - pOneTableBlock = *p; - } - - return code; -} - //////////////////////////////////////////////////////////////////////////////// - // functions for insertion statement preparation - static int doBindParam(STableDataBlocks* pBlock, char* data, SParamInfo* param, TAOS_BIND* bind, int32_t colNum) { - if (bind->is_null != NULL && *(bind->is_null)) { - setNull(data + param->offset, param->type, param->bytes); - return TSDB_CODE_SUCCESS; - } - #if 0 if (0) { // allow user bind param data with different type @@ -1309,12 +1106,9 @@ static int insertStmtExecute(STscStmt* stmt) { pBlk->uid = pTableMeta->id.uid; pBlk->tid = pTableMeta->id.tid; - int code = fillTablesPayload(stmt->pSql); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + fillTablesColumnsNull(stmt->pSql); - code = tscMergeTableDataBlocks(&stmt->pSql->cmd.insertParam, false); + int code = tscMergeTableDataBlocks(&stmt->pSql->cmd.insertParam, false); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1391,7 +1185,7 @@ static int insertBatchStmtExecute(STscStmt* pStmt) { return TSDB_CODE_TSC_APP_ERROR; } - fillTablesPayload(pStmt->pSql); + fillTablesColumnsNull(pStmt->pSql); if ((code = tscMergeTableDataBlocks(&pStmt->pSql->cmd.insertParam, false)) != TSDB_CODE_SUCCESS) { return code; @@ -2012,6 +1806,7 @@ int taos_stmt_execute(TAOS_STMT* stmt) { pStmt->last = STMT_EXECUTE; + pStmt->pSql->cmd.insertParam.payloadType = PAYLOAD_TYPE_RAW; if (pStmt->multiTbInsert) { ret = insertBatchStmtExecute(pStmt); } else { diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 2c82f767bd..26e3a8915d 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1860,7 +1860,8 @@ static SMemRow tdGenMemRowFromBuilder(SMemRowBuilder* pBuilder) { } // Erase the empty space reserved for binary data -static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, bool includeSchema, SBlockKeyTuple *blkKeyTuple) { +static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SInsertStatementParam* insertParam, + SBlockKeyTuple* blkKeyTuple) { // TODO: optimize this function, handle the case while binary is not presented STableMeta* pTableMeta = pTableDataBlock->pTableMeta; STableComInfo tinfo = tscGetTableInfo(pTableMeta); @@ -1873,7 +1874,7 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, bo int32_t flen = 0; // original total length of row // schema needs to be included into the submit data block - if (includeSchema) { + if (insertParam->schemaAttached) { int32_t numOfCols = tscGetNumOfColumns(pTableDataBlock->pTableMeta); for(int32_t j = 0; j < numOfCols; ++j) { STColumn* pCol = (STColumn*) pDataBlock; @@ -1900,18 +1901,38 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, bo pBlock->dataLen = 0; int32_t numOfRows = htons(pBlock->numOfRows); - SMemRowBuilder rowBuilder; - rowBuilder.pSchema = pSchema; - rowBuilder.sversion = pTableMeta->sversion; - rowBuilder.flen = flen; - rowBuilder.nCols = tinfo.numOfColumns; - rowBuilder.pDataBlock = pDataBlock; - rowBuilder.pSubmitBlk = pBlock; - rowBuilder.buf = p; + if (IS_RAW_PAYLOAD(insertParam->payloadType)) { + for (int32_t i = 0; i < numOfRows; ++i) { + SMemRow memRow = (SMemRow)pDataBlock; + memRowSetType(memRow, SMEM_ROW_DATA); + SDataRow trow = memRowDataBody(memRow); + dataRowSetLen(trow, (uint16_t)(TD_DATA_ROW_HEAD_SIZE + flen)); + dataRowSetVersion(trow, pTableMeta->sversion); - for (int32_t i = 0; i < numOfRows; ++i) { - rowBuilder.buf = (blkKeyTuple + i)->payloadAddr; - tdGenMemRowFromBuilder(&rowBuilder); + int toffset = 0; + for (int32_t j = 0; j < tinfo.numOfColumns; j++) { + tdAppendColVal(trow, p, pSchema[j].type, toffset); + toffset += TYPE_BYTES[pSchema[j].type]; + p += pSchema[j].bytes; + } + + pDataBlock = (char*)pDataBlock + memRowTLen(memRow); + pBlock->dataLen += memRowTLen(memRow); + } + } else { + SMemRowBuilder rowBuilder; + rowBuilder.pSchema = pSchema; + rowBuilder.sversion = pTableMeta->sversion; + rowBuilder.flen = flen; + rowBuilder.nCols = tinfo.numOfColumns; + rowBuilder.pDataBlock = pDataBlock; + rowBuilder.pSubmitBlk = pBlock; + rowBuilder.buf = p; + + for (int32_t i = 0; i < numOfRows; ++i) { + rowBuilder.buf = (blkKeyTuple + i)->payloadAddr; + tdGenMemRowFromBuilder(&rowBuilder); + } } int32_t len = pBlock->dataLen + pBlock->schemaLen; @@ -1959,6 +1980,7 @@ static void extractTableNameList(SInsertStatementParam *pInsertParam, bool freeB int32_t tscMergeTableDataBlocks(SInsertStatementParam *pInsertParam, bool freeBlockMap) { const int INSERT_HEAD_SIZE = sizeof(SMsgDesc) + sizeof(SSubmitMsg); int code = 0; + bool isRawPayload = IS_RAW_PAYLOAD(pInsertParam->payloadType); void* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); SArray* pVnodeDataBlockList = taosArrayInit(8, POINTER_BYTES); @@ -1967,7 +1989,7 @@ int32_t tscMergeTableDataBlocks(SInsertStatementParam *pInsertParam, bool freeBl STableDataBlocks* pOneTableBlock = *p; SBlockKeyInfo blkKeyInfo = {0}; // share by pOneTableBlock - + while(pOneTableBlock) { SSubmitBlk* pBlocks = (SSubmitBlk*) pOneTableBlock->pData; if (pBlocks->numOfRows > 0) { @@ -2008,21 +2030,29 @@ int32_t tscMergeTableDataBlocks(SInsertStatementParam *pInsertParam, bool freeBl } } - if ((code = tscSortRemoveDataBlockDupRows(pOneTableBlock, &blkKeyInfo)) != 0) { - taosHashCleanup(pVnodeDataBlockHashList); - tscDestroyBlockArrayList(pVnodeDataBlockList); - tfree(dataBuf->pData); - tfree(blkKeyInfo.pKeyTuple); - return code; + if (isRawPayload) { + tscSortRemoveDataBlockDupRowsRaw(pOneTableBlock); + char* ekey = (char*)pBlocks->data + pOneTableBlock->rowSize * (pBlocks->numOfRows - 1); + + tscDebug("0x%" PRIx64 " name:%s, tid:%d rows:%d sversion:%d skey:%" PRId64 ", ekey:%" PRId64, + pInsertParam->objectId, tNameGetTableName(&pOneTableBlock->tableName), pBlocks->tid, + pBlocks->numOfRows, pBlocks->sversion, GET_INT64_VAL(pBlocks->data), GET_INT64_VAL(ekey)); + } else { + if ((code = tscSortRemoveDataBlockDupRows(pOneTableBlock, &blkKeyInfo)) != 0) { + taosHashCleanup(pVnodeDataBlockHashList); + tscDestroyBlockArrayList(pVnodeDataBlockList); + tfree(dataBuf->pData); + tfree(blkKeyInfo.pKeyTuple); + return code; + } + ASSERT(blkKeyInfo.pKeyTuple != NULL && pBlocks->numOfRows > 0); + + SBlockKeyTuple* pLastKeyTuple = blkKeyInfo.pKeyTuple + pBlocks->numOfRows - 1; + tscDebug("0x%" PRIx64 " name:%s, tid:%d rows:%d sversion:%d skey:%" PRId64 ", ekey:%" PRId64, + pInsertParam->objectId, tNameGetTableName(&pOneTableBlock->tableName), pBlocks->tid, + pBlocks->numOfRows, pBlocks->sversion, blkKeyInfo.pKeyTuple->skey, pLastKeyTuple->skey); } - - ASSERT(blkKeyInfo.pKeyTuple != NULL && pBlocks->numOfRows > 0); - - SBlockKeyTuple* pLastKeyTuple = blkKeyInfo.pKeyTuple + pBlocks->numOfRows - 1; - tscDebug("0x%" PRIx64 " name:%s, tid:%d rows:%d sversion:%d skey:%" PRId64 ", ekey:%" PRId64, - pInsertParam->objectId, tNameGetTableName(&pOneTableBlock->tableName), pBlocks->tid, pBlocks->numOfRows, - pBlocks->sversion, blkKeyInfo.pKeyTuple->skey, pLastKeyTuple->skey); - + int32_t len = pBlocks->numOfRows * (pOneTableBlock->rowSize + expandSize) + sizeof(STColumn) * tscGetNumOfColumns(pOneTableBlock->pTableMeta); pBlocks->tid = htonl(pBlocks->tid); @@ -2032,7 +2062,7 @@ int32_t tscMergeTableDataBlocks(SInsertStatementParam *pInsertParam, bool freeBl pBlocks->schemaLen = 0; // erase the empty space reserved for binary data - int32_t finalLen = trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock, pInsertParam->schemaAttached, blkKeyInfo.pKeyTuple); + int32_t finalLen = trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock, pInsertParam, blkKeyInfo.pKeyTuple); assert(finalLen <= len); dataBuf->size += (finalLen + sizeof(SSubmitBlk)); From 78f58194c44201a4dd362c7f0425133ecd1c20f0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 19 Jul 2021 17:55:42 +0800 Subject: [PATCH 62/67] [td-5404]: fix memory leak in union query. --- src/client/src/tscSQLParser.c | 29 +++++++++++++++-- src/client/src/tscUtil.c | 57 +++++++++++++++------------------- src/util/CMakeLists.txt | 2 +- src/util/inc/tarray.h | 7 +++++ src/util/src/tarray.c | 40 ++++++++++++++++++++++++ tests/script/sh/prepare_udf.sh | 1 + 6 files changed, 101 insertions(+), 35 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 77c37239da..f82e1b62e9 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -8005,6 +8005,28 @@ static void freeElem(void* p) { tfree(*(char**)p); } +int32_t tnameComparFn(const void* p1, const void* p2) { + SName* pn1 = (SName*)p1; + SName* pn2 = (SName*)p2; + + int32_t ret = strncmp(pn1->acctId, pn2->acctId, tListLen(pn1->acctId)); + if (ret != 0) { + return ret > 0? 1:-1; + } else { + ret = strncmp(pn1->dbname, pn2->dbname, tListLen(pn1->dbname)); + if (ret != 0) { + return ret > 0? 1:-1; + } else { + ret = strncmp(pn1->tname, pn2->tname, tListLen(pn1->tname)); + if (ret != 0) { + return ret > 0? 1:-1; + } else { + return 0; + } + } + } +} + int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo) { SSqlCmd* pCmd = &pSql->cmd; @@ -8055,6 +8077,9 @@ int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo) { plist = taosArrayInit(4, POINTER_BYTES); pVgroupList = taosArrayInit(4, POINTER_BYTES); + taosArraySort(tableNameList, tnameComparFn); + taosArrayRemoveDuplicate(tableNameList, tnameComparFn, NULL); + size_t numOfTables = taosArrayGetSize(tableNameList); for (int32_t i = 0; i < numOfTables; ++i) { SName* pname = taosArrayGet(tableNameList, i); @@ -8075,8 +8100,7 @@ int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo) { continue; } } else if (pTableMeta->tableType == TSDB_SUPER_TABLE) { - // the vgroup list of a super table is not kept in local buffer, so here need retrieve it - // from the mnode each time + // the vgroup list of super table is not kept in local buffer, so here need retrieve it from the mnode each time char* t = strdup(name); taosArrayPush(pVgroupList, &t); } @@ -8103,6 +8127,7 @@ int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo) { if (pInfo->funcs) { funcSize = taosArrayGetSize(pInfo->funcs); } + if (funcSize > 0) { for (size_t i = 0; i < funcSize; ++i) { SStrToken* t = taosArrayGet(pInfo->funcs, i); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 6ae0249329..5b291c6e1f 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1315,8 +1315,7 @@ void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeMeta) { return; } - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - + SQueryInfo* pQueryInfo = pCmd->pQueryInfo; while(pQueryInfo != NULL) { SQueryInfo* p = pQueryInfo->sibling; @@ -1596,7 +1595,7 @@ void freeUdfInfo(SUdfInfo* pUdfInfo) { taosCloseDll(pUdfInfo->handle); } - +// todo refactor void* tscDestroyUdfArrayList(SArray* pUdfList) { if (pUdfList == NULL) { return NULL; @@ -2986,6 +2985,7 @@ void tscInitQueryInfo(SQueryInfo* pQueryInfo) { int32_t tscAddQueryInfo(SSqlCmd* pCmd) { assert(pCmd != NULL); SQueryInfo* pQueryInfo = calloc(1, sizeof(SQueryInfo)); + if (pQueryInfo == NULL) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -3968,17 +3968,21 @@ void tscTryQueryNextClause(SSqlObj* pSql, __async_cb_func_t fp) { //backup the total number of result first int64_t num = pRes->numOfTotal + pRes->numOfClauseTotal; - // DON't free final since it may be recoreded and used later in APP TAOS_FIELD* finalBk = pRes->final; pRes->final = NULL; tscFreeSqlResult(pSql); + pRes->final = finalBk; - pRes->numOfTotal = num; - + + for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) { + taos_free_result(pSql->pSubs[i]); + } + tfree(pSql->pSubs); pSql->subState.numOfSub = 0; + pSql->fp = fp; tscDebug("0x%"PRIx64" try data in the next subclause", pSql->self); @@ -4618,6 +4622,20 @@ int32_t nameComparFn(const void* n1, const void* n2) { } } +static void freeContent(void* p) { + char* ptr = *(char**)p; + tfree(ptr); +} + +static int32_t contCompare(const void* p1, const void* p2) { + int32_t ret = strcmp(p1, p2); + if (ret == 0) { + return 0; + } else { + return ret > 0 ? 1:-1; + } +} + int tscTransferTableNameList(SSqlObj *pSql, const char *pNameList, int32_t length, SArray* pNameArray) { SSqlCmd *pCmd = &pSql->cmd; @@ -4665,32 +4683,7 @@ int tscTransferTableNameList(SSqlObj *pSql, const char *pNameList, int32_t lengt } taosArraySort(pNameArray, nameComparFn); - - int32_t pos = 0; - for(int32_t i = 1; i < len; ++i) { - char** p1 = taosArrayGet(pNameArray, pos); - char** p2 = taosArrayGet(pNameArray, i); - - if (strcmp(*p1, *p2) == 0) { - // do nothing - } else { - if (pos + 1 != i) { - char* p = taosArrayGetP(pNameArray, pos + 1); - tfree(p); - taosArraySet(pNameArray, pos + 1, p2); - pos += 1; - } else { - pos += 1; - } - } - } - - for(int32_t i = pos + 1; i < pNameArray->size; ++i) { - char* p = taosArrayGetP(pNameArray, i); - tfree(p); - } - - pNameArray->size = pos + 1; + taosArrayRemoveDuplicate(pNameArray, contCompare, freeContent); return TSDB_CODE_SUCCESS; } diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index 27d9e11053..44808b1025 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -11,7 +11,7 @@ TARGET_LINK_LIBRARIES(tutil pthread os lz4 z rmonotonic) IF (TD_LINUX) TARGET_LINK_LIBRARIES(tutil m rt) -# ADD_SUBDIRECTORY(tests) + ADD_SUBDIRECTORY(tests) FIND_PATH(ICONV_INCLUDE_EXIST iconv.h /usr/include/ /usr/local/include/) IF (ICONV_INCLUDE_EXIST) diff --git a/src/util/inc/tarray.h b/src/util/inc/tarray.h index 97aeaa61d4..2da74eac82 100644 --- a/src/util/inc/tarray.h +++ b/src/util/inc/tarray.h @@ -60,6 +60,13 @@ void *taosArrayAddBatch(SArray *pArray, const void *pData, int nEles); */ void taosArrayRemoveBatch(SArray *pArray, const int32_t* pData, int32_t numOfElems); +/** + * + * @param pArray + * @param comparFn + * @param fp + */ +void taosArrayRemoveDuplicate(SArray *pArray, __compar_fn_t comparFn, void (*fp)(void*)); /** * add all element from the source array list into the destination diff --git a/src/util/src/tarray.c b/src/util/src/tarray.c index d25d547be1..d0d126c1e4 100644 --- a/src/util/src/tarray.c +++ b/src/util/src/tarray.c @@ -124,6 +124,46 @@ void taosArrayRemoveBatch(SArray *pArray, const int32_t* pData, int32_t numOfEle pArray->size -= numOfElems; } +void taosArrayRemoveDuplicate(SArray *pArray, __compar_fn_t comparFn, void (*fp)(void*)) { + assert(pArray); + + size_t size = pArray->size; + if (size <= 1) { + return; + } + + int32_t pos = 0; + for(int32_t i = 1; i < size; ++i) { + char* p1 = taosArrayGet(pArray, pos); + char* p2 = taosArrayGet(pArray, i); + + if (comparFn(p1, p2) == 0) { + // do nothing + } else { + if (pos + 1 != i) { + void* p = taosArrayGet(pArray, pos + 1); + if (fp != NULL) { + fp(p); + } + + taosArraySet(pArray, pos + 1, p2); + pos += 1; + } else { + pos += 1; + } + } + } + + if (fp != NULL) { + for(int32_t i = pos + 1; i < pArray->size; ++i) { + void* p = taosArrayGet(pArray, i); + fp(p); + } + } + + pArray->size = pos + 1; +} + void* taosArrayAddAll(SArray* pArray, const SArray* pInput) { return taosArrayAddBatch(pArray, pInput->pData, (int32_t) taosArrayGetSize(pInput)); } diff --git a/tests/script/sh/prepare_udf.sh b/tests/script/sh/prepare_udf.sh index ecfc53660e..a856b96c98 100644 --- a/tests/script/sh/prepare_udf.sh +++ b/tests/script/sh/prepare_udf.sh @@ -5,6 +5,7 @@ echo -n "" > /tmp/empty dd if=/dev/zero bs=3584 of=/tmp/big count=1 rm -rf /tmp/sum_double.so /tmp/add_one.so +touch /tmp/normal gcc -g -O0 -fPIC -shared sh/sum_double.c -o /tmp/sum_double.so gcc -g -O0 -fPIC -shared sh/add_one.c -o /tmp/add_one.so From 5bea3eabe55af4b9630e6b0d4d4d8587f1251fa1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 19 Jul 2021 22:58:54 +0800 Subject: [PATCH 63/67] [td-225]fix compiler error. --- src/tsdb/src/tsdbRead.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 6c16c72c2a..a68a170554 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -3098,7 +3098,7 @@ STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList) { assert(totalNumOfTable == 0 && taosArrayGetSize(groupList->pGroupList) == numOfGroups); } - taosArrayRemoveBatch(groupList->pGroupList, TARRAY_GET_START(emptyGroup), taosArrayGetSize(emptyGroup)); + taosArrayRemoveBatch(groupList->pGroupList, TARRAY_GET_START(emptyGroup), (int32_t) taosArrayGetSize(emptyGroup)); taosArrayDestroy(emptyGroup); groupList->numOfTables = totalNumOfTable; From e8b3fde4dc50a759d935dd81b16edf4cc7587ffa Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 20 Jul 2021 10:14:55 +0800 Subject: [PATCH 64/67] [TD-5398]: taosdemo 4096 columns table creation failed. (#6930) * [TD-5398]: taosdemo 4096 columns table creation failed. * change column name prefix from col to c for maxium column number to fix buffer length. * fix test case for column name be shorter. * use accurate buffer length. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 392 +++++++++--------- tests/pytest/query/query1970YearsAf.py | 8 +- .../TD-4985/query-limit-offset.py | 54 +-- .../TD-5213/insertSigcolumnsNum4096.py | 38 +- .../taosdemoTestInsertWithJson.py | 154 +++---- tests/pytest/tools/taosdemoTest.py | 4 +- tests/pytest/wal/sdbComp.py | 36 +- 7 files changed, 344 insertions(+), 342 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 964f1a1d0f..2beb0c8e7e 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -71,16 +71,15 @@ extern char configDir[]; #define HEAD_BUFF_LEN TSDB_MAX_COLUMNS*24 // 16*MAX_COLUMNS + (192+32)*2 + insert into .. -#define MAX_SQL_SIZE 65536 -#define BUFFER_SIZE (65536*2) -#define COND_BUF_LEN (BUFFER_SIZE - 30) +#define COL_BUFFER_LEN (TSDB_MAX_BYTES_PER_ROW - 50) +#define BUFFER_SIZE (50 + TSDB_DB_NAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_MAX_BYTES_PER_ROW + TSDB_MAX_TAGS_LEN) +#define COND_BUF_LEN (BUFFER_SIZE - 30) #define MAX_USERNAME_SIZE 64 #define MAX_PASSWORD_SIZE 64 #define MAX_HOSTNAME_SIZE 64 #define MAX_TB_NAME_SIZE 64 #define MAX_DATA_SIZE (16*TSDB_MAX_COLUMNS)+20 // max record len: 16*MAX_COLUMNS, timestamp string and ,('') need extra space #define OPT_ABORT 1 /* –abort */ -#define STRING_LEN 60000 #define MAX_PREPARED_RAND 1000000 #define MAX_FILE_NAME_LEN 256 // max file name length on linux is 255. @@ -2707,7 +2706,7 @@ static int createSuperTable( char command[BUFFER_SIZE] = "\0"; - char cols[STRING_LEN] = "\0"; + char cols[COL_BUFFER_LEN] = "\0"; int colIndex; int len = 0; @@ -2723,55 +2722,55 @@ static int createSuperTable( char* dataType = superTbl->columns[colIndex].dataType; if (strcasecmp(dataType, "BINARY") == 0) { - len += snprintf(cols + len, STRING_LEN - len, - ", col%d %s(%d)", colIndex, "BINARY", + len += snprintf(cols + len, COL_BUFFER_LEN - len, + ", C%d %s(%d)", colIndex, "BINARY", superTbl->columns[colIndex].dataLen); lenOfOneRow += superTbl->columns[colIndex].dataLen + 3; } else if (strcasecmp(dataType, "NCHAR") == 0) { - len += snprintf(cols + len, STRING_LEN - len, - ", col%d %s(%d)", colIndex, "NCHAR", + len += snprintf(cols + len, COL_BUFFER_LEN - len, + ", C%d %s(%d)", colIndex, "NCHAR", superTbl->columns[colIndex].dataLen); lenOfOneRow += superTbl->columns[colIndex].dataLen + 3; } else if (strcasecmp(dataType, "INT") == 0) { if ((g_args.demo_mode) && (colIndex == 1)) { - len += snprintf(cols + len, STRING_LEN - len, + len += snprintf(cols + len, COL_BUFFER_LEN - len, ", VOLTAGE INT"); } else { - len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "INT"); + len += snprintf(cols + len, COL_BUFFER_LEN - len, ", C%d %s", colIndex, "INT"); } lenOfOneRow += 11; } else if (strcasecmp(dataType, "BIGINT") == 0) { - len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", + len += snprintf(cols + len, COL_BUFFER_LEN - len, ", C%d %s", colIndex, "BIGINT"); lenOfOneRow += 21; } else if (strcasecmp(dataType, "SMALLINT") == 0) { - len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", + len += snprintf(cols + len, COL_BUFFER_LEN - len, ", C%d %s", colIndex, "SMALLINT"); lenOfOneRow += 6; } else if (strcasecmp(dataType, "TINYINT") == 0) { - len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "TINYINT"); + len += snprintf(cols + len, COL_BUFFER_LEN - len, ", C%d %s", colIndex, "TINYINT"); lenOfOneRow += 4; } else if (strcasecmp(dataType, "BOOL") == 0) { - len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "BOOL"); + len += snprintf(cols + len, COL_BUFFER_LEN - len, ", C%d %s", colIndex, "BOOL"); lenOfOneRow += 6; } else if (strcasecmp(dataType, "FLOAT") == 0) { if (g_args.demo_mode) { if (colIndex == 0) { - len += snprintf(cols + len, STRING_LEN - len, ", CURRENT FLOAT"); + len += snprintf(cols + len, COL_BUFFER_LEN - len, ", CURRENT FLOAT"); } else if (colIndex == 2) { - len += snprintf(cols + len, STRING_LEN - len, ", PHASE FLOAT"); + len += snprintf(cols + len, COL_BUFFER_LEN - len, ", PHASE FLOAT"); } } else { - len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "FLOAT"); + len += snprintf(cols + len, COL_BUFFER_LEN - len, ", C%d %s", colIndex, "FLOAT"); } lenOfOneRow += 22; } else if (strcasecmp(dataType, "DOUBLE") == 0) { - len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", + len += snprintf(cols + len, COL_BUFFER_LEN - len, ", C%d %s", colIndex, "DOUBLE"); lenOfOneRow += 42; } else if (strcasecmp(dataType, "TIMESTAMP") == 0) { - len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", + len += snprintf(cols + len, COL_BUFFER_LEN - len, ", C%d %s", colIndex, "TIMESTAMP"); lenOfOneRow += 21; } else { @@ -2803,60 +2802,63 @@ static int createSuperTable( return -1; } - char tags[STRING_LEN] = "\0"; + char tags[TSDB_MAX_TAGS_LEN] = "\0"; int tagIndex; len = 0; int lenOfTagOfOneRow = 0; - len += snprintf(tags + len, STRING_LEN - len, "("); + len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, "("); for (tagIndex = 0; tagIndex < superTbl->tagCount; tagIndex++) { char* dataType = superTbl->tags[tagIndex].dataType; if (strcasecmp(dataType, "BINARY") == 0) { if ((g_args.demo_mode) && (tagIndex == 1)) { - len += snprintf(tags + len, STRING_LEN - len, + len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, "location BINARY(%d), ", superTbl->tags[tagIndex].dataLen); } else { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", - tagIndex, "BINARY", superTbl->tags[tagIndex].dataLen); + len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, + "t%d %s(%d), ", tagIndex, "BINARY", + superTbl->tags[tagIndex].dataLen); } lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 3; } else if (strcasecmp(dataType, "NCHAR") == 0) { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex, + len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, + "t%d %s(%d), ", tagIndex, "NCHAR", superTbl->tags[tagIndex].dataLen); lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 3; } else if (strcasecmp(dataType, "INT") == 0) { if ((g_args.demo_mode) && (tagIndex == 0)) { - len += snprintf(tags + len, STRING_LEN - len, "groupId INT, "); + len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, + "groupId INT, "); } else { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, - "INT"); + len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, + "t%d %s, ", tagIndex, "INT"); } lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 11; } else if (strcasecmp(dataType, "BIGINT") == 0) { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, - "BIGINT"); + len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, + "t%d %s, ", tagIndex, "BIGINT"); lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 21; } else if (strcasecmp(dataType, "SMALLINT") == 0) { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, - "SMALLINT"); + len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, + "t%d %s, ", tagIndex, "SMALLINT"); lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 6; } else if (strcasecmp(dataType, "TINYINT") == 0) { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, - "TINYINT"); + len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, + "t%d %s, ", tagIndex, "TINYINT"); lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 4; } else if (strcasecmp(dataType, "BOOL") == 0) { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, - "BOOL"); + len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, + "t%d %s, ", tagIndex, "BOOL"); lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 6; } else if (strcasecmp(dataType, "FLOAT") == 0) { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, - "FLOAT"); + len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, + "t%d %s, ", tagIndex, "FLOAT"); lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 22; } else if (strcasecmp(dataType, "DOUBLE") == 0) { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, - "DOUBLE"); + len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, + "t%d %s, ", tagIndex, "DOUBLE"); lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 42; } else { taos_close(taos); @@ -2867,7 +2869,7 @@ static int createSuperTable( } len -= 2; - len += snprintf(tags + len, STRING_LEN - len, ")"); + len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, ")"); superTbl->lenOfTagOfOneRow = lenOfTagOfOneRow; @@ -3020,175 +3022,175 @@ static int createDatabasesAndStables() { static void* createTable(void *sarg) { - threadInfo *pThreadInfo = (threadInfo *)sarg; - SSuperTable* superTblInfo = pThreadInfo->superTblInfo; + threadInfo *pThreadInfo = (threadInfo *)sarg; + SSuperTable* superTblInfo = pThreadInfo->superTblInfo; - uint64_t lastPrintTime = taosGetTimestampMs(); + uint64_t lastPrintTime = taosGetTimestampMs(); - int buff_len; - buff_len = BUFFER_SIZE / 8; + int buff_len; + buff_len = BUFFER_SIZE; - pThreadInfo->buffer = calloc(buff_len, 1); - if (pThreadInfo->buffer == NULL) { - errorPrint("%s() LN%d, Memory allocated failed!\n", __func__, __LINE__); - exit(-1); - } - - int len = 0; - int batchNum = 0; - - verbosePrint("%s() LN%d: Creating table from %"PRIu64" to %"PRIu64"\n", - __func__, __LINE__, - pThreadInfo->start_table_from, pThreadInfo->end_table_to); - - for (uint64_t i = pThreadInfo->start_table_from; - i <= pThreadInfo->end_table_to; i++) { - if (0 == g_Dbs.use_metric) { - snprintf(pThreadInfo->buffer, buff_len, - "create table if not exists %s.%s%"PRIu64" %s;", - pThreadInfo->db_name, - g_args.tb_prefix, i, - pThreadInfo->cols); - } else { - if (superTblInfo == NULL) { - errorPrint("%s() LN%d, use metric, but super table info is NULL\n", - __func__, __LINE__); - free(pThreadInfo->buffer); + pThreadInfo->buffer = calloc(buff_len, 1); + if (pThreadInfo->buffer == NULL) { + errorPrint("%s() LN%d, Memory allocated failed!\n", __func__, __LINE__); exit(-1); - } else { - if (0 == len) { - batchNum = 0; - memset(pThreadInfo->buffer, 0, buff_len); - len += snprintf(pThreadInfo->buffer + len, - buff_len - len, "create table "); - } - char* tagsValBuf = NULL; - if (0 == superTblInfo->tagSource) { - tagsValBuf = generateTagVaulesForStb(superTblInfo, i); + } + + int len = 0; + int batchNum = 0; + + verbosePrint("%s() LN%d: Creating table from %"PRIu64" to %"PRIu64"\n", + __func__, __LINE__, + pThreadInfo->start_table_from, pThreadInfo->end_table_to); + + for (uint64_t i = pThreadInfo->start_table_from; + i <= pThreadInfo->end_table_to; i++) { + if (0 == g_Dbs.use_metric) { + snprintf(pThreadInfo->buffer, buff_len, + "create table if not exists %s.%s%"PRIu64" %s;", + pThreadInfo->db_name, + g_args.tb_prefix, i, + pThreadInfo->cols); } else { - tagsValBuf = getTagValueFromTagSample( - superTblInfo, - i % superTblInfo->tagSampleCount); + if (superTblInfo == NULL) { + errorPrint("%s() LN%d, use metric, but super table info is NULL\n", + __func__, __LINE__); + free(pThreadInfo->buffer); + exit(-1); + } else { + if (0 == len) { + batchNum = 0; + memset(pThreadInfo->buffer, 0, buff_len); + len += snprintf(pThreadInfo->buffer + len, + buff_len - len, "create table "); + } + char* tagsValBuf = NULL; + if (0 == superTblInfo->tagSource) { + tagsValBuf = generateTagVaulesForStb(superTblInfo, i); + } else { + tagsValBuf = getTagValueFromTagSample( + superTblInfo, + i % superTblInfo->tagSampleCount); + } + if (NULL == tagsValBuf) { + free(pThreadInfo->buffer); + return NULL; + } + len += snprintf(pThreadInfo->buffer + len, + buff_len - len, + "if not exists %s.%s%"PRIu64" using %s.%s tags %s ", + pThreadInfo->db_name, superTblInfo->childTblPrefix, + i, pThreadInfo->db_name, + superTblInfo->sTblName, tagsValBuf); + free(tagsValBuf); + batchNum++; + if ((batchNum < superTblInfo->batchCreateTableNum) + && ((buff_len - len) + >= (superTblInfo->lenOfTagOfOneRow + 256))) { + continue; + } + } } - if (NULL == tagsValBuf) { - free(pThreadInfo->buffer); - return NULL; + + len = 0; + if (0 != queryDbExec(pThreadInfo->taos, pThreadInfo->buffer, + NO_INSERT_TYPE, false)){ + errorPrint( "queryDbExec() failed. buffer:\n%s\n", pThreadInfo->buffer); + free(pThreadInfo->buffer); + return NULL; } - len += snprintf(pThreadInfo->buffer + len, - buff_len - len, - "if not exists %s.%s%"PRIu64" using %s.%s tags %s ", - pThreadInfo->db_name, superTblInfo->childTblPrefix, - i, pThreadInfo->db_name, - superTblInfo->sTblName, tagsValBuf); - free(tagsValBuf); - batchNum++; - if ((batchNum < superTblInfo->batchCreateTableNum) - && ((buff_len - len) - >= (superTblInfo->lenOfTagOfOneRow + 256))) { - continue; + + uint64_t currentPrintTime = taosGetTimestampMs(); + if (currentPrintTime - lastPrintTime > 30*1000) { + printf("thread[%d] already create %"PRIu64" - %"PRIu64" tables\n", + pThreadInfo->threadID, pThreadInfo->start_table_from, i); + lastPrintTime = currentPrintTime; } - } } - len = 0; - if (0 != queryDbExec(pThreadInfo->taos, pThreadInfo->buffer, - NO_INSERT_TYPE, false)){ - errorPrint( "queryDbExec() failed. buffer:\n%s\n", pThreadInfo->buffer); - free(pThreadInfo->buffer); - return NULL; + if (0 != len) { + if (0 != queryDbExec(pThreadInfo->taos, pThreadInfo->buffer, + NO_INSERT_TYPE, false)) { + errorPrint( "queryDbExec() failed. buffer:\n%s\n", pThreadInfo->buffer); + } } - uint64_t currentPrintTime = taosGetTimestampMs(); - if (currentPrintTime - lastPrintTime > 30*1000) { - printf("thread[%d] already create %"PRIu64" - %"PRIu64" tables\n", - pThreadInfo->threadID, pThreadInfo->start_table_from, i); - lastPrintTime = currentPrintTime; - } - } - - if (0 != len) { - if (0 != queryDbExec(pThreadInfo->taos, pThreadInfo->buffer, - NO_INSERT_TYPE, false)) { - errorPrint( "queryDbExec() failed. buffer:\n%s\n", pThreadInfo->buffer); - } - } - - free(pThreadInfo->buffer); - return NULL; + free(pThreadInfo->buffer); + return NULL; } static int startMultiThreadCreateChildTable( char* cols, int threads, uint64_t tableFrom, int64_t ntables, char* db_name, SSuperTable* superTblInfo) { - pthread_t *pids = calloc(1, threads * sizeof(pthread_t)); - threadInfo *infos = calloc(1, threads * sizeof(threadInfo)); + pthread_t *pids = calloc(1, threads * sizeof(pthread_t)); + threadInfo *infos = calloc(1, threads * sizeof(threadInfo)); - if ((NULL == pids) || (NULL == infos)) { - printf("malloc failed\n"); - exit(-1); - } - - if (threads < 1) { - threads = 1; - } - - int64_t a = ntables / threads; - if (a < 1) { - threads = ntables; - a = 1; - } - - int64_t b = 0; - b = ntables % threads; - - for (int64_t i = 0; i < threads; i++) { - threadInfo *pThreadInfo = infos + i; - pThreadInfo->threadID = i; - tstrncpy(pThreadInfo->db_name, db_name, TSDB_DB_NAME_LEN); - pThreadInfo->superTblInfo = superTblInfo; - verbosePrint("%s() %d db_name: %s\n", __func__, __LINE__, db_name); - pThreadInfo->taos = taos_connect( - g_Dbs.host, - g_Dbs.user, - g_Dbs.password, - db_name, - g_Dbs.port); - if (pThreadInfo->taos == NULL) { - errorPrint( "%s() LN%d, Failed to connect to TDengine, reason:%s\n", - __func__, __LINE__, taos_errstr(NULL)); - free(pids); - free(infos); - return -1; + if ((NULL == pids) || (NULL == infos)) { + printf("malloc failed\n"); + exit(-1); } - pThreadInfo->start_table_from = tableFrom; - pThreadInfo->ntables = iend_table_to = i < b ? tableFrom + a : tableFrom + a - 1; - tableFrom = pThreadInfo->end_table_to + 1; - pThreadInfo->use_metric = true; - pThreadInfo->cols = cols; - pThreadInfo->minDelay = UINT64_MAX; - pthread_create(pids + i, NULL, createTable, pThreadInfo); - } + if (threads < 1) { + threads = 1; + } - for (int i = 0; i < threads; i++) { - pthread_join(pids[i], NULL); - } + int64_t a = ntables / threads; + if (a < 1) { + threads = ntables; + a = 1; + } - for (int i = 0; i < threads; i++) { - threadInfo *pThreadInfo = infos + i; - taos_close(pThreadInfo->taos); - } + int64_t b = 0; + b = ntables % threads; - free(pids); - free(infos); + for (int64_t i = 0; i < threads; i++) { + threadInfo *pThreadInfo = infos + i; + pThreadInfo->threadID = i; + tstrncpy(pThreadInfo->db_name, db_name, TSDB_DB_NAME_LEN); + pThreadInfo->superTblInfo = superTblInfo; + verbosePrint("%s() %d db_name: %s\n", __func__, __LINE__, db_name); + pThreadInfo->taos = taos_connect( + g_Dbs.host, + g_Dbs.user, + g_Dbs.password, + db_name, + g_Dbs.port); + if (pThreadInfo->taos == NULL) { + errorPrint( "%s() LN%d, Failed to connect to TDengine, reason:%s\n", + __func__, __LINE__, taos_errstr(NULL)); + free(pids); + free(infos); + return -1; + } - return 0; + pThreadInfo->start_table_from = tableFrom; + pThreadInfo->ntables = iend_table_to = i < b ? tableFrom + a : tableFrom + a - 1; + tableFrom = pThreadInfo->end_table_to + 1; + pThreadInfo->use_metric = true; + pThreadInfo->cols = cols; + pThreadInfo->minDelay = UINT64_MAX; + pthread_create(pids + i, NULL, createTable, pThreadInfo); + } + + for (int i = 0; i < threads; i++) { + pthread_join(pids[i], NULL); + } + + for (int i = 0; i < threads; i++) { + threadInfo *pThreadInfo = infos + i; + taos_close(pThreadInfo->taos); + } + + free(pids); + free(infos); + + return 0; } static void createChildTables() { - char tblColsBuf[MAX_SQL_SIZE]; + char tblColsBuf[TSDB_MAX_BYTES_PER_ROW]; int len; for (int i = 0; i < g_Dbs.dbCount; i++) { @@ -3220,21 +3222,21 @@ static void createChildTables() { } } else { // normal table - len = snprintf(tblColsBuf, MAX_SQL_SIZE, "(TS TIMESTAMP"); + len = snprintf(tblColsBuf, TSDB_MAX_BYTES_PER_ROW, "(TS TIMESTAMP"); for (int j = 0; j < g_args.num_of_CPR; j++) { if ((strncasecmp(g_args.datatype[j], "BINARY", strlen("BINARY")) == 0) || (strncasecmp(g_args.datatype[j], "NCHAR", strlen("NCHAR")) == 0)) { - snprintf(tblColsBuf + len, MAX_SQL_SIZE - len, - ", COL%d %s(%d)", j, g_args.datatype[j], g_args.len_of_binary); + snprintf(tblColsBuf + len, TSDB_MAX_BYTES_PER_ROW - len, + ",C%d %s(%d)", j, g_args.datatype[j], g_args.len_of_binary); } else { - snprintf(tblColsBuf + len, MAX_SQL_SIZE - len, - ", COL%d %s", j, g_args.datatype[j]); + snprintf(tblColsBuf + len, TSDB_MAX_BYTES_PER_ROW - len, + ",C%d %s", j, g_args.datatype[j]); } len = strlen(tblColsBuf); } - snprintf(tblColsBuf + len, MAX_SQL_SIZE - len, ")"); + snprintf(tblColsBuf + len, TSDB_MAX_BYTES_PER_ROW - len, ")"); verbosePrint("%s() LN%d: dbName: %s num of tb: %"PRId64" schema: %s\n", __func__, __LINE__, @@ -7986,7 +7988,7 @@ static void initOfQueryMeta() { tstrncpy(g_queryInfo.password, TSDB_DEFAULT_PASS, MAX_PASSWORD_SIZE); } -static void setParaFromArg(){ +static void setParaFromArg() { if (g_args.host) { tstrncpy(g_Dbs.host, g_args.host, MAX_HOSTNAME_SIZE); } else { @@ -8022,10 +8024,10 @@ static void setParaFromArg(){ g_Dbs.do_aggreFunc = true; - char dataString[STRING_LEN]; + char dataString[TSDB_MAX_BYTES_PER_ROW]; char **data_type = g_args.datatype; - memset(dataString, 0, STRING_LEN); + memset(dataString, 0, TSDB_MAX_BYTES_PER_ROW); if (strcasecmp(data_type[0], "BINARY") == 0 || strcasecmp(data_type[0], "BOOL") == 0 @@ -8143,7 +8145,7 @@ static void querySqlFile(TAOS* taos, char* sqlFile) } int read_len = 0; - char * cmd = calloc(1, MAX_SQL_SIZE); + char * cmd = calloc(1, TSDB_MAX_BYTES_PER_ROW); size_t cmd_len = 0; char * line = NULL; size_t line_len = 0; @@ -8151,7 +8153,7 @@ static void querySqlFile(TAOS* taos, char* sqlFile) double t = taosGetTimestampMs(); while((read_len = tgetline(&line, &line_len, fp)) != -1) { - if (read_len >= MAX_SQL_SIZE) continue; + if (read_len >= TSDB_MAX_BYTES_PER_ROW) continue; line[--read_len] = '\0'; if (read_len == 0 || isCommentLine(line)) { // line starts with # @@ -8174,7 +8176,7 @@ static void querySqlFile(TAOS* taos, char* sqlFile) tmfclose(fp); return; } - memset(cmd, 0, MAX_SQL_SIZE); + memset(cmd, 0, TSDB_MAX_BYTES_PER_ROW); cmd_len = 0; } diff --git a/tests/pytest/query/query1970YearsAf.py b/tests/pytest/query/query1970YearsAf.py index 93404afd59..a365369b21 100644 --- a/tests/pytest/query/query1970YearsAf.py +++ b/tests/pytest/query/query1970YearsAf.py @@ -187,19 +187,19 @@ class TDTestCase: "select * from t9 where t9.ts > '1969-12-31 22:00:00.000' and t9.ts <'1970-01-01 02:00:00.000' " ) tdSql.checkRows(719) - + tdSql.query( "select * from t0,t1 where t0.ts=t1.ts and t1.ts >= '1970-01-01 00:00:00.000' " ) tdSql.checkRows(680) - + tdSql.query( - "select diff(col1) from t0 where t0.ts >= '1970-01-01 00:00:00.000' " + "select diff(c1) from t0 where t0.ts >= '1970-01-01 00:00:00.000' " ) tdSql.checkRows(679) tdSql.query( - "select t0,col1 from stb2 where stb2.ts < '1970-01-01 00:00:00.000' order by ts" + "select t0,c1 from stb2 where stb2.ts < '1970-01-01 00:00:00.000' order by ts" ) tdSql.checkRows(43200) diff --git a/tests/pytest/tools/taosdemoAllTest/TD-4985/query-limit-offset.py b/tests/pytest/tools/taosdemoAllTest/TD-4985/query-limit-offset.py index 081057f180..4edef88cf1 100644 --- a/tests/pytest/tools/taosdemoAllTest/TD-4985/query-limit-offset.py +++ b/tests/pytest/tools/taosdemoAllTest/TD-4985/query-limit-offset.py @@ -23,7 +23,7 @@ class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) - + def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) @@ -39,7 +39,7 @@ class TDTestCase: buildPath = root[:len(root)-len("/build/bin")] break return buildPath - + def run(self): buildPath = self.getBuildPath() if (buildPath == ""): @@ -48,7 +48,7 @@ class TDTestCase: tdLog.info("taosd found in %s" % buildPath) binPath = buildPath+ "/build/bin/" - # insert: create one or mutiple tables per sql and insert multiple rows per sql + # insert: create one or mutiple tables per sql and insert multiple rows per sql # test case for https://jira.taosdata.com:18080/browse/TD-4985 os.system("%staosdemo -f tools/taosdemoAllTest/TD-4985/query-limit-offset.json -y " % binPath) tdSql.execute("use db") @@ -56,27 +56,27 @@ class TDTestCase: tdSql.checkData(0, 0, 10000) for i in range(1000): - tdSql.execute('''insert into stb00_9999 values(%d, %d, %d,'test99.%s')''' + tdSql.execute('''insert into stb00_9999 values(%d, %d, %d,'test99.%s')''' % (1600000000000 + i, i, -10000+i, i)) - tdSql.execute('''insert into stb00_8888 values(%d, %d, %d,'test98.%s')''' + tdSql.execute('''insert into stb00_8888 values(%d, %d, %d,'test98.%s')''' % (1600000000000 + i, i, -10000+i, i)) - tdSql.execute('''insert into stb00_7777 values(%d, %d, %d,'test97.%s')''' + tdSql.execute('''insert into stb00_7777 values(%d, %d, %d,'test97.%s')''' % (1600000000000 + i, i, -10000+i, i)) - tdSql.execute('''insert into stb00_6666 values(%d, %d, %d,'test96.%s')''' + tdSql.execute('''insert into stb00_6666 values(%d, %d, %d,'test96.%s')''' % (1600000000000 + i, i, -10000+i, i)) - tdSql.execute('''insert into stb00_5555 values(%d, %d, %d,'test95.%s')''' + tdSql.execute('''insert into stb00_5555 values(%d, %d, %d,'test95.%s')''' % (1600000000000 + i, i, -10000+i, i)) - tdSql.execute('''insert into stb00_4444 values(%d, %d, %d,'test94.%s')''' + tdSql.execute('''insert into stb00_4444 values(%d, %d, %d,'test94.%s')''' % (1600000000000 + i, i, -10000+i, i)) - tdSql.execute('''insert into stb00_3333 values(%d, %d, %d,'test93.%s')''' + tdSql.execute('''insert into stb00_3333 values(%d, %d, %d,'test93.%s')''' % (1600000000000 + i, i, -10000+i, i)) - tdSql.execute('''insert into stb00_2222 values(%d, %d, %d,'test92.%s')''' + tdSql.execute('''insert into stb00_2222 values(%d, %d, %d,'test92.%s')''' % (1600000000000 + i, i, -10000+i, i)) - tdSql.execute('''insert into stb00_1111 values(%d, %d, %d,'test91.%s')''' + tdSql.execute('''insert into stb00_1111 values(%d, %d, %d,'test91.%s')''' % (1600000000000 + i, i, -10000+i, i)) - tdSql.execute('''insert into stb00_100 values(%d, %d, %d,'test90.%s')''' + tdSql.execute('''insert into stb00_100 values(%d, %d, %d,'test90.%s')''' % (1600000000000 + i, i, -10000+i, i)) - tdSql.query("select * from stb0 where col2 like 'test99%' ") + tdSql.query("select * from stb0 where c2 like 'test99%' ") tdSql.checkRows(1000) tdSql.query("select * from stb0 where tbname like 'stb00_9999' limit 10" ) tdSql.checkData(0, 1, 0) @@ -86,7 +86,7 @@ class TDTestCase: tdSql.checkData(0, 1, 5) tdSql.checkData(1, 1, 6) tdSql.checkData(2, 1, 7) - tdSql.query("select * from stb0 where col2 like 'test98%' ") + tdSql.query("select * from stb0 where c2 like 'test98%' ") tdSql.checkRows(1000) tdSql.query("select * from stb0 where tbname like 'stb00_8888' limit 10" ) tdSql.checkData(0, 1, 0) @@ -96,7 +96,7 @@ class TDTestCase: tdSql.checkData(0, 1, 5) tdSql.checkData(1, 1, 6) tdSql.checkData(2, 1, 7) - tdSql.query("select * from stb0 where col2 like 'test97%' ") + tdSql.query("select * from stb0 where c2 like 'test97%' ") tdSql.checkRows(1000) tdSql.query("select * from stb0 where tbname like 'stb00_7777' limit 10" ) tdSql.checkData(0, 1, 0) @@ -106,7 +106,7 @@ class TDTestCase: tdSql.checkData(0, 1, 5) tdSql.checkData(1, 1, 6) tdSql.checkData(2, 1, 7) - tdSql.query("select * from stb0 where col2 like 'test96%' ") + tdSql.query("select * from stb0 where c2 like 'test96%' ") tdSql.checkRows(1000) tdSql.query("select * from stb0 where tbname like 'stb00_6666' limit 10" ) tdSql.checkData(0, 1, 0) @@ -116,7 +116,7 @@ class TDTestCase: tdSql.checkData(0, 1, 5) tdSql.checkData(1, 1, 6) tdSql.checkData(2, 1, 7) - tdSql.query("select * from stb0 where col2 like 'test95%' ") + tdSql.query("select * from stb0 where c2 like 'test95%' ") tdSql.checkRows(1000) tdSql.query("select * from stb0 where tbname like 'stb00_5555' limit 10" ) tdSql.checkData(0, 1, 0) @@ -126,7 +126,7 @@ class TDTestCase: tdSql.checkData(0, 1, 5) tdSql.checkData(1, 1, 6) tdSql.checkData(2, 1, 7) - tdSql.query("select * from stb0 where col2 like 'test94%' ") + tdSql.query("select * from stb0 where c2 like 'test94%' ") tdSql.checkRows(1000) tdSql.query("select * from stb0 where tbname like 'stb00_4444' limit 10" ) tdSql.checkData(0, 1, 0) @@ -136,7 +136,7 @@ class TDTestCase: tdSql.checkData(0, 1, 5) tdSql.checkData(1, 1, 6) tdSql.checkData(2, 1, 7) - tdSql.query("select * from stb0 where col2 like 'test93%' ") + tdSql.query("select * from stb0 where c2 like 'test93%' ") tdSql.checkRows(1000) tdSql.query("select * from stb0 where tbname like 'stb00_3333' limit 100" ) tdSql.checkData(0, 1, 0) @@ -146,7 +146,7 @@ class TDTestCase: tdSql.checkData(0, 1, 5) tdSql.checkData(1, 1, 6) tdSql.checkData(2, 1, 7) - tdSql.query("select * from stb0 where col2 like 'test92%' ") + tdSql.query("select * from stb0 where c2 like 'test92%' ") tdSql.checkRows(1000) tdSql.query("select * from stb0 where tbname like 'stb00_2222' limit 100" ) tdSql.checkData(0, 1, 0) @@ -156,7 +156,7 @@ class TDTestCase: tdSql.checkData(0, 1, 5) tdSql.checkData(1, 1, 6) tdSql.checkData(2, 1, 7) - tdSql.query("select * from stb0 where col2 like 'test91%' ") + tdSql.query("select * from stb0 where c2 like 'test91%' ") tdSql.checkRows(1000) tdSql.query("select * from stb0 where tbname like 'stb00_1111' limit 100" ) tdSql.checkData(0, 1, 0) @@ -166,7 +166,7 @@ class TDTestCase: tdSql.checkData(0, 1, 5) tdSql.checkData(1, 1, 6) tdSql.checkData(2, 1, 7) - tdSql.query("select * from stb0 where col2 like 'test90%' ") + tdSql.query("select * from stb0 where c2 like 'test90%' ") tdSql.checkRows(1000) tdSql.query("select * from stb0 where tbname like 'stb00_100' limit 100" ) tdSql.checkData(0, 1, 0) @@ -178,10 +178,10 @@ class TDTestCase: tdSql.checkData(2, 1, 7) - os.system("rm -rf tools/taosdemoAllTest/TD-4985/query-limit-offset.py.sql") - - - + os.system("rm -rf tools/taosdemoAllTest/TD-4985/query-limit-offset.py.sql") + + + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.py b/tests/pytest/tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.py index b710e34d8c..78bd0c7e60 100755 --- a/tests/pytest/tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.py +++ b/tests/pytest/tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.py @@ -23,7 +23,7 @@ class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) - + def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) @@ -39,7 +39,7 @@ class TDTestCase: buildPath = root[:len(root)-len("/build/bin")] break return buildPath - + def run(self): buildPath = self.getBuildPath() if (buildPath == ""): @@ -48,7 +48,7 @@ class TDTestCase: tdLog.info("taosd found in %s" % buildPath) binPath = buildPath+ "/build/bin/" - # insert: create one or mutiple tables per sql and insert multiple rows per sql + # insert: create one or mutiple tables per sql and insert multiple rows per sql # test case for https://jira.taosdata.com:18080/browse/TD-5213 os.system("%staosdemo -f tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.json -y " % binPath) tdSql.execute("use db") @@ -58,29 +58,29 @@ class TDTestCase: # tdSql.query("select * from stb_old") # tdSql.checkRows(10) # tdSql.checkCols(1024) - + # tdSql.query("select count (tbname) from stb_new") # tdSql.checkData(0, 0, 10) # tdSql.query("select * from stb_new") # tdSql.checkRows(10) # tdSql.checkCols(4096) - + # tdLog.info("stop dnode to commit data to disk") # tdDnodes.stop(1) - # tdDnodes.start(1) + # tdDnodes.start(1) #regular table sql = "create table tb(ts timestamp, " for i in range(1022): - sql += "col%d binary(14), " % (i + 1) - sql += "col1023 binary(22))" + sql += "c%d binary(14), " % (i + 1) + sql += "c1023 binary(22))" tdSql.execute(sql) for i in range(4): sql = "insert into tb values(%d, " for j in range(1022): - str = "'%s', " % self.get_random_string(14) + str = "'%s', " % self.get_random_string(14) sql += str sql += "'%s')" % self.get_random_string(22) tdSql.execute(sql % (self.ts + i)) @@ -94,19 +94,19 @@ class TDTestCase: time.sleep(1) tdSql.query("select count(*) from tb") - tdSql.checkData(0, 0, 4) + tdSql.checkData(0, 0, 4) sql = "create table tb1(ts timestamp, " for i in range(4094): - sql += "col%d binary(14), " % (i + 1) - sql += "col4095 binary(22))" + sql += "c%d binary(14), " % (i + 1) + sql += "c4095 binary(22))" tdSql.execute(sql) for i in range(4): sql = "insert into tb1 values(%d, " for j in range(4094): - str = "'%s', " % self.get_random_string(14) + str = "'%s', " % self.get_random_string(14) sql += str sql += "'%s')" % self.get_random_string(22) tdSql.execute(sql % (self.ts + i)) @@ -120,14 +120,14 @@ class TDTestCase: time.sleep(1) tdSql.query("select count(*) from tb1") - tdSql.checkData(0, 0, 4) - + tdSql.checkData(0, 0, 4) + + + + #os.system("rm -rf tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.py.sql") + - #os.system("rm -rf tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.py.sql") - - - def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoTestInsertWithJson.py b/tests/pytest/tools/taosdemoAllTest/taosdemoTestInsertWithJson.py index 01e46eaaa0..d9d6cc1082 100644 --- a/tests/pytest/tools/taosdemoAllTest/taosdemoTestInsertWithJson.py +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoTestInsertWithJson.py @@ -23,7 +23,7 @@ class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) - + def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) @@ -39,7 +39,7 @@ class TDTestCase: buildPath = root[:len(root)-len("/build/bin")] break return buildPath - + def run(self): buildPath = self.getBuildPath() if (buildPath == ""): @@ -48,7 +48,7 @@ class TDTestCase: tdLog.info("taosd found in %s" % buildPath) binPath = buildPath+ "/build/bin/" - # insert: create one or mutiple tables per sql and insert multiple rows per sql + # insert: create one or mutiple tables per sql and insert multiple rows per sql os.system("%staosdemo -f tools/taosdemoAllTest/insert-1s1tnt1r.json -y " % binPath) tdSql.execute("use db") tdSql.query("select count (tbname) from stb0") @@ -62,7 +62,7 @@ class TDTestCase: tdSql.query("select count(*) from stb01_1") tdSql.checkData(0, 0, 200) tdSql.query("select count(*) from stb1") - tdSql.checkData(0, 0, 200000) + tdSql.checkData(0, 0, 200000) # restful connector insert data os.system("%staosdemo -f tools/taosdemoAllTest/insertRestful.json -y " % binPath) @@ -81,7 +81,7 @@ class TDTestCase: tdSql.checkData(0, 0, 200) - # insert: create mutiple tables per sql and insert one rows per sql . + # insert: create mutiple tables per sql and insert one rows per sql . os.system("%staosdemo -f tools/taosdemoAllTest/insert-1s1tntmr.json -y " % binPath) tdSql.execute("use db") tdSql.query("select count (tbname) from stb0") @@ -89,34 +89,34 @@ class TDTestCase: tdSql.query("select count (tbname) from stb1") tdSql.checkData(0, 0, 20) tdSql.query("select count(*) from stb00_0") - tdSql.checkData(0, 0, 10000) + tdSql.checkData(0, 0, 10000) tdSql.query("select count(*) from stb0") - tdSql.checkData(0, 0, 100000) + tdSql.checkData(0, 0, 100000) tdSql.query("select count(*) from stb01_0") - tdSql.checkData(0, 0, 20000) + tdSql.checkData(0, 0, 20000) tdSql.query("select count(*) from stb1") - tdSql.checkData(0, 0, 400000) + tdSql.checkData(0, 0, 400000) - # insert: using parament "insert_interval to controls spped of insert. + # insert: using parament "insert_interval to controls spped of insert. # but We need to have accurate methods to control the speed, such as getting the speed value, checking the count and so on。 os.system("%staosdemo -f tools/taosdemoAllTest/insert-interval-speed.json -y" % binPath) tdSql.execute("use db") tdSql.query("show stables") tdSql.checkData(0, 4, 100) tdSql.query("select count(*) from stb00_0") - tdSql.checkData(0, 0, 20000) + tdSql.checkData(0, 0, 20000) tdSql.query("select count(*) from stb0") - tdSql.checkData(0, 0, 2000000) + tdSql.checkData(0, 0, 2000000) tdSql.query("show stables") tdSql.checkData(1, 4, 100) tdSql.query("select count(*) from stb01_0") - tdSql.checkData(0, 0, 20000) + tdSql.checkData(0, 0, 20000) tdSql.query("select count(*) from stb1") - tdSql.checkData(0, 0, 2000000) - + tdSql.checkData(0, 0, 2000000) + # spend 2min30s for 3 testcases. # insert: drop and child_table_exists combination test - # insert: using parament "childtable_offset and childtable_limit" to control table'offset point and offset + # insert: using parament "childtable_offset and childtable_limit" to control table'offset point and offset os.system("%staosdemo -f tools/taosdemoAllTest/insert-nodbnodrop.json -y" % binPath) tdSql.error("show dbno.stables") os.system("%staosdemo -f tools/taosdemoAllTest/insert-newdb.json -y" % binPath) @@ -128,41 +128,41 @@ class TDTestCase: tdSql.query("select count (tbname) from stb2") tdSql.checkData(0, 0, 7) tdSql.query("select count (tbname) from stb3") - tdSql.checkData(0, 0, 8) + tdSql.checkData(0, 0, 8) tdSql.query("select count (tbname) from stb4") - tdSql.checkData(0, 0, 8) + tdSql.checkData(0, 0, 8) os.system("%staosdemo -f tools/taosdemoAllTest/insert-offset.json -y" % binPath) - tdSql.execute("use db") + tdSql.execute("use db") tdSql.query("select count(*) from stb0") - tdSql.checkData(0, 0, 50) + tdSql.checkData(0, 0, 50) tdSql.query("select count(*) from stb1") - tdSql.checkData(0, 0, 240) + tdSql.checkData(0, 0, 240) tdSql.query("select count(*) from stb2") - tdSql.checkData(0, 0, 220) + tdSql.checkData(0, 0, 220) tdSql.query("select count(*) from stb3") tdSql.checkData(0, 0, 180) tdSql.query("select count(*) from stb4") tdSql.checkData(0, 0, 160) os.system("%staosdemo -f tools/taosdemoAllTest/insert-newtable.json -y" % binPath) - tdSql.execute("use db") + tdSql.execute("use db") tdSql.query("select count(*) from stb0") - tdSql.checkData(0, 0, 150) + tdSql.checkData(0, 0, 150) tdSql.query("select count(*) from stb1") - tdSql.checkData(0, 0, 360) + tdSql.checkData(0, 0, 360) tdSql.query("select count(*) from stb2") - tdSql.checkData(0, 0, 360) + tdSql.checkData(0, 0, 360) tdSql.query("select count(*) from stb3") tdSql.checkData(0, 0, 340) tdSql.query("select count(*) from stb4") tdSql.checkData(0, 0, 400) os.system("%staosdemo -f tools/taosdemoAllTest/insert-renewdb.json -y" % binPath) - tdSql.execute("use db") + tdSql.execute("use db") tdSql.query("select count(*) from stb0") - tdSql.checkData(0, 0, 50) + tdSql.checkData(0, 0, 50) tdSql.query("select count(*) from stb1") - tdSql.checkData(0, 0, 120) + tdSql.checkData(0, 0, 120) tdSql.query("select count(*) from stb2") - tdSql.checkData(0, 0, 140) + tdSql.checkData(0, 0, 140) tdSql.query("select count(*) from stb3") tdSql.checkData(0, 0, 160) tdSql.query("select count(*) from stb4") @@ -170,59 +170,59 @@ class TDTestCase: # insert: let parament in json file is illegal, it'll expect error. - tdSql.execute("drop database if exists db") + tdSql.execute("drop database if exists db") os.system("%staosdemo -f tools/taosdemoAllTest/insertColumnsAndTagNumLarge1024.json -y " % binPath) tdSql.error("use db") - tdSql.execute("drop database if exists db") + tdSql.execute("drop database if exists db") os.system("%staosdemo -f tools/taosdemoAllTest/insertSigcolumnsNum1024.json -y " % binPath) tdSql.error("select * from db.stb0") - tdSql.execute("drop database if exists db") + tdSql.execute("drop database if exists db") os.system("%staosdemo -f tools/taosdemoAllTest/insertColumnsAndTagNum1024.json -y " % binPath) tdSql.query("select count(*) from db.stb0") - tdSql.checkData(0, 0, 10000) - tdSql.execute("drop database if exists db") + tdSql.checkData(0, 0, 10000) + tdSql.execute("drop database if exists db") os.system("%staosdemo -f tools/taosdemoAllTest/insertInterlaceRowsLarge1M.json -y " % binPath) tdSql.query("select count(*) from db.stb0") tdSql.checkRows(0) - tdSql.execute("drop database if exists db") + tdSql.execute("drop database if exists db") os.system("%staosdemo -f tools/taosdemoAllTest/insertColumnsNum0.json -y " % binPath) - tdSql.execute("use db") + tdSql.execute("use db") tdSql.query("show stables like 'stb0%' ") tdSql.checkData(0, 2, 11) - tdSql.execute("drop database if exists db") - os.system("%staosdemo -f tools/taosdemoAllTest/insertTagsNumLarge128.json -y " % binPath) - tdSql.error("use db1") - tdSql.execute("drop database if exists db") - os.system("%staosdemo -f tools/taosdemoAllTest/insertBinaryLenLarge16374AllcolLar16384.json -y " % binPath) - tdSql.query("select count(*) from db.stb0") + tdSql.execute("drop database if exists db") + os.system("%staosdemo -f tools/taosdemoAllTest/insertTagsNumLarge128.json -y " % binPath) + tdSql.error("use db1") + tdSql.execute("drop database if exists db") + os.system("%staosdemo -f tools/taosdemoAllTest/insertBinaryLenLarge16374AllcolLar16384.json -y " % binPath) + tdSql.query("select count(*) from db.stb0") tdSql.checkRows(1) - tdSql.query("select count(*) from db.stb1") + tdSql.query("select count(*) from db.stb1") tdSql.checkRows(1) tdSql.error("select * from db.stb3") tdSql.error("select * from db.stb2") - tdSql.execute("drop database if exists db") - os.system("%staosdemo -f tools/taosdemoAllTest/insertNumOfrecordPerReq0.json -y " % binPath) - tdSql.error("select count(*) from db.stb0") - tdSql.execute("drop database if exists db") - os.system("%staosdemo -f tools/taosdemoAllTest/insertNumOfrecordPerReqless0.json -y " % binPath) - tdSql.error("use db") - tdSql.execute("drop database if exists db") - os.system("%staosdemo -f tools/taosdemoAllTest/insertChildTab0.json -y " % binPath) - tdSql.error("use db") - tdSql.execute("drop database if exists db") - os.system("%staosdemo -f tools/taosdemoAllTest/insertChildTabLess0.json -y " % binPath) - tdSql.error("use db") - tdSql.execute("drop database if exists blf") - os.system("%staosdemo -f tools/taosdemoAllTest/insertTimestepMulRowsLargeint16.json -y " % binPath) - tdSql.execute("use blf") - tdSql.query("select ts from blf.p_0_topics_7 limit 262800,1") + tdSql.execute("drop database if exists db") + os.system("%staosdemo -f tools/taosdemoAllTest/insertNumOfrecordPerReq0.json -y " % binPath) + tdSql.error("select count(*) from db.stb0") + tdSql.execute("drop database if exists db") + os.system("%staosdemo -f tools/taosdemoAllTest/insertNumOfrecordPerReqless0.json -y " % binPath) + tdSql.error("use db") + tdSql.execute("drop database if exists db") + os.system("%staosdemo -f tools/taosdemoAllTest/insertChildTab0.json -y " % binPath) + tdSql.error("use db") + tdSql.execute("drop database if exists db") + os.system("%staosdemo -f tools/taosdemoAllTest/insertChildTabLess0.json -y " % binPath) + tdSql.error("use db") + tdSql.execute("drop database if exists blf") + os.system("%staosdemo -f tools/taosdemoAllTest/insertTimestepMulRowsLargeint16.json -y " % binPath) + tdSql.execute("use blf") + tdSql.query("select ts from blf.p_0_topics_7 limit 262800,1") tdSql.checkData(0, 0, "2020-03-31 12:00:00.000") tdSql.query("select first(ts) from blf.p_0_topics_2") tdSql.checkData(0, 0, "2019-10-01 00:00:00") - tdSql.query("select last(ts) from blf.p_0_topics_6 ") + tdSql.query("select last(ts) from blf.p_0_topics_6 ") tdSql.checkData(0, 0, "2020-09-29 23:59:00") - os.system("%staosdemo -f tools/taosdemoAllTest/insertMaxNumPerReq.json -y " % binPath) - tdSql.execute("use db") + os.system("%staosdemo -f tools/taosdemoAllTest/insertMaxNumPerReq.json -y " % binPath) + tdSql.execute("use db") tdSql.query("select count(*) from stb0") tdSql.checkData(0, 0, 5000000) tdSql.query("select count(*) from stb1") @@ -230,7 +230,7 @@ class TDTestCase: - # insert: timestamp and step + # insert: timestamp and step os.system("%staosdemo -f tools/taosdemoAllTest/insert-timestep.json -y " % binPath) tdSql.execute("use db") tdSql.query("show stables") @@ -239,13 +239,13 @@ class TDTestCase: tdSql.query("select count (tbname) from stb1") tdSql.checkData(0, 0, 20) tdSql.query("select last(ts) from db.stb00_0") - tdSql.checkData(0, 0, "2020-10-01 00:00:00.019000") + tdSql.checkData(0, 0, "2020-10-01 00:00:00.019000") tdSql.query("select count(*) from stb0") - tdSql.checkData(0, 0, 200) + tdSql.checkData(0, 0, 200) tdSql.query("select last(ts) from db.stb01_0") - tdSql.checkData(0, 0, "2020-11-01 00:00:00.190000") + tdSql.checkData(0, 0, "2020-11-01 00:00:00.190000") tdSql.query("select count(*) from stb1") - tdSql.checkData(0, 0, 400) + tdSql.checkData(0, 0, 400) # # insert: disorder_ratio os.system("%staosdemo -f tools/taosdemoAllTest/insert-disorder.json -g 2>&1 -y " % binPath) @@ -255,14 +255,14 @@ class TDTestCase: tdSql.query("select count (tbname) from stb1") tdSql.checkData(0, 0, 1) tdSql.query("select count(*) from stb0") - tdSql.checkData(0, 0, 10) + tdSql.checkData(0, 0, 10) tdSql.query("select count(*) from stb1") - tdSql.checkData(0, 0, 10) + tdSql.checkData(0, 0, 10) # insert: sample json os.system("%staosdemo -f tools/taosdemoAllTest/insert-sample.json -y " % binPath) tdSql.execute("use dbtest123") - tdSql.query("select col2 from stb0") + tdSql.query("select c2 from stb0") tdSql.checkData(0, 0, 2147483647) tdSql.query("select * from stb1 where t1=-127") tdSql.checkRows(20) @@ -271,13 +271,13 @@ class TDTestCase: tdSql.query("select * from stb1 where t2=126") tdSql.checkRows(10) - # insert: test interlace parament + # insert: test interlace parament os.system("%staosdemo -f tools/taosdemoAllTest/insert-interlace-row.json -y " % binPath) tdSql.execute("use db") tdSql.query("select count (tbname) from stb0") tdSql.checkData(0, 0, 100) tdSql.query("select count (*) from stb0") - tdSql.checkData(0, 0, 15000) + tdSql.checkData(0, 0, 15000) # # insert: auto_create @@ -317,10 +317,10 @@ class TDTestCase: tdSql.checkRows(20) os.system("rm -rf ./insert_res.txt") - os.system("rm -rf tools/taosdemoAllTest/taosdemoTestInsertWithJson.py.sql") - - - + os.system("rm -rf tools/taosdemoAllTest/taosdemoTestInsertWithJson.py.sql") + + + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/tools/taosdemoTest.py b/tests/pytest/tools/taosdemoTest.py index 4cae8dfd3c..5662881031 100644 --- a/tests/pytest/tools/taosdemoTest.py +++ b/tests/pytest/tools/taosdemoTest.py @@ -59,11 +59,11 @@ class TDTestCase: tdSql.checkData(0, 0, self.numberOfTables * self.numberOfRecords) tdSql.query( - "select sum(col1) from test.meters interval(1h) sliding(30m)") + "select sum(c1) from test.meters interval(1h) sliding(30m)") tdSql.checkRows(2) tdSql.query( - "select apercentile(col1, 1) from test.meters interval(100s)") + "select apercentile(c1, 1) from test.meters interval(100s)") tdSql.checkRows(1) tdSql.error("select loc, count(loc) from test.meters") diff --git a/tests/pytest/wal/sdbComp.py b/tests/pytest/wal/sdbComp.py index 56b18c49eb..428fbc9a14 100644 --- a/tests/pytest/wal/sdbComp.py +++ b/tests/pytest/wal/sdbComp.py @@ -26,11 +26,11 @@ class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) - + def getBuildPath(self): global selfPath selfPath = os.path.dirname(os.path.realpath(__file__)) - + if ("community" in selfPath): projPath = selfPath[:selfPath.find("community")] else: @@ -43,7 +43,7 @@ class TDTestCase: buildPath = root[:len(root)-len("/build/bin")] break return buildPath - + def run(self): # set path para @@ -62,9 +62,9 @@ class TDTestCase: os.system("rm -rf %s/sim/dnode1/data/mnode_bak/" % testPath) tdSql.execute("drop database if exists db2") os.system("%staosdemo -f wal/insertDataDb1.json -y " % binPath) - tdSql.execute("drop database if exists db1") + tdSql.execute("drop database if exists db1") os.system("%staosdemo -f wal/insertDataDb2.json -y " % binPath) - tdSql.execute("drop table if exists db2.stb0") + tdSql.execute("drop table if exists db2.stb0") os.system("%staosdemo -f wal/insertDataDb2Newstab.json -y " % binPath) query_pid1 = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1]) print(query_pid1) @@ -72,14 +72,14 @@ class TDTestCase: tdSql.execute("drop table if exists stb1_0") tdSql.execute("drop table if exists stb1_1") tdSql.execute("insert into stb0_0 values(1614218412000,8637,78.861045,'R','bf3')(1614218422000,8637,98.861045,'R','bf3')") - tdSql.execute("alter table db2.stb0 add column col4 int") - tdSql.execute("alter table db2.stb0 drop column col2") - tdSql.execute("alter table db2.stb0 add tag t3 int;") + tdSql.execute("alter table db2.stb0 add column c4 int") + tdSql.execute("alter table db2.stb0 drop column c2") + tdSql.execute("alter table db2.stb0 add tag t3 int;") tdSql.execute("alter table db2.stb0 drop tag t1") - tdSql.execute("create table if not exists stb2_0 (ts timestamp, col0 int, col1 float) ") + tdSql.execute("create table if not exists stb2_0 (ts timestamp, c0 int, c1 float) ") tdSql.execute("insert into stb2_0 values(1614218412000,8637,78.861045)") - tdSql.execute("alter table stb2_0 add column col2 binary(4)") - tdSql.execute("alter table stb2_0 drop column col1") + tdSql.execute("alter table stb2_0 add column c2 binary(4)") + tdSql.execute("alter table stb2_0 drop column c1") tdSql.execute("insert into stb2_0 values(1614218422000,8638,'R')") # stop taosd and compact wal file @@ -87,9 +87,9 @@ class TDTestCase: sleep(10) os.system("nohup %s/taosd --compact-mnode-wal -c %s/sim/dnode1/cfg/ & " %(binPath,testPath) ) sleep(5) - assert os.path.exists(walFilePath) , "%s is not generated, compact didn't take effect " % walFilePath + assert os.path.exists(walFilePath) , "%s is not generated, compact didn't take effect " % walFilePath - # use new wal file to start taosd + # use new wal file to start taosd tdDnodes.start(1) sleep(5) tdSql.execute("reset query cache") @@ -108,14 +108,14 @@ class TDTestCase: tdSql.checkData(0, 0, 2) tdSql.query("select count(*) from stb2_0") tdSql.checkData(0, 0, 2) - + # delete useless file testcaseFilename = os.path.split(__file__)[-1] os.system("rm -rf ./insert_res.txt") - os.system("rm -rf wal/%s.sql" % testcaseFilename ) - - - + os.system("rm -rf wal/%s.sql" % testcaseFilename) + + + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) From 524641123acaa0f4401102ed2550f5cee55684d4 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Tue, 20 Jul 2021 10:23:21 +0800 Subject: [PATCH 65/67] [TD-5399]: update scripts for daily performance test --- tests/perftest-scripts/perftest-query.sh | 34 +++++++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/tests/perftest-scripts/perftest-query.sh b/tests/perftest-scripts/perftest-query.sh index bf62f401bf..ffb645fefb 100755 --- a/tests/perftest-scripts/perftest-query.sh +++ b/tests/perftest-scripts/perftest-query.sh @@ -1,5 +1,22 @@ #!/bin/bash +branch= +if [ x$1 != x ];then + branch=$1 + echo "Testing branch: $branch" +else + echo "Please enter branch name as a parameter" + exit 1 +fi +jemalloc= +if [ x$2 != x ];then + jemalloc=1 + echo "Building TDengine using jemalloc" +else + jemalloc=0 + echo "Building TDengine using glibc" +fi + today=`date +"%Y%m%d"` WORK_DIR=/home/ubuntu/pxiao PERFORMANCE_TEST_REPORT=$WORK_DIR/TDengine/tests/performance-test-report-$today.log @@ -40,8 +57,8 @@ function buildTDengine { git remote update > /dev/null git reset --hard HEAD - git checkout master - REMOTE_COMMIT=`git rev-parse --short remotes/origin/master` + git checkout $branch + REMOTE_COMMIT=`git rev-parse --short remotes/origin/$branch` LOCAL_COMMIT=`git rev-parse --short @` echo " LOCAL: $LOCAL_COMMIT" @@ -53,9 +70,18 @@ function buildTDengine { git pull > /dev/null 2>&1 LOCAL_COMMIT=`git rev-parse --short @` + if [ $jemalloc = "1" ];then + git submodule update --init --recursive + fi + cd debug rm -rf * - cmake .. > /dev/null + + if [ $jemalloc = "1" ];then + cmake .. -DJEMALLOC_ENABLED=true > /dev/null + else + cmake .. > /dev/null + fi make && make install > /dev/null fi } @@ -95,7 +121,7 @@ function sendReport { sed -i 's/\x1b\[[0-9;]*m//g' $PERFORMANCE_TEST_REPORT BODY_CONTENT=`cat $PERFORMANCE_TEST_REPORT` - echo -e "From: \nto: ${receiver}\nsubject: Query Performace Report ${today}, commit ID: ${LOCAL_COMMIT}\n\n${today}:\n${BODY_CONTENT}" | \ + echo -e "From: \nto: ${receiver}\nsubject: Query Performace Report ${branch} ${today}, commit ID: ${LOCAL_COMMIT}\n\n${today}:\n${BODY_CONTENT}" | \ (cat - && uuencode $PERFORMANCE_TEST_REPORT performance-test-report-$today.log) | \ /usr/sbin/ssmtp "${receiver}" && echo "Report Sent!" } From a8ced0f493350416c347e134fb44b094edd2f2a3 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Tue, 20 Jul 2021 10:40:33 +0800 Subject: [PATCH 66/67] fix runtime error --- tests/perftest-scripts/perftest-query.sh | 12 +++++++----- tests/pytest/tools/taosdemoPerformance.py | 1 - 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/perftest-scripts/perftest-query.sh b/tests/perftest-scripts/perftest-query.sh index ffb645fefb..bcc944dadb 100755 --- a/tests/perftest-scripts/perftest-query.sh +++ b/tests/perftest-scripts/perftest-query.sh @@ -10,10 +10,9 @@ else fi jemalloc= if [ x$2 != x ];then - jemalloc=1 + jemalloc=jemalloc echo "Building TDengine using jemalloc" else - jemalloc=0 echo "Building TDengine using glibc" fi @@ -70,14 +69,16 @@ function buildTDengine { git pull > /dev/null 2>&1 LOCAL_COMMIT=`git rev-parse --short @` - if [ $jemalloc = "1" ];then + if [ $jemalloc = "jemalloc" ];then + echo "git submodule update --init --recursive" git submodule update --init --recursive fi cd debug rm -rf * - if [ $jemalloc = "1" ];then + if [ $jemalloc = "jemalloc" ];then + echo "cmake .. -DJEMALLOC_ENABLED=true > /dev/null" cmake .. -DJEMALLOC_ENABLED=true > /dev/null else cmake .. > /dev/null @@ -121,7 +122,8 @@ function sendReport { sed -i 's/\x1b\[[0-9;]*m//g' $PERFORMANCE_TEST_REPORT BODY_CONTENT=`cat $PERFORMANCE_TEST_REPORT` - echo -e "From: \nto: ${receiver}\nsubject: Query Performace Report ${branch} ${today}, commit ID: ${LOCAL_COMMIT}\n\n${today}:\n${BODY_CONTENT}" | \ + + echo -e "From: \nto: ${receiver}\nsubject: Query Performace Report ${branch} ${jemalloc} ${today}, commit ID: ${LOCAL_COMMIT}\n\n${today}:\n${BODY_CONTENT}" | \ (cat - && uuencode $PERFORMANCE_TEST_REPORT performance-test-report-$today.log) | \ /usr/sbin/ssmtp "${receiver}" && echo "Report Sent!" } diff --git a/tests/pytest/tools/taosdemoPerformance.py b/tests/pytest/tools/taosdemoPerformance.py index fb74588984..c8293ee31f 100644 --- a/tests/pytest/tools/taosdemoPerformance.py +++ b/tests/pytest/tools/taosdemoPerformance.py @@ -122,7 +122,6 @@ class taosdemoPerformace: return buildPath def insertData(self): - tdSql.prepare() buildPath = self.getBuildPath() if (buildPath == ""): From d1110fe7b0be9193b205d727de19ff3d5f822397 Mon Sep 17 00:00:00 2001 From: Linhe Huo Date: Tue, 20 Jul 2021 15:43:44 +0800 Subject: [PATCH 67/67] [TD-5406]: dll export for stmt* api (#6932) --- src/client/src/taos.def | 7 +++++-- src/inc/taos.h | 31 ++++++++++++++++--------------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/src/client/src/taos.def b/src/client/src/taos.def index 43cd819061..7d3b8e80c2 100644 --- a/src/client/src/taos.def +++ b/src/client/src/taos.def @@ -7,11 +7,16 @@ taos_connect_auth taos_close taos_stmt_init taos_stmt_prepare +taos_stmt_set_tbname_tags +taos_stmt_set_tbname +taos_stmt_is_insert +taos_stmt_num_params taos_stmt_bind_param taos_stmt_add_batch taos_stmt_execute taos_stmt_use_result taos_stmt_close +taos_stmt_errstr taos_query taos_fetch_row taos_result_precision @@ -37,6 +42,4 @@ taos_consume taos_unsubscribe taos_open_stream taos_close_stream -taos_fetch_block taos_load_table_info - diff --git a/src/inc/taos.h b/src/inc/taos.h index a62f387924..83cf0f57cf 100644 --- a/src/inc/taos.h +++ b/src/inc/taos.h @@ -72,6 +72,7 @@ DLL_EXPORT int taos_init(); DLL_EXPORT void taos_cleanup(void); DLL_EXPORT int taos_options(TSDB_OPTION option, const void *arg, ...); DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port); +DLL_EXPORT TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port); DLL_EXPORT void taos_close(TAOS *taos); const char *taos_data_type(int type); @@ -110,21 +111,21 @@ typedef struct TAOS_MULTI_BIND { } TAOS_MULTI_BIND; -TAOS_STMT *taos_stmt_init(TAOS *taos); -int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length); -int taos_stmt_set_tbname_tags(TAOS_STMT* stmt, const char* name, TAOS_BIND* tags); -int taos_stmt_set_tbname(TAOS_STMT* stmt, const char* name); -int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert); -int taos_stmt_num_params(TAOS_STMT *stmt, int *nums); +DLL_EXPORT TAOS_STMT *taos_stmt_init(TAOS *taos); +DLL_EXPORT int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length); +DLL_EXPORT int taos_stmt_set_tbname_tags(TAOS_STMT* stmt, const char* name, TAOS_BIND* tags); +DLL_EXPORT int taos_stmt_set_tbname(TAOS_STMT* stmt, const char* name); +DLL_EXPORT int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert); +DLL_EXPORT int taos_stmt_num_params(TAOS_STMT *stmt, int *nums); int taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes); -int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND *bind); +DLL_EXPORT int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND *bind); int taos_stmt_bind_param_batch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind); int taos_stmt_bind_single_param_batch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int colIdx); -int taos_stmt_add_batch(TAOS_STMT *stmt); -int taos_stmt_execute(TAOS_STMT *stmt); -TAOS_RES * taos_stmt_use_result(TAOS_STMT *stmt); -int taos_stmt_close(TAOS_STMT *stmt); -char * taos_stmt_errstr(TAOS_STMT *stmt); +DLL_EXPORT int taos_stmt_add_batch(TAOS_STMT *stmt); +DLL_EXPORT int taos_stmt_execute(TAOS_STMT *stmt); +DLL_EXPORT TAOS_RES * taos_stmt_use_result(TAOS_STMT *stmt); +DLL_EXPORT int taos_stmt_close(TAOS_STMT *stmt); +DLL_EXPORT char * taos_stmt_errstr(TAOS_STMT *stmt); DLL_EXPORT TAOS_RES *taos_query(TAOS *taos, const char *sql); DLL_EXPORT TAOS_ROW taos_fetch_row(TAOS_RES *res); @@ -139,10 +140,10 @@ DLL_EXPORT int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int n DLL_EXPORT void taos_stop_query(TAOS_RES *res); DLL_EXPORT bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col); -int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows); -int taos_validate_sql(TAOS *taos, const char *sql); +DLL_EXPORT int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows); +DLL_EXPORT int taos_validate_sql(TAOS *taos, const char *sql); -int* taos_fetch_lengths(TAOS_RES *res); +DLL_EXPORT int* taos_fetch_lengths(TAOS_RES *res); // TAOS_RES *taos_list_tables(TAOS *mysql, const char *wild); // TAOS_RES *taos_list_dbs(TAOS *mysql, const char *wild);