From 4c169bc09b9067df7a3506971ac04abd195fb648 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 25 Jun 2023 00:17:40 +0800 Subject: [PATCH 1/4] fix: system stable query with limit/offset --- source/common/src/tdatablock.c | 7 +- .../0-others/information_schema.py | 66 +++++++++++++++++-- 2 files changed, 64 insertions(+), 9 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index d06f9beb7f..2d788f7ff0 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1701,7 +1701,7 @@ static int32_t colDataMoveVarData(SColumnInfoData* pColInfoData, size_t start, s dataLen += varDataTLen(data); } beigin++; - } + } if (dataOffset > 0) { memmove(pColInfoData->pData, pColInfoData->pData + dataOffset, dataLen); @@ -1713,7 +1713,8 @@ static int32_t colDataMoveVarData(SColumnInfoData* pColInfoData, size_t start, s static void colDataTrimFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_t total) { if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { - pColInfoData->varmeta.length = colDataMoveVarData(pColInfoData, n, total); + // pColInfoData->varmeta.length = colDataMoveVarData(pColInfoData, n, total); + memmove(pColInfoData->varmeta.offset, &pColInfoData->varmeta.offset[n], (total - n) * sizeof(int32_t)); // clear the offset value of the unused entries. memset(&pColInfoData->varmeta.offset[total - n], 0, n); @@ -1745,7 +1746,7 @@ int32_t blockDataTrimFirstRows(SSDataBlock* pBlock, size_t n) { static void colDataKeepFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_t total) { if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { - pColInfoData->varmeta.length = colDataMoveVarData(pColInfoData, 0, n); + // pColInfoData->varmeta.length = colDataMoveVarData(pColInfoData, 0, n); memset(&pColInfoData->varmeta.offset[n], 0, total - n); } else { // reset the bitmap value /*int32_t stopIndex = BitmapLen(n) * 8; diff --git a/tests/system-test/0-others/information_schema.py b/tests/system-test/0-others/information_schema.py index 8245407ade..b0d1c06c12 100644 --- a/tests/system-test/0-others/information_schema.py +++ b/tests/system-test/0-others/information_schema.py @@ -132,8 +132,61 @@ class TDTestCase: else: tdSql.checkEqual(result[i][0],f'stb_{i-1}') tdSql.checkEqual(result[i][1],ctbnum) + + def ins_stable_check2(self): + tdSql.execute('drop database if exists restful_test') + tdSql.execute('drop database if exists log') + tdSql.execute('drop database if exists d0') + tdSql.execute('drop database if exists d1') + tdSql.execute('create database restful_test vgroups 4 replica 1') + tdSql.execute('create database log vgroups 2 replica 1') + tdSql.execute('create database d0 vgroups 4 replica 1') + tdSql.execute('create database d1 vgroups 4 replica 1') + log_stb_num = 5 + rest_stb_num = 51 + for i in range(rest_stb_num): + tdSql.execute(f'create stable restful_test._stb_{i} (ts timestamp,c0 int) tags(t0 int);') + tdSql.execute(f'create stable d0._stb_{i} (ts timestamp,c0 int, c1 int) tags(t0 int,t1 int);') + tdSql.execute(f'create stable d1._stb_{i} (ts timestamp,c0 int, c1 int, c2 int) tags(t0 int,t1 int, t2 int);') + tdSql.execute(f'CREATE STABLE log.`taosadapter_restful_http_request_summary_milliseconds` (`_ts` TIMESTAMP, `sum` DOUBLE) TAGS (`request_uri` NCHAR(128));') + tdSql.execute(f'CREATE STABLE log.`taosadapter_system_cpu_percent` (`_ts` TIMESTAMP, `gauge` DOUBLE) TAGS (`endpoint` NCHAR(45));') + tdSql.execute(f'CREATE STABLE log.`taosadapter_restful_http_request_total` (`_ts` TIMESTAMP, `gauge` DOUBLE) TAGS (`client_ip` NCHAR(40));') + tdSql.execute(f'CREATE STABLE log.`taosadapter_system_mem_percent` (`_ts` TIMESTAMP, `gauge` DOUBLE) TAGS (`endpoint` NCHAR(45));') + tdSql.execute(f'CREATE STABLE log.`taosadapter_restful_http_request_fail` (`_ts` TIMESTAMP, `gauge` DOUBLE) TAGS (`request_uri` NCHAR(128), `status_code` NCHAR(4));') + + tdSql.query(f'select * from information_schema.ins_stables where db_name="restful_test" limit 0,25;') # condition 1 + result = tdSql.queryResult + tdSql.checkEqual(len(result),25) + for i in range(len(result)): + tdSql.checkEqual(result[i][0][0:5],f'_stb_') # stable_name + tdSql.checkEqual(result[i][1],f'restful_test') # db_name + tdSql.checkEqual(result[i][5]>=result[i][2],True) # last_update >= create_time + tdSql.checkEqual(result[i][3]>1,True) # columns + tdSql.checkEqual(result[i][4]>0,True) # tags + tdSql.checkEqual(result[i][6],None) # table_comment + tdSql.checkEqual(result[i][7],f'5000a,5000a') # watermark + tdSql.checkEqual(result[i][8],f'-1a,-1a') # max_delay + tdSql.checkEqual(result[i][9],f'') # rollup + tdSql.query(f'select create_time from information_schema.ins_stables where db_name="restful_test" order by create_time asc limit 10,1') + result = tdSql.queryResult + tdSql.checkEqual(len(result),1) + _create_time=result[0][0] + tdSql.query("select * from information_schema.ins_stables where db_name='restful_test' and create_time > '%s' limit 10,30" % (_create_time)) # condition 2 + result = tdSql.queryResult + tdSql.checkEqual(len(result),30) + for i in range(len(result)): + tdSql.checkEqual(result[i][0][0:5],f'_stb_') # stable_name + tdSql.checkEqual(result[i][1],f'restful_test') # db_name + tdSql.checkEqual(result[i][5]>=result[i][2],True) # last_update >= create_time + tdSql.checkEqual(result[i][2]>_create_time,True) # create_time + tdSql.checkEqual(result[i][3]>1,True) # columns + tdSql.checkEqual(result[i][4]>0,True) # tags + tdSql.checkEqual(result[i][6],None) # table_comment + tdSql.checkEqual(result[i][7],f'5000a,5000a') # watermark + tdSql.checkEqual(result[i][8],f'-1a,-1a') # max_delay + tdSql.checkEqual(result[i][9],f'') # rollup def ins_columns_check(self): tdSql.execute('drop database if exists db2') @@ -211,12 +264,13 @@ class TDTestCase: tdSql.checkEqual(tdSql.queryResult[0][0],'') def run(self): - self.prepare_data() - self.count_check() - self.ins_columns_check() + # self.prepare_data() + # self.count_check() + # self.ins_columns_check() # self.ins_col_check_4096() - self.ins_stable_check() - self.ins_dnodes_check() + # self.ins_stable_check() + self.ins_stable_check2() + # self.ins_dnodes_check() def stop(self): @@ -224,4 +278,4 @@ class TDTestCase: tdLog.success("%s successfully executed" % __file__) tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file +tdCases.addLinux(__file__, TDTestCase()) From afa186108ebfa0385bccb47b86b84c23b9ec068e Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 25 Jun 2023 00:19:30 +0800 Subject: [PATCH 2/4] chore: more code --- source/common/src/tdatablock.c | 2 +- tests/system-test/0-others/information_schema.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 2d788f7ff0..a5c575962d 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1701,7 +1701,7 @@ static int32_t colDataMoveVarData(SColumnInfoData* pColInfoData, size_t start, s dataLen += varDataTLen(data); } beigin++; - } + } if (dataOffset > 0) { memmove(pColInfoData->pData, pColInfoData->pData + dataOffset, dataLen); diff --git a/tests/system-test/0-others/information_schema.py b/tests/system-test/0-others/information_schema.py index b0d1c06c12..762361f051 100644 --- a/tests/system-test/0-others/information_schema.py +++ b/tests/system-test/0-others/information_schema.py @@ -264,13 +264,13 @@ class TDTestCase: tdSql.checkEqual(tdSql.queryResult[0][0],'') def run(self): - # self.prepare_data() - # self.count_check() - # self.ins_columns_check() + self.prepare_data() + self.count_check() + self.ins_columns_check() # self.ins_col_check_4096() - # self.ins_stable_check() + self.ins_stable_check() self.ins_stable_check2() - # self.ins_dnodes_check() + self.ins_dnodes_check() def stop(self): From a868ed48952bd0afd3684054024e53b3804ce63a Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 25 Jun 2023 16:50:34 +0800 Subject: [PATCH 3/4] fix: use max column length --- source/client/src/clientImpl.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 5963e419e1..6a2a7bb5a2 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1699,6 +1699,7 @@ static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, i len += lenTmp; pStart += lenTmp; + int32_t estimateColLen = 0; for (int32_t j = 0; j < numOfRows; ++j) { if (offset[j] == -1) { continue; @@ -1708,20 +1709,21 @@ static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, i int32_t jsonInnerType = *data; char* jsonInnerData = data + CHAR_BYTES; if (jsonInnerType == TSDB_DATA_TYPE_NULL) { - len += (VARSTR_HEADER_SIZE + strlen(TSDB_DATA_NULL_STR_L)); + estimateColLen += (VARSTR_HEADER_SIZE + strlen(TSDB_DATA_NULL_STR_L)); } else if (tTagIsJson(data)) { - len += (VARSTR_HEADER_SIZE + ((const STag*)(data))->len); + estimateColLen += (VARSTR_HEADER_SIZE + ((const STag*)(data))->len); } else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) { // value -> "value" - len += varDataTLen(jsonInnerData) + CHAR_BYTES * 2; + estimateColLen += varDataTLen(jsonInnerData) + CHAR_BYTES * 2; } else if (jsonInnerType == TSDB_DATA_TYPE_DOUBLE) { - len += (VARSTR_HEADER_SIZE + 32); + estimateColLen += (VARSTR_HEADER_SIZE + 32); } else if (jsonInnerType == TSDB_DATA_TYPE_BOOL) { - len += (VARSTR_HEADER_SIZE + 5); + estimateColLen += (VARSTR_HEADER_SIZE + 5); } else { tscError("estimateJsonLen error: invalid type:%d", jsonInnerType); return -1; } } + len += TMAX(colLen, estimateColLen); } else if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) { int32_t lenTmp = numOfRows * sizeof(int32_t); len += (lenTmp + colLen); @@ -1793,6 +1795,9 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int tscError("doConvertJson error: colLen:%d >= dataLen:%d", colLen, dataLen); return TSDB_CODE_TSC_INTERNAL_ERROR; } + + if(colLen < dataLen) + if (pResultInfo->fields[i].type == TSDB_DATA_TYPE_JSON) { int32_t* offset = (int32_t*)pStart; int32_t* offset1 = (int32_t*)pStart1; From f1eb637f03fd1b0002b97d2444914ad14fac23ec Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 25 Jun 2023 16:52:22 +0800 Subject: [PATCH 4/4] chore: code revert --- source/client/src/clientImpl.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 6a2a7bb5a2..85255caff7 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1795,9 +1795,6 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int tscError("doConvertJson error: colLen:%d >= dataLen:%d", colLen, dataLen); return TSDB_CODE_TSC_INTERNAL_ERROR; } - - if(colLen < dataLen) - if (pResultInfo->fields[i].type == TSDB_DATA_TYPE_JSON) { int32_t* offset = (int32_t*)pStart; int32_t* offset1 = (int32_t*)pStart1;