diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index cbfd1efb84..99cdb53103 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -277,7 +277,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq2** pReq, const SSDataBlock* pData bool alreadyAddGroupId(char* ctbName, int64_t groupId); bool isAutoTableName(char* ctbName); -void buildCtbNameAddGroupId(const char* stbName, char* ctbName, uint64_t groupId); +int32_t buildCtbNameAddGroupId(const char* stbName, char* ctbName, uint64_t groupId, size_t cap); int32_t buildCtbNameByGroupId(const char* stbName, uint64_t groupId, char** pName); int32_t buildCtbNameByGroupIdImpl(const char* stbName, uint64_t groupId, char* pBuf); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index d727be30ce..18f0df82be 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -2446,9 +2446,11 @@ _error: return NULL; } -static char* formatTimestamp(char* buf, int32_t bufSize, int64_t val, int precision) { +static int32_t formatTimestamp(char* buf, size_t cap, int64_t val, int precision) { time_t tt; int32_t ms = 0; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (precision == TSDB_TIME_PRECISION_NANO) { tt = (time_t)(val / 1000000000); ms = val % 1000000000; @@ -2460,14 +2462,6 @@ static char* formatTimestamp(char* buf, int32_t bufSize, int64_t val, int precis ms = val % 1000; } - /* comment out as it make testcases like select_with_tags.sim fail. - but in windows, this may cause the call to localtime crash if tt < 0, - need to find a better solution. - if (tt < 0) { - tt = 0; - } - */ - if (tt <= 0 && ms < 0) { tt--; if (precision == TSDB_TIME_PRECISION_NANO) { @@ -2479,20 +2473,35 @@ static char* formatTimestamp(char* buf, int32_t bufSize, int64_t val, int precis } } struct tm ptm = {0}; - if (taosLocalTime(&tt, &ptm, buf, bufSize) == NULL) { - return buf; + if (taosLocalTime(&tt, &ptm, buf, cap) == NULL) { + code = TSDB_CODE_INTERNAL_ERROR; + TSDB_CHECK_CODE(code, lino, _end); } - size_t pos = strftime(buf, bufSize, "%Y-%m-%d %H:%M:%S", &ptm); + size_t pos = strftime(buf, cap, "%Y-%m-%d %H:%M:%S", &ptm); + if (pos == 0) { + code = TSDB_CODE_OUT_OF_BUFFER; + TSDB_CHECK_CODE(code, lino, _end); + } + int32_t nwritten = 0; if (precision == TSDB_TIME_PRECISION_NANO) { - sprintf(buf + pos, ".%09d", ms); + nwritten = snprintf(buf + pos, cap - pos, ".%09d", ms); } else if (precision == TSDB_TIME_PRECISION_MICRO) { - sprintf(buf + pos, ".%06d", ms); + nwritten = snprintf(buf + pos, cap - pos, ".%06d", ms); } else { - sprintf(buf + pos, ".%03d", ms); + nwritten = snprintf(buf + pos, cap - pos, ".%03d", ms); } - return buf; + if (nwritten >= cap - pos) { + code = TSDB_CODE_OUT_OF_BUFFER; + TSDB_CHECK_CODE(code, lino, _end); + } + +_end: + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } // for debug @@ -2543,7 +2552,10 @@ int32_t dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf switch (pColInfoData->info.type) { case TSDB_DATA_TYPE_TIMESTAMP: memset(pBuf, 0, sizeof(pBuf)); - (void)formatTimestamp(pBuf, sizeof(pBuf), *(uint64_t*)var, pColInfoData->info.precision); + code = formatTimestamp(pBuf, sizeof(pBuf), *(uint64_t*)var, pColInfoData->info.precision); + if (code != TSDB_CODE_SUCCESS) { + snprintf(pBuf, sizeof(pBuf), "NaN"); + } len += snprintf(dumpBuf + len, size - len, " %25s |", pBuf); if (len >= size - 1) goto _exit; break; @@ -2857,27 +2869,98 @@ _end: return code; } -void buildCtbNameAddGroupId(const char* stbName, char* ctbName, uint64_t groupId) { - char tmp[TSDB_TABLE_NAME_LEN] = {0}; - if (stbName == NULL){ - snprintf(tmp, TSDB_TABLE_NAME_LEN, "_%"PRIu64, groupId); - }else{ +// Construct the child table name in the form of __ and store it in `ctbName`. +// If the name length exceeds TSDB_TABLE_NAME_LEN, first convert _ to an MD5 value and then +// concatenate. If the length is still too long, convert to an MD5 value as well. +int32_t buildCtbNameAddGroupId(const char* stbName, char* ctbName, uint64_t groupId, size_t cap) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + char tmp[TSDB_TABLE_NAME_LEN] = {0}; + char* suffix = tmp; + size_t suffixCap = sizeof(tmp); + size_t suffixLen = 0; + size_t prefixLen = 0; + T_MD5_CTX context; + + if (ctbName == NULL || cap < TSDB_TABLE_NAME_LEN) { + code = TSDB_CODE_INTERNAL_ERROR; + TSDB_CHECK_CODE(code, lino, _end); + } + + prefixLen = strlen(ctbName); + + if (stbName == NULL) { + suffixLen = snprintf(suffix, suffixCap, "%" PRIu64, groupId); + if (suffixLen >= suffixCap) { + code = TSDB_CODE_INTERNAL_ERROR; + TSDB_CHECK_CODE(code, lino, _end); + } + } else { int32_t i = strlen(stbName) - 1; - for(; i >= 0; i--){ - if (stbName[i] == '.'){ + for (; i >= 0; i--) { + if (stbName[i] == '.') { break; } } - snprintf(tmp, TSDB_TABLE_NAME_LEN, "_%s_%"PRIu64, stbName + i + 1, groupId); - } - - ctbName[TSDB_TABLE_NAME_LEN - strlen(tmp) - 1] = 0; // put stbname + groupId to the end - (void)strcat(ctbName, tmp); - for(int i = 0; i < strlen(ctbName); i++){ - if(ctbName[i] == '.'){ - ctbName[i] = '_'; + suffixLen = snprintf(suffix, suffixCap, "%s_%" PRIu64, stbName + i + 1, groupId); + if (suffixLen >= suffixCap) { + suffixCap = suffixLen + 1; + suffix = taosMemoryMalloc(suffixCap); + TSDB_CHECK_NULL(suffix, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY); + suffixLen = snprintf(suffix, suffixCap, "%s_%" PRIu64, stbName + i + 1, groupId); + if (suffixLen >= suffixCap) { + code = TSDB_CODE_INTERNAL_ERROR; + TSDB_CHECK_CODE(code, lino, _end); + } } } + + if (prefixLen + suffixLen + 1 >= TSDB_TABLE_NAME_LEN) { + // If the name length exceeeds the limit, convert the suffix to MD5 value. + tMD5Init(&context); + tMD5Update(&context, (uint8_t*)suffix, suffixLen); + tMD5Final(&context); + suffixLen = snprintf(suffix, suffixCap, "%016" PRIx64 "%016" PRIx64, *(uint64_t*)context.digest, + *(uint64_t*)(context.digest + 8)); + if (suffixLen >= suffixCap) { + code = TSDB_CODE_INTERNAL_ERROR; + TSDB_CHECK_CODE(code, lino, _end); + } + } + + if (prefixLen + suffixLen + 1 >= TSDB_TABLE_NAME_LEN) { + // If the name is still too long, convert the ctbName to MD5 value. + tMD5Init(&context); + tMD5Update(&context, (uint8_t*)ctbName, prefixLen); + tMD5Final(&context); + prefixLen = snprintf(ctbName, cap, "t_%016" PRIx64 "%016" PRIx64, *(uint64_t*)context.digest, + *(uint64_t*)(context.digest + 8)); + if (prefixLen >= cap) { + code = TSDB_CODE_INTERNAL_ERROR; + TSDB_CHECK_CODE(code, lino, _end); + } + } + + if (prefixLen + suffixLen + 1 >= TSDB_TABLE_NAME_LEN) { + code = TSDB_CODE_INTERNAL_ERROR; + TSDB_CHECK_CODE(code, lino, _end); + } + + ctbName[prefixLen] = '_'; + tstrncpy(&ctbName[prefixLen + 1], suffix, cap - prefixLen - 1); + + for (char* p = ctbName; *p; ++p) { + if (*p == '.') *p = '_'; + } + +_end: + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + if (suffix != tmp) { + taosMemoryFree(suffix); + } + return code; } // auto stream subtable name starts with 't_', followed by the first segment of MD5 digest for group vals. diff --git a/source/common/test/dataformatTest.cpp b/source/common/test/dataformatTest.cpp index e8b7b132f2..ebf91025bb 100644 --- a/source/common/test/dataformatTest.cpp +++ b/source/common/test/dataformatTest.cpp @@ -480,7 +480,7 @@ TEST(testCase, StreamAllNormTest) { char ctbName[TSDB_TABLE_NAME_LEN] = {0}; uint64_t groupId = 12345; - buildCtbNameAddGroupId(NULL, ctbName, groupId); + buildCtbNameAddGroupId(NULL, ctbName, groupId, sizeof(ctbName)); ASSERT_STREQ("_12345", ctbName); } @@ -490,7 +490,7 @@ TEST(testCase, StreamWithStbName) { char ctbName[TSDB_TABLE_NAME_LEN] = {0}; uint64_t groupId = 12345; - buildCtbNameAddGroupId(stbName, ctbName, groupId); + buildCtbNameAddGroupId(stbName, ctbName, groupId, sizeof(ctbName)); ASSERT_STREQ("_stb_12345", ctbName); } @@ -500,7 +500,7 @@ TEST(testCase, StreamWithoutDotInStbName) { char ctbName[TSDB_TABLE_NAME_LEN] = {0}; uint64_t groupId = 12345; - buildCtbNameAddGroupId(stbName, ctbName, groupId); + buildCtbNameAddGroupId(stbName, ctbName, groupId, sizeof(ctbName)); ASSERT_STREQ("_table_12345", ctbName); } @@ -510,11 +510,59 @@ TEST(testCase, StreamWithoutDotInStbName2) { char ctbName[TSDB_TABLE_NAME_LEN] = {0}; uint64_t groupId = 12345; - buildCtbNameAddGroupId(stbName, ctbName, groupId); + buildCtbNameAddGroupId(stbName, ctbName, groupId, sizeof(ctbName)); ASSERT_STREQ("__12345", ctbName); } +TEST(testCase, StreamWithLongStbName) { + char ctbName[TSDB_TABLE_NAME_LEN]; + char expectName[TSDB_TABLE_NAME_LEN]; + char *stbName = "a_simle_stb_name"; + uint64_t groupId = UINT64_MAX; + + // test basic function + strcpy(ctbName, "a_simple_ctb_name"); + EXPECT_EQ(buildCtbNameAddGroupId(stbName, ctbName, groupId, sizeof(ctbName)), TSDB_CODE_SUCCESS); + EXPECT_STREQ(ctbName, "a_simple_ctb_name_a_simle_stb_name_18446744073709551615"); + + // test null stbName + strcpy(ctbName, "a_simple_ctb_name"); + stbName = NULL; + EXPECT_EQ(buildCtbNameAddGroupId(stbName, ctbName, groupId, sizeof(ctbName)), TSDB_CODE_SUCCESS); + EXPECT_STREQ(ctbName, "a_simple_ctb_name_18446744073709551615"); + + // test buffer capcity check + EXPECT_EQ(buildCtbNameAddGroupId(stbName, NULL, groupId, sizeof(ctbName)), TSDB_CODE_INTERNAL_ERROR); + EXPECT_EQ(buildCtbNameAddGroupId(stbName, ctbName, groupId, sizeof(ctbName) - 1), TSDB_CODE_INTERNAL_ERROR); + + // test md5 conversion of stbName with groupid + for (int32_t i = 0; i < 159; ++i) ctbName[i] = 'A'; + ctbName[159] = '\0'; + stbName = taosStrdup(ctbName); + snprintf(expectName, TSDB_TABLE_NAME_LEN, "%s_d85f0d87946d76eeedd7b7b78b7492a2", ctbName); + EXPECT_EQ(buildCtbNameAddGroupId(stbName, ctbName, groupId, sizeof(ctbName)), TSDB_CODE_SUCCESS); + EXPECT_STREQ(ctbName, expectName); + + // test md5 conversion of all parts + for (int32_t i = 0; i < 190; ++i) ctbName[i] = 'A'; + ctbName[190] = '\0'; + tstrncpy(expectName, "t_d38a8b2df999bef0082ffc80a59a9cd7_d85f0d87946d76eeedd7b7b78b7492a2", TSDB_TABLE_NAME_LEN); + EXPECT_EQ(buildCtbNameAddGroupId(stbName, ctbName, groupId, sizeof(ctbName)), TSDB_CODE_SUCCESS); + EXPECT_STREQ(ctbName, expectName); + + // test larger stbName + taosMemoryFree(stbName); + for (int32_t i = 0; i < 190; ++i) ctbName[i] = 'A'; + ctbName[190] = '\0'; + stbName = taosStrdup(ctbName); + tstrncpy(expectName, "t_d38a8b2df999bef0082ffc80a59a9cd7_9c99cc7c52073b63fb750af402d9b84b", TSDB_TABLE_NAME_LEN); + EXPECT_EQ(buildCtbNameAddGroupId(stbName, ctbName, groupId, sizeof(ctbName)), TSDB_CODE_SUCCESS); + EXPECT_STREQ(ctbName, expectName); + + taosMemoryFree(stbName); +} + #if 1 TEST(testCase, NoneTest) { const static int nCols = 14; @@ -570,4 +618,4 @@ for (int r = 0; r < nRows; ++r) { taosArrayDestroy(pArray); taosMemoryFree(pTSchema); } -#endif \ No newline at end of file +#endif diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index b18fa42cae..d8f10e6b6e 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -73,14 +73,19 @@ int32_t tqBuildDeleteReq(STQ* pTq, const char* stbFullName, const SSDataBlock* p } if (varTbName != NULL && varTbName != (void*)-1) { - name = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN); + size_t cap = TMAX(TSDB_TABLE_NAME_LEN, varDataLen(varTbName) + 1); + name = taosMemoryMalloc(cap); if (name == NULL) { return terrno; } memcpy(name, varDataVal(varTbName), varDataLen(varTbName)); + name[varDataLen(varTbName)] = '\0'; if (newSubTableRule && !isAutoTableName(name) && !alreadyAddGroupId(name, groupId) && groupId != 0 && stbFullName) { - buildCtbNameAddGroupId(stbFullName, name, groupId); + int32_t code = buildCtbNameAddGroupId(stbFullName, name, groupId, cap); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } } else if (stbFullName) { int32_t code = buildCtbNameByGroupId(stbFullName, groupId, &name); @@ -236,7 +241,10 @@ int32_t setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* } strcpy(pCreateTableReq->name, pDataBlock->info.parTbName); - buildCtbNameAddGroupId(stbFullName, pCreateTableReq->name, gid); + int32_t code = buildCtbNameAddGroupId(stbFullName, pCreateTableReq->name, gid, TSDB_TABLE_NAME_LEN); + if (code != TSDB_CODE_SUCCESS) { + return code; + } // tqDebug("gen name from:%s", pDataBlock->info.parTbName); } else { pCreateTableReq->name = taosStrdup(pDataBlock->info.parTbName); @@ -852,9 +860,12 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat !alreadyAddGroupId(dstTableName, groupId) && groupId != 0) { tqDebug("s-task:%s append groupId:%" PRId64 " for generated dstTable:%s", id, groupId, dstTableName); if (pTask->ver == SSTREAM_TASK_SUBTABLE_CHANGED_VER) { - buildCtbNameAddGroupId(NULL, dstTableName, groupId); + code = buildCtbNameAddGroupId(NULL, dstTableName, groupId, sizeof(pDataBlock->info.parTbName)); } else if (pTask->ver > SSTREAM_TASK_SUBTABLE_CHANGED_VER && stbFullName) { - buildCtbNameAddGroupId(stbFullName, dstTableName, groupId); + code = buildCtbNameAddGroupId(stbFullName, dstTableName, groupId, sizeof(pDataBlock->info.parTbName)); + } + if (code != TSDB_CODE_SUCCESS) { + return code; } } } @@ -1250,4 +1261,4 @@ int32_t doBuildAndSendDeleteMsg(SVnode* pVnode, char* stbFullName, SSDataBlock* } return TSDB_CODE_SUCCESS; -} \ No newline at end of file +} diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 78cbd844a0..133663ac28 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -706,9 +706,13 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S if (pTask->subtableWithoutMd5 != 1 && !isAutoTableName(pDataBlock->info.parTbName) && !alreadyAddGroupId(pDataBlock->info.parTbName, groupId) && groupId != 0) { if (pTask->ver == SSTREAM_TASK_SUBTABLE_CHANGED_VER) { - buildCtbNameAddGroupId(NULL, pDataBlock->info.parTbName, groupId); + code = buildCtbNameAddGroupId(NULL, pDataBlock->info.parTbName, groupId, sizeof(pDataBlock->info.parTbName)); } else if (pTask->ver > SSTREAM_TASK_SUBTABLE_CHANGED_VER) { - buildCtbNameAddGroupId(pTask->outputInfo.shuffleDispatcher.stbFullName, pDataBlock->info.parTbName, groupId); + code = buildCtbNameAddGroupId(pTask->outputInfo.shuffleDispatcher.stbFullName, pDataBlock->info.parTbName, + groupId, sizeof(pDataBlock->info.parTbName)); + } + if (code != TSDB_CODE_SUCCESS) { + return code; } } } else {