diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index 3e2e879e38..ab1609f35f 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG a2e9920 + GIT_TAG 69eee2e SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/docs/examples/csharp/stmtInsert/Program.cs b/docs/examples/csharp/stmtInsert/Program.cs index 87e1971feb..80cadb2ff8 100644 --- a/docs/examples/csharp/stmtInsert/Program.cs +++ b/docs/examples/csharp/stmtInsert/Program.cs @@ -42,7 +42,7 @@ namespace TDengineExample // 5. execute res = TDengine.StmtExecute(stmt); - CheckStmtRes(res, "faild to execute"); + CheckStmtRes(res, "failed to execute"); // 6. free TaosMultiBind.FreeTaosBind(tags); @@ -92,7 +92,7 @@ namespace TDengineExample int code = TDengine.StmtClose(stmt); if (code != 0) { - throw new Exception($"falied to close stmt, {code} reason: {TDengine.StmtErrorStr(stmt)} "); + throw new Exception($"failed to close stmt, {code} reason: {TDengine.StmtErrorStr(stmt)} "); } } } diff --git a/docs/examples/python/connect_rest_examples.py b/docs/examples/python/connect_rest_examples.py index 900ec1022e..dba00b5a82 100644 --- a/docs/examples/python/connect_rest_examples.py +++ b/docs/examples/python/connect_rest_examples.py @@ -15,10 +15,10 @@ cursor.execute("CREATE DATABASE power") cursor.execute("CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)") # insert data -cursor.execute("""INSERT INTO power.d1001 USING power.meters TAGS(California.SanFrancisco, 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000) - power.d1002 USING power.meters TAGS(California.SanFrancisco, 3) VALUES ('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000) - power.d1003 USING power.meters TAGS(California.LosAngeles, 2) VALUES ('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000) ('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) - power.d1004 USING power.meters TAGS(California.LosAngeles, 3) VALUES ('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000) ('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000)""") +cursor.execute("""INSERT INTO power.d1001 USING power.meters TAGS('California.SanFrancisco', 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000) + power.d1002 USING power.meters TAGS('California.SanFrancisco', 3) VALUES ('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000) + power.d1003 USING power.meters TAGS('California.LosAngeles', 2) VALUES ('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000) ('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) + power.d1004 USING power.meters TAGS('California.LosAngeles', 3) VALUES ('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000) ('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000)""") print("inserted row count:", cursor.rowcount) # query data diff --git a/docs/examples/python/native_insert_example.py b/docs/examples/python/native_insert_example.py index 94fd00a6e9..cdde7d23d2 100644 --- a/docs/examples/python/native_insert_example.py +++ b/docs/examples/python/native_insert_example.py @@ -25,10 +25,10 @@ def create_stable(conn: taos.TaosConnection): # The generated SQL is: -# INSERT INTO d1001 USING meters TAGS(California.SanFrancisco, 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000) -# d1002 USING meters TAGS(California.SanFrancisco, 3) VALUES ('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000) -# d1003 USING meters TAGS(California.LosAngeles, 2) VALUES ('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000) ('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) -# d1004 USING meters TAGS(California.LosAngeles, 3) VALUES ('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000) ('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000) +# INSERT INTO d1001 USING meters TAGS('California.SanFrancisco', 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000) +# d1002 USING meters TAGS('California.SanFrancisco', 3) VALUES ('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000) +# d1003 USING meters TAGS('California.LosAngeles', 2) VALUES ('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000) ('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) +# d1004 USING meters TAGS('California.LosAngeles', 3) VALUES ('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000) ('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000) def get_sql(): global lines diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index d9b8ae266b..20ffb48ab0 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -41,6 +41,12 @@ typedef struct SBlockOrderInfo { BMCharPos(bm_, r_) |= (1u << (7u - BitPos(r_))); \ } while (0) +#define colDataSetNull_f_s(c_, r_) \ + do { \ + colDataSetNull_f((c_)->nullbitmap, r_); \ + memset(((char*)(c_)->pData) + (c_)->info.bytes * (r_), 0, (c_)->info.bytes); \ + } while (0) + #define colDataClearNull_f(bm_, r_) \ do { \ BMCharPos(bm_, r_) &= ((char)(~(1u << (7u - BitPos(r_))))); \ @@ -136,7 +142,7 @@ static FORCE_INLINE void colDataAppendNULL(SColumnInfoData* pColumnInfoData, uin if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { colDataSetNull_var(pColumnInfoData, currentRow); // it is a null value of VAR type. } else { - colDataSetNull_f(pColumnInfoData->nullbitmap, currentRow); + colDataSetNull_f_s(pColumnInfoData, currentRow); } pColumnInfoData->hasNull = true; @@ -151,6 +157,7 @@ static FORCE_INLINE void colDataAppendNNULL(SColumnInfoData* pColumnInfoData, ui for (int32_t i = start; i < start + nRows; ++i) { colDataSetNull_f(pColumnInfoData->nullbitmap, i); } + memset(pColumnInfoData->pData + start * pColumnInfoData->info.bytes, 0, pColumnInfoData->info.bytes * nRows); } pColumnInfoData->hasNull = true; @@ -231,7 +238,6 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows, bool clearPayload); int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows); -int32_t blockDataEnsureCapacityNoClear(SSDataBlock* pDataBlock, uint32_t numOfRows); void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows); void blockDataCleanup(SSDataBlock* pDataBlock); diff --git a/include/common/tmsgcb.h b/include/common/tmsgcb.h index a1ebd855cd..7cf092b144 100644 --- a/include/common/tmsgcb.h +++ b/include/common/tmsgcb.h @@ -39,7 +39,7 @@ typedef enum { QUEUE_MAX, } EQueueType; -typedef int32_t (*UpdateDnodeInfoFp)(void* pData, int32_t* dnodeId, int64_t* clusterId, char* fqdn, uint16_t* port); +typedef void (*UpdateDnodeInfoFp)(void* pData, int32_t* dnodeId, int64_t* clusterId, char* fqdn, uint16_t* port); typedef int32_t (*PutToQueueFp)(void* pMgmt, EQueueType qtype, SRpcMsg* pMsg); typedef int32_t (*GetQueueSizeFp)(void* pMgmt, int32_t vgId, EQueueType qtype); typedef int32_t (*SendReqFp)(const SEpSet* pEpSet, SRpcMsg* pMsg); @@ -70,7 +70,7 @@ void tmsgSendRsp(SRpcMsg* pMsg); void tmsgRegisterBrokenLinkArg(SRpcMsg* pMsg); void tmsgReleaseHandle(SRpcHandleInfo* pHandle, int8_t type); void tmsgReportStartup(const char* name, const char* desc); -int32_t tmsgUpdateDnodeInfo(int32_t* dnodeId, int64_t* clusterId, char* fqdn, uint16_t* port); +void tmsgUpdateDnodeInfo(int32_t* dnodeId, int64_t* clusterId, char* fqdn, uint16_t* port); #ifdef __cplusplus } diff --git a/include/util/tdef.h b/include/util/tdef.h index f2e2ab4284..9036befc02 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -500,7 +500,7 @@ enum { #define DEFAULT_PAGESIZE 4096 #define VNODE_TIMEOUT_SEC 60 -#define MNODE_TIMEOUT_SEC 10 +#define MNODE_TIMEOUT_SEC 60 #ifdef __cplusplus } diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 8bf646486d..28a55156aa 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -428,7 +428,11 @@ _return: taosLogCrashInfo("taos", pMsg, msgLen, signum, sigInfo); +#ifdef _TD_DARWIN_64 exit(signum); +#elif defined(WINDOWS) + exit(signum); +#endif } void crashReportThreadFuncUnexpectedStopped(void) { atomic_store_32(&clientStop, -1); } diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 8c4a97443c..f25456999b 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -521,7 +521,7 @@ STableMeta *smlGetMeta(SSmlHandle *info, const void *measure, int32_t measureLen memcpy(pName.tname, measure, measureLen); int32_t code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); - if(code != TSDB_CODE_SUCCESS){ + if (code != TSDB_CODE_SUCCESS) { return NULL; } return pTableMeta; @@ -996,7 +996,7 @@ static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols } } else { size_t tmp = taosArrayGetSize(metaArray); - if(tmp > INT16_MAX){ + if (tmp > INT16_MAX) { uError("too many cols or tags"); return -1; } @@ -1017,9 +1017,9 @@ void smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag) { taosHashCleanup(kvHash); } - if(info->parseJsonByLib){ + if (info->parseJsonByLib) { SSmlLineInfo *key = (SSmlLineInfo *)(tag->key); - if(key != NULL) taosMemoryFree(key->tags); + if (key != NULL) taosMemoryFree(key->tags); } taosMemoryFree(tag->key); taosArrayDestroy(tag->cols); @@ -1027,10 +1027,10 @@ void smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag) { taosMemoryFree(tag); } -void clearColValArray(SArray* pCols) { +void clearColValArray(SArray *pCols) { int32_t num = taosArrayGetSize(pCols); for (int32_t i = 0; i < num; ++i) { - SColVal* pCol = taosArrayGet(pCols, i); + SColVal *pCol = taosArrayGet(pCols, i); if (TSDB_DATA_TYPE_NCHAR == pCol->type) { taosMemoryFreeClear(pCol->value.pData); } @@ -1066,7 +1066,7 @@ void smlDestroyInfo(SSmlHandle *info) { // destroy info->pVgHash taosHashCleanup(info->pVgHash); - for(int i = 0; i< taosArrayGetSize(info->tagJsonArray); i++){ + for (int i = 0; i < taosArrayGetSize(info->tagJsonArray); i++) { cJSON *tags = (cJSON *)taosArrayGetP(info->tagJsonArray, i); cJSON_Delete(tags); } @@ -1079,7 +1079,7 @@ void smlDestroyInfo(SSmlHandle *info) { if (!info->dataFormat) { for (int i = 0; i < info->lineNum; i++) { taosArrayDestroy(info->lines[i].colArray); - if(info->parseJsonByLib){ + if (info->parseJsonByLib) { taosMemoryFree(info->lines[i].tags); } } @@ -1232,7 +1232,7 @@ static int32_t smlInsertData(SSmlHandle *info) { SSmlSTableMeta *pMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, tableData->sTableName, tableData->sTableNameLen, NULL); - if(unlikely(NULL == pMeta || NULL == pMeta->tableMeta)){ + if (unlikely(NULL == pMeta || NULL == pMeta->tableMeta)) { uError("SML:0x%" PRIx64 " NULL == pMeta. table name: %s", info->id, tableData->childTableName); return TSDB_CODE_SML_INTERNAL_ERROR; } @@ -1300,7 +1300,7 @@ int32_t smlClearForRerun(SSmlHandle *info) { pList = pList->next; } - if (!info->dataFormat){ + if (!info->dataFormat) { if (unlikely(info->lines != NULL)) { uError("SML:0x%" PRIx64 " info->lines != NULL", info->id); return TSDB_CODE_SML_INVALID_DATA; diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 60a673ef9c..6c86743b69 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -67,6 +67,8 @@ static const SSysDbTableSchema clusterSchema[] = { {.name = "name", .bytes = TSDB_CLUSTER_ID_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "uptime", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true}, + {.name = "version", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "expire_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true}, }; static const SSysDbTableSchema userDBSchema[] = { diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index f09cf9e630..9b5b32cf69 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -70,7 +70,7 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { pColumnInfoData->varmeta.offset[currentRow] = -1; // it is a null value of VAR type. } else { - colDataSetNull_f(pColumnInfoData->nullbitmap, currentRow); + colDataSetNull_f_s(pColumnInfoData, currentRow); } pColumnInfoData->hasNull = true; @@ -315,8 +315,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows, const SDataBlockInfo* pBlockInfo) { - if (pColumnInfoData->info.type != pSource->info.type || - (pBlockInfo != NULL && pBlockInfo->capacity < numOfRows)) { + if (pColumnInfoData->info.type != pSource->info.type || (pBlockInfo != NULL && pBlockInfo->capacity < numOfRows)) { return TSDB_CODE_FAILED; } @@ -826,7 +825,7 @@ static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataB } else { for (int32_t j = 0; j < pDataBlock->info.rows; ++j) { if (colDataIsNull_f(pSrc->nullbitmap, index[j])) { - colDataSetNull_f(pDst->nullbitmap, j); + colDataSetNull_f_s(pDst, j); continue; } memcpy(pDst->pData + j * pDst->info.bytes, pSrc->pData + index[j] * pDst->info.bytes, pDst->info.bytes); @@ -1162,16 +1161,17 @@ void blockDataEmpty(SSDataBlock* pDataBlock) { pInfo->window.skey = 0; } -// todo temporarily disable it +/* + * NOTE: the type of the input column may be TSDB_DATA_TYPE_NULL, which is used to denote + * the all NULL value in this column. It is an internal representation of all NULL value column, and no visible to + * any users. The length of TSDB_DATA_TYPE_NULL is 0, and it is an special case. + */ static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* pBlockInfo, uint32_t numOfRows, bool clearPayload) { if (numOfRows <= 0 || numOfRows <= pBlockInfo->capacity) { return TSDB_CODE_SUCCESS; } - // todo temp disable it - // ASSERT(pColumn->info.bytes != 0); - int32_t existedRows = pBlockInfo->rows; if (IS_VAR_DATA_TYPE(pColumn->info.type)) { @@ -1196,7 +1196,8 @@ static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* return TSDB_CODE_FAILED; } - // make sure the allocated memory is MALLOC_ALIGN_BYTES aligned + // here we employ the aligned malloc function, to make sure that the address of allocated memory is aligned + // to MALLOC_ALIGN_BYTES tmp = taosMemoryMallocAlign(MALLOC_ALIGN_BYTES, numOfRows * pColumn->info.bytes); if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; @@ -1210,7 +1211,7 @@ static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* pColumn->pData = tmp; - // todo remove it soon + // check if the allocated memory is aligned to the requried bytes. #if defined LINUX if ((((uint64_t)pColumn->pData) & (MALLOC_ALIGN_BYTES - 1)) != 0x0) { return TSDB_CODE_FAILED; @@ -1251,25 +1252,6 @@ int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) { return TSDB_CODE_SUCCESS; } - size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); - code = doEnsureCapacity(p, &pDataBlock->info, numOfRows, true); - if (code) { - return code; - } - } - - pDataBlock->info.capacity = numOfRows; - return TSDB_CODE_SUCCESS; -} - -int32_t blockDataEnsureCapacityNoClear(SSDataBlock* pDataBlock, uint32_t numOfRows) { - int32_t code = 0; - if (numOfRows == 0 || numOfRows <= pDataBlock->info.capacity) { - return TSDB_CODE_SUCCESS; - } - size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); @@ -1963,7 +1945,7 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) "|rows:%d|version:%" PRIu64 "|cal start:%" PRIu64 "|cal end:%" PRIu64 "|tbl:%s\n", flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId, pDataBlock->info.id.groupId, pDataBlock->info.id.uid, pDataBlock->info.rows, pDataBlock->info.version, - pDataBlock->info.calWin.skey, pDataBlock->info.calWin.ekey, pDataBlock->info.parTbName); + pDataBlock->info.calWin.skey, pDataBlock->info.calWin.ekey, pDataBlock->info.parTbName); if (len >= size - 1) return dumpBuf; for (int32_t j = 0; j < rows; j++) { diff --git a/source/common/src/tname.c b/source/common/src/tname.c index e45cec6560..e5ed7a3728 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -160,9 +160,7 @@ int32_t tNameGetFullDbName(const SName* name, char* dst) { return 0; } -bool tNameIsEmpty(const SName* name) { - return name->type == 0 || name->acctId == 0; -} +bool tNameIsEmpty(const SName* name) { return name->type == 0 || name->acctId == 0; } const char* tNameGetTableName(const SName* name) { ASSERT(name != NULL && name->type == TSDB_TABLE_NAME_T); @@ -328,5 +326,4 @@ void buildChildTableName(RandTableName* rName) { strcat(rName->ctbShortName, temp); } taosStringBuilderDestroy(&sb); -// rName->uid = *(uint64_t*)(context.digest); } diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index 711280ea58..4910b0ac3f 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -100,7 +100,11 @@ _return: taosLogCrashInfo("taosd", pMsg, msgLen, signum, sigInfo); +#ifdef _TD_DARWIN_64 exit(signum); +#elif defined(WINDOWS) + exit(signum); +#endif } static void dmSetSignalHandle() { diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c index dd05fe673a..f06669a610 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c @@ -17,117 +17,97 @@ #include "mmInt.h" #include "tjson.h" -int32_t mmReadFile(const char *path, SMnodeOpt *pOption) { - int32_t code = TSDB_CODE_INVALID_JSON_FORMAT; - int32_t len = 0; - int32_t maxLen = 4096; - char *content = taosMemoryCalloc(1, maxLen + 1); - cJSON *root = NULL; - char file[PATH_MAX] = {0}; - TdFilePtr pFile = NULL; +static int32_t mmDecodeOption(SJson *pJson, SMnodeOpt *pOption) { + int32_t code = 0; + tjsonGetInt32ValueFromDouble(pJson, "deployed", pOption->deploy, code); + if (code < 0) return -1; + tjsonGetInt32ValueFromDouble(pJson, "selfIndex", pOption->selfIndex, code); + if (code < 0) return 0; + + SJson *replicas = tjsonGetObjectItem(pJson, "replicas"); + if (replicas == NULL) return 0; + pOption->numOfReplicas = tjsonGetArraySize(replicas); + + for (int32_t i = 0; i < pOption->numOfReplicas; ++i) { + SJson *replica = tjsonGetArrayItem(replicas, i); + if (replica == NULL) return -1; + + SReplica *pReplica = pOption->replicas + i; + tjsonGetInt32ValueFromDouble(replica, "id", pReplica->id, code); + if (code < 0) return -1; + code = tjsonGetStringValue(replica, "fqdn", pReplica->fqdn); + if (code < 0) return -1; + tjsonGetUInt16ValueFromDouble(replica, "port", pReplica->port, code); + if (code < 0) return -1; + } + + return 0; +} + +int32_t mmReadFile(const char *path, SMnodeOpt *pOption) { + int32_t code = -1; + TdFilePtr pFile = NULL; + char *pData = NULL; + SJson *pJson = NULL; + char file[PATH_MAX] = {0}; snprintf(file, sizeof(file), "%s%smnode.json", path, TD_DIRSEP); + + if (taosStatFile(file, NULL, NULL) < 0) { + dInfo("mnode file:%s not exist", file); + return 0; + } + pFile = taosOpenFile(file, TD_FILE_READ); if (pFile == NULL) { - code = 0; + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to open mnode file:%s since %s", file, terrstr()); goto _OVER; } - len = (int32_t)taosReadFile(pFile, content, maxLen); - if (len <= 0) { - dError("failed to read %s since content is null", file); + int64_t size = 0; + if (taosFStatFile(pFile, &size, NULL) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to fstat mnode file:%s since %s", file, terrstr()); goto _OVER; } - content[len] = 0; - root = cJSON_Parse(content); - if (root == NULL) { - dError("failed to read %s since invalid json format", file); + pData = taosMemoryMalloc(size + 1); + if (pData == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } - cJSON *deployed = cJSON_GetObjectItem(root, "deployed"); - if (!deployed || deployed->type != cJSON_Number) { - dError("failed to read %s since deployed not found", file); + if (taosReadFile(pFile, pData, size) != size) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to read mnode file:%s since %s", file, terrstr()); goto _OVER; } - pOption->deploy = deployed->valueint; - cJSON *selfIndex = cJSON_GetObjectItem(root, "selfIndex"); - if (selfIndex) { - if (selfIndex->type != cJSON_Number) { - dError("failed to read %s since selfIndex not found", file); - goto _OVER; - } - pOption->selfIndex = selfIndex->valueint; + pData[size] = '\0'; + + pJson = tjsonParse(pData); + if (pJson == NULL) { + terrno = TSDB_CODE_INVALID_JSON_FORMAT; + goto _OVER; } - cJSON *replicas = cJSON_GetObjectItem(root, "replicas"); - if (replicas) { - if (replicas->type != cJSON_Array) { - dError("failed to read %s since replicas not found", file); - goto _OVER; - } - - int32_t numOfReplicas = cJSON_GetArraySize(replicas); - if (numOfReplicas <= 0) { - dError("failed to read %s since numOfReplicas:%d invalid", file, numOfReplicas); - goto _OVER; - } - pOption->numOfReplicas = numOfReplicas; - - for (int32_t i = 0; i < numOfReplicas; ++i) { - SReplica *pReplica = pOption->replicas + i; - - cJSON *replica = cJSON_GetArrayItem(replicas, i); - if (replica == NULL) break; - - cJSON *id = cJSON_GetObjectItem(replica, "id"); - if (id) { - if (id->type != cJSON_Number) { - dError("failed to read %s since id not found", file); - goto _OVER; - } - if (pReplica) { - pReplica->id = id->valueint; - } - } - - cJSON *fqdn = cJSON_GetObjectItem(replica, "fqdn"); - if (fqdn) { - if (fqdn->type != cJSON_String || fqdn->valuestring == NULL) { - dError("failed to read %s since fqdn not found", file); - goto _OVER; - } - if (pReplica) { - tstrncpy(pReplica->fqdn, fqdn->valuestring, TSDB_FQDN_LEN); - } - } - - cJSON *port = cJSON_GetObjectItem(replica, "port"); - if (port) { - if (port->type != cJSON_Number) { - dError("failed to read %s since port not found", file); - goto _OVER; - } - if (pReplica) { - pReplica->port = (uint16_t)port->valueint; - } - } - } + if (mmDecodeOption(pJson, pOption) < 0) { + terrno = TSDB_CODE_INVALID_JSON_FORMAT; + goto _OVER; } code = 0; + dInfo("succceed to read mnode file %s", file); _OVER: - if (content != NULL) taosMemoryFree(content); - if (root != NULL) cJSON_Delete(root); + if (pData != NULL) taosMemoryFree(pData); + if (pJson != NULL) cJSON_Delete(pJson); if (pFile != NULL) taosCloseFile(&pFile); - if (code == 0) { - dDebug("succcessed to read file %s, deployed:%d", file, pOption->deploy); - } - terrno = code; + if (code != 0) { + dError("failed to read mnode file:%s since %s", file, terrstr()); + } return code; } diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c index 8337fb5d10..bf176ebb40 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c @@ -14,8 +14,8 @@ */ #define _DEFAULT_SOURCE -#include "vmInt.h" #include "tjson.h" +#include "vmInt.h" #define MAX_CONTENT_LEN 2 * 1024 * 1024 @@ -46,102 +46,108 @@ SVnodeObj **vmGetVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes) { return pVnodes; } -int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *numOfVnodes) { - int32_t code = TSDB_CODE_INVALID_JSON_FORMAT; - int32_t len = 0; - int32_t maxLen = MAX_CONTENT_LEN; - char *content = taosMemoryCalloc(1, maxLen + 1); - cJSON *root = NULL; - FILE *fp = NULL; - char file[PATH_MAX] = {0}; +static int32_t vmDecodeVnodeList(SJson *pJson, SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *numOfVnodes) { + int32_t code = -1; SWrapperCfg *pCfgs = NULL; - TdFilePtr pFile = NULL; + *ppCfgs = NULL; - snprintf(file, sizeof(file), "%s%svnodes.json", pMgmt->path, TD_DIRSEP); - - pFile = taosOpenFile(file, TD_FILE_READ); - if (pFile == NULL) { - dInfo("file %s not exist", file); - code = 0; - goto _OVER; - } - - if (content == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - len = (int32_t)taosReadFile(pFile, content, maxLen); - if (len <= 0) { - dError("failed to read %s since content is null", file); - goto _OVER; - } - - content[len] = 0; - root = cJSON_Parse(content); - if (root == NULL) { - dError("failed to read %s since invalid json format", file); - goto _OVER; - } - - cJSON *vnodes = cJSON_GetObjectItem(root, "vnodes"); - if (!vnodes || vnodes->type != cJSON_Array) { - dError("failed to read %s since vnodes not found", file); - goto _OVER; - } + SJson *vnodes = tjsonGetObjectItem(pJson, "vnodes"); + if (vnodes == NULL) return -1; int32_t vnodesNum = cJSON_GetArraySize(vnodes); if (vnodesNum > 0) { pCfgs = taosMemoryCalloc(vnodesNum, sizeof(SWrapperCfg)); - if (pCfgs == NULL) { - dError("failed to read %s since out of memory", file); - code = TSDB_CODE_OUT_OF_MEMORY; - goto _OVER; - } - - for (int32_t i = 0; i < vnodesNum; ++i) { - cJSON *vnode = cJSON_GetArrayItem(vnodes, i); - SWrapperCfg *pCfg = &pCfgs[i]; - - cJSON *vgId = cJSON_GetObjectItem(vnode, "vgId"); - if (!vgId || vgId->type != cJSON_Number) { - dError("failed to read %s since vgId not found", file); - taosMemoryFree(pCfgs); - goto _OVER; - } - pCfg->vgId = vgId->valueint; - snprintf(pCfg->path, sizeof(pCfg->path), "%s%svnode%d", pMgmt->path, TD_DIRSEP, pCfg->vgId); - - cJSON *dropped = cJSON_GetObjectItem(vnode, "dropped"); - if (!dropped || dropped->type != cJSON_Number) { - dError("failed to read %s since dropped not found", file); - taosMemoryFree(pCfgs); - goto _OVER; - } - pCfg->dropped = dropped->valueint; - - cJSON *vgVersion = cJSON_GetObjectItem(vnode, "vgVersion"); - if (!vgVersion || vgVersion->type != cJSON_Number) { - dError("failed to read %s since vgVersion not found", file); - taosMemoryFree(pCfgs); - goto _OVER; - } - pCfg->vgVersion = vgVersion->valueint; - } - - *ppCfgs = pCfgs; + if (pCfgs == NULL) return -1; + } + + for (int32_t i = 0; i < vnodesNum; ++i) { + SJson *vnode = tjsonGetArrayItem(vnodes, i); + if (vnode == NULL) goto _OVER; + + SWrapperCfg *pCfg = &pCfgs[i]; + tjsonGetInt32ValueFromDouble(vnode, "vgId", pCfg->vgId, code); + if (code < 0) goto _OVER; + tjsonGetInt32ValueFromDouble(vnode, "dropped", pCfg->dropped, code); + if (code < 0) goto _OVER; + tjsonGetInt32ValueFromDouble(vnode, "vgVersion", pCfg->vgVersion, code); + if (code < 0) goto _OVER; + + snprintf(pCfg->path, sizeof(pCfg->path), "%s%svnode%d", pMgmt->path, TD_DIRSEP, pCfg->vgId); } - *numOfVnodes = vnodesNum; code = 0; - dInfo("succcessed to read file %s, numOfVnodes:%d", file, vnodesNum); + *ppCfgs = pCfgs; + *numOfVnodes = vnodesNum; _OVER: - if (content != NULL) taosMemoryFree(content); - if (root != NULL) cJSON_Delete(root); + if (*ppCfgs == NULL) taosMemoryFree(pCfgs); + return code; +} + +int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *numOfVnodes) { + int32_t code = -1; + TdFilePtr pFile = NULL; + char *pData = NULL; + SJson *pJson = NULL; + char file[PATH_MAX] = {0}; + SWrapperCfg *pCfgs = NULL; + snprintf(file, sizeof(file), "%s%svnodes.json", pMgmt->path, TD_DIRSEP); + + if (taosStatFile(file, NULL, NULL) < 0) { + dInfo("vnode file:%s not exist", file); + return 0; + } + + pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to open vnode file:%s since %s", file, terrstr()); + goto _OVER; + } + + int64_t size = 0; + if (taosFStatFile(pFile, &size, NULL) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to fstat mnode file:%s since %s", file, terrstr()); + goto _OVER; + } + + pData = taosMemoryMalloc(size + 1); + if (pData == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _OVER; + } + + if (taosReadFile(pFile, pData, size) != size) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to read vnode file:%s since %s", file, terrstr()); + goto _OVER; + } + + pData[size] = '\0'; + + pJson = tjsonParse(pData); + if (pJson == NULL) { + terrno = TSDB_CODE_INVALID_JSON_FORMAT; + goto _OVER; + } + + if (vmDecodeVnodeList(pJson, pMgmt, ppCfgs, numOfVnodes) < 0) { + terrno = TSDB_CODE_INVALID_JSON_FORMAT; + goto _OVER; + } + + code = 0; + dInfo("succceed to read vnode file %s", file); + +_OVER: + if (pData != NULL) taosMemoryFree(pData); + if (pJson != NULL) cJSON_Delete(pJson); if (pFile != NULL) taosCloseFile(&pFile); - terrno = code; + if (code != 0) { + dError("failed to read vnode file:%s since %s", file, terrstr()); + } return code; } diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index e2887d979c..47772acbdc 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -137,7 +137,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { pNode->nodeId = pCreate->replicas[i].id; pNode->nodePort = pCreate->replicas[i].port; tstrncpy(pNode->nodeFqdn, pCreate->replicas[i].fqdn, TSDB_FQDN_LEN); - (void)tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); + tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); } } diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index 693fe97daa..99ba9b9b3b 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -343,13 +343,12 @@ static void vmCheckSyncTimeout(SVnodeMgmt *pMgmt) { int32_t numOfVnodes = 0; SVnodeObj **ppVnodes = vmGetVnodeListFromHash(pMgmt, &numOfVnodes); - for (int32_t i = 0; i < numOfVnodes; ++i) { - SVnodeObj *pVnode = ppVnodes[i]; - vnodeSyncCheckTimeout(pVnode->pImpl); - vmReleaseVnode(pMgmt, pVnode); - } - if (ppVnodes != NULL) { + for (int32_t i = 0; i < numOfVnodes; ++i) { + SVnodeObj *pVnode = ppVnodes[i]; + vnodeSyncCheckTimeout(pVnode->pImpl); + vmReleaseVnode(pMgmt, pVnode); + } taosMemoryFree(ppVnodes); } } diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index 1d0236c0c5..acf96ad397 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -16,9 +16,9 @@ #define _DEFAULT_SOURCE #include "dmMgmt.h" -static SDnode global = {0}; +static SDnode globalDnode = {0}; -SDnode *dmInstance() { return &global; } +SDnode *dmInstance() { return &globalDnode; } static int32_t dmCheckRepeatInit(SDnode *pDnode) { if (atomic_val_compare_exchange_8(&pDnode->once, DND_ENV_INIT, DND_ENV_READY) != DND_ENV_INIT) { @@ -270,6 +270,6 @@ void dmReportStartup(const char *pName, const char *pDesc) { } int64_t dmGetClusterId() { - return global.data.clusterId; + return globalDnode.data.clusterId; } diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h index 92b66230e3..8ecce20f53 100644 --- a/source/dnode/mgmt/node_util/inc/dmUtil.h +++ b/source/dnode/mgmt/node_util/inc/dmUtil.h @@ -167,7 +167,7 @@ void dmUpdateEps(SDnodeData *pData, SArray *pDnodeEps); void dmGetMnodeEpSet(SDnodeData *pData, SEpSet *pEpSet); void dmGetMnodeEpSetForRedirect(SDnodeData *pData, SRpcMsg *pMsg, SEpSet *pEpSet); void dmSetMnodeEpSet(SDnodeData *pData, SEpSet *pEpSet); -int32_t dmUpdateDnodeInfo(void *pData, int32_t *dnodeId, int64_t *clusterId, char *fqdn, uint16_t *port); +void dmUpdateDnodeInfo(void *pData, int32_t *dnodeId, int64_t *clusterId, char *fqdn, uint16_t *port); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/node_util/src/dmEps.c b/source/dnode/mgmt/node_util/src/dmEps.c index 3e2d8b53aa..9640c0b5c9 100644 --- a/source/dnode/mgmt/node_util/src/dmEps.c +++ b/source/dnode/mgmt/node_util/src/dmEps.c @@ -41,14 +41,49 @@ static void dmGetDnodeEp(SDnodeData *pData, int32_t dnodeId, char *pEp, char *pF taosThreadRwlockUnlock(&pData->lock); } +static int32_t dmDecodeEps(SJson *pJson, SDnodeData *pData) { + int32_t code = 0; + + tjsonGetInt32ValueFromDouble(pJson, "dnodeId", pData->dnodeId, code); + if (code < 0) return -1; + tjsonGetNumberValue(pJson, "dnodeVer", pData->dnodeVer, code); + if (code < 0) return -1; + tjsonGetNumberValue(pJson, "clusterId", pData->clusterId, code); + if (code < 0) return -1; + tjsonGetInt32ValueFromDouble(pJson, "dropped", pData->dropped, code); + if (code < 0) return -1; + + SJson *dnodes = tjsonGetObjectItem(pJson, "dnodes"); + if (dnodes == NULL) return 0; + int32_t numOfDnodes = tjsonGetArraySize(dnodes); + + for (int32_t i = 0; i < numOfDnodes; ++i) { + SJson *dnode = tjsonGetArrayItem(dnodes, i); + if (dnode == NULL) return -1; + + SDnodeEp dnodeEp = {0}; + tjsonGetInt32ValueFromDouble(dnode, "id", dnodeEp.id, code); + if (code < 0) return -1; + code = tjsonGetStringValue(dnode, "fqdn", dnodeEp.ep.fqdn); + if (code < 0) return -1; + tjsonGetUInt16ValueFromDouble(dnode, "port", dnodeEp.ep.port, code); + if (code < 0) return -1; + tjsonGetInt8ValueFromDouble(dnode, "isMnode", dnodeEp.isMnode, code); + if (code < 0) return -1; + + if (taosArrayPush(pData->dnodeEps, &dnodeEp) == NULL) return -1; + } + + return 0; +} + int32_t dmReadEps(SDnodeData *pData) { - int32_t code = TSDB_CODE_INVALID_JSON_FORMAT; - int32_t len = 0; - int32_t maxLen = 256 * 1024; - char *content = taosMemoryCalloc(1, maxLen + 1); - cJSON *root = NULL; - char file[PATH_MAX] = {0}; + int32_t code = -1; TdFilePtr pFile = NULL; + char *content = NULL; + SJson *pJson = NULL; + char file[PATH_MAX] = {0}; + snprintf(file, sizeof(file), "%s%sdnode%sdnode.json", tsDataDir, TD_DIRSEP, TD_DIRSEP); pData->dnodeEps = taosArrayInit(1, sizeof(SDnodeEp)); if (pData->dnodeEps == NULL) { @@ -56,113 +91,63 @@ int32_t dmReadEps(SDnodeData *pData) { goto _OVER; } - snprintf(file, sizeof(file), "%s%sdnode%sdnode.json", tsDataDir, TD_DIRSEP, TD_DIRSEP); - pFile = taosOpenFile(file, TD_FILE_READ); - if (pFile == NULL) { + if (taosStatFile(file, NULL, NULL) < 0) { + dInfo("dnode file:%s not exist", file); code = 0; goto _OVER; } - len = (int32_t)taosReadFile(pFile, content, maxLen); - if (len <= 0) { - dError("failed to read %s since content is null", file); + pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to open dnode file:%s since %s", file, terrstr()); goto _OVER; } - content[len] = 0; - root = cJSON_Parse(content); - if (root == NULL) { - dError("failed to read %s since invalid json format", file); + int64_t size = 0; + if (taosFStatFile(pFile, &size, NULL) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to fstat dnode file:%s since %s", file, terrstr()); goto _OVER; } - cJSON *dnodeId = cJSON_GetObjectItem(root, "dnodeId"); - if (!dnodeId || dnodeId->type != cJSON_Number) { - dError("failed to read %s since dnodeId not found", file); - goto _OVER; - } - pData->dnodeId = dnodeId->valueint; - - cJSON *dnodeVer = cJSON_GetObjectItem(root, "dnodeVer"); - if (!dnodeVer || dnodeVer->type != cJSON_String) { - dError("failed to read %s since dnodeVer not found", file); - goto _OVER; - } - pData->dnodeVer = atoll(dnodeVer->valuestring); - - cJSON *clusterId = cJSON_GetObjectItem(root, "clusterId"); - if (!clusterId || clusterId->type != cJSON_String) { - dError("failed to read %s since clusterId not found", file); - goto _OVER; - } - pData->clusterId = atoll(clusterId->valuestring); - - cJSON *dropped = cJSON_GetObjectItem(root, "dropped"); - if (!dropped || dropped->type != cJSON_Number) { - dError("failed to read %s since dropped not found", file); - goto _OVER; - } - pData->dropped = dropped->valueint; - - cJSON *dnodes = cJSON_GetObjectItem(root, "dnodes"); - if (!dnodes || dnodes->type != cJSON_Array) { - dError("failed to read %s since dnodes not found", file); + content = taosMemoryMalloc(size + 1); + if (content == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } - int32_t numOfDnodes = cJSON_GetArraySize(dnodes); - if (numOfDnodes <= 0) { - dError("failed to read %s since numOfDnodes:%d invalid", file, numOfDnodes); + if (taosReadFile(pFile, content, size) != size) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to read dnode file:%s since %s", file, terrstr()); goto _OVER; } - for (int32_t i = 0; i < numOfDnodes; ++i) { - cJSON *node = cJSON_GetArrayItem(dnodes, i); - if (node == NULL) break; + content[size] = '\0'; - SDnodeEp dnodeEp = {0}; + pJson = tjsonParse(content); + if (pJson == NULL) { + terrno = TSDB_CODE_INVALID_JSON_FORMAT; + goto _OVER; + } - cJSON *did = cJSON_GetObjectItem(node, "id"); - if (!did || did->type != cJSON_Number) { - dError("failed to read %s since dnodeId not found", file); - goto _OVER; - } - - dnodeEp.id = did->valueint; - - cJSON *dnodeFqdn = cJSON_GetObjectItem(node, "fqdn"); - if (!dnodeFqdn || dnodeFqdn->type != cJSON_String || dnodeFqdn->valuestring == NULL) { - dError("failed to read %s since dnodeFqdn not found", file); - goto _OVER; - } - tstrncpy(dnodeEp.ep.fqdn, dnodeFqdn->valuestring, TSDB_FQDN_LEN); - - cJSON *dnodePort = cJSON_GetObjectItem(node, "port"); - if (!dnodePort || dnodePort->type != cJSON_Number) { - dError("failed to read %s since dnodePort not found", file); - goto _OVER; - } - - dnodeEp.ep.port = dnodePort->valueint; - - cJSON *isMnode = cJSON_GetObjectItem(node, "isMnode"); - if (!isMnode || isMnode->type != cJSON_Number) { - dError("failed to read %s since isMnode not found", file); - goto _OVER; - } - dnodeEp.isMnode = isMnode->valueint; - - taosArrayPush(pData->dnodeEps, &dnodeEp); + if (dmDecodeEps(pJson, pData) < 0) { + terrno = TSDB_CODE_INVALID_JSON_FORMAT; + goto _OVER; } code = 0; - dDebug("succcessed to read file %s", file); + dInfo("succceed to read mnode file %s", file); _OVER: if (content != NULL) taosMemoryFree(content); - if (root != NULL) cJSON_Delete(root); + if (pJson != NULL) cJSON_Delete(pJson); if (pFile != NULL) taosCloseFile(&pFile); + if (code != 0) { + dError("failed to read dnode file:%s since %s", file, terrstr()); + } + if (taosArrayGetSize(pData->dnodeEps) == 0) { SDnodeEp dnodeEp = {0}; dnodeEp.isMnode = 1; @@ -178,7 +163,6 @@ _OVER: return -1; } - terrno = code; return code; } @@ -247,7 +231,7 @@ _OVER: if (code != 0) { if (terrno == 0) terrno = TAOS_SYSTEM_ERROR(errno); - dInfo("succeed to write dnode file:%s since %s, dnodeVer:%" PRId64, realfile, terrstr(), pData->dnodeVer); + dError("failed to write dnode file:%s since %s, dnodeVer:%" PRId64, realfile, terrstr(), pData->dnodeVer); } return code; } @@ -348,7 +332,7 @@ void dmSetMnodeEpSet(SDnodeData *pData, SEpSet *pEpSet) { } } -int32_t dmUpdateDnodeInfo(void *data, int32_t *dnodeId, int64_t *clusterId, char *fqdn, uint16_t *port) { +void dmUpdateDnodeInfo(void *data, int32_t *dnodeId, int64_t *clusterId, char *fqdn, uint16_t *port) { SDnodeData *pData = data; int32_t ret = -1; taosThreadRwlockRdlock(&pData->lock); @@ -358,7 +342,7 @@ int32_t dmUpdateDnodeInfo(void *data, int32_t *dnodeId, int64_t *clusterId, char if (strcmp(pDnodeEp->ep.fqdn, fqdn) == 0 && pDnodeEp->ep.port == *port) { dInfo("dnode:%s:%u, update dnodeId from %d to %d", fqdn, *port, *dnodeId, pDnodeEp->id); *dnodeId = pDnodeEp->id; - *clusterId = pData->clusterId; + if (clusterId != NULL) *clusterId = pData->clusterId; ret = 0; } } @@ -376,12 +360,12 @@ int32_t dmUpdateDnodeInfo(void *data, int32_t *dnodeId, int64_t *clusterId, char dInfo("dnode:%d, update port from %u to %u", *dnodeId, *port, pDnodeEp->ep.port); *port = pDnodeEp->ep.port; } - *clusterId = pData->clusterId; + if (clusterId != NULL) *clusterId = pData->clusterId; ret = 0; } else { dInfo("dnode:%d, failed to update dnode info", *dnodeId); } } taosThreadRwlockUnlock(&pData->lock); - return ret; + // return ret; } \ No newline at end of file diff --git a/source/dnode/mgmt/node_util/src/dmFile.c b/source/dnode/mgmt/node_util/src/dmFile.c index 4dcc962a20..fb05f08c0c 100644 --- a/source/dnode/mgmt/node_util/src/dmFile.c +++ b/source/dnode/mgmt/node_util/src/dmFile.c @@ -19,48 +19,81 @@ #define MAXLEN 1024 -int32_t dmReadFile(const char *path, const char *name, bool *pDeployed) { - int32_t code = TSDB_CODE_INVALID_JSON_FORMAT; - int64_t len = 0; - char content[MAXLEN + 1] = {0}; - cJSON *root = NULL; - char file[PATH_MAX] = {0}; - TdFilePtr pFile = NULL; +static int32_t dmDecodeFile(SJson *pJson, bool *deployed) { + int32_t code = 0; + int32_t value = 0; + tjsonGetInt32ValueFromDouble(pJson, "deployed", value, code); + if (code < 0) return -1; + + *deployed = (value != 0); + return code; +} + +int32_t dmReadFile(const char *path, const char *name, bool *pDeployed) { + int32_t code = -1; + TdFilePtr pFile = NULL; + char *content = NULL; + SJson *pJson = NULL; + char file[PATH_MAX] = {0}; snprintf(file, sizeof(file), "%s%s%s.json", path, TD_DIRSEP, name); - pFile = taosOpenFile(file, TD_FILE_READ); - if (pFile == NULL) { + + if (taosStatFile(file, NULL, NULL) < 0) { + dInfo("file:%s not exist", file); code = 0; goto _OVER; } - len = taosReadFile(pFile, content, MAXLEN); - if (len <= 0) { - dError("failed to read %s since content is null", file); + pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to open file:%s since %s", file, terrstr()); goto _OVER; } - root = cJSON_Parse(content); - if (root == NULL) { - dError("failed to read %s since invalid json format", file); + int64_t size = 0; + if (taosFStatFile(pFile, &size, NULL) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to fstat file:%s since %s", file, terrstr()); goto _OVER; } - cJSON *deployed = cJSON_GetObjectItem(root, "deployed"); - if (!deployed || deployed->type != cJSON_Number) { - dError("failed to read %s since deployed not found", file); + content = taosMemoryMalloc(size + 1); + if (content == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _OVER; + } + + if (taosReadFile(pFile, content, size) != size) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to read file:%s since %s", file, terrstr()); + goto _OVER; + } + + content[size] = '\0'; + + pJson = tjsonParse(content); + if (pJson == NULL) { + terrno = TSDB_CODE_INVALID_JSON_FORMAT; + goto _OVER; + } + + if (dmDecodeFile(pJson, pDeployed) < 0) { + terrno = TSDB_CODE_INVALID_JSON_FORMAT; goto _OVER; } - *pDeployed = deployed->valueint != 0; - dDebug("succcessed to read file %s, deployed:%d", file, *pDeployed); code = 0; + dInfo("succceed to read mnode file %s", file); _OVER: - if (root != NULL) cJSON_Delete(root); + if (content != NULL) taosMemoryFree(content); + if (pJson != NULL) cJSON_Delete(pJson); if (pFile != NULL) taosCloseFile(&pFile); - terrno = code; + if (code != 0) { + dError("failed to read dnode file:%s since %s", file, terrstr()); + } return code; } diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index ca03207d2b..e0d8ecb3eb 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -20,6 +20,8 @@ #define CLUSTER_VER_NUMBE 1 #define CLUSTER_RESERVE_SIZE 60 +char tsVersionName[16] = "community"; +int64_t tsExpireTime = 0; static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster); static SSdbRow *mndClusterActionDecode(SSdbRaw *pRaw); @@ -291,6 +293,18 @@ static int32_t mndRetrieveClusters(SRpcMsg *pMsg, SShowObj *pShow, SSDataBlock * pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&pCluster->createdTime, false); + char ver[12] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(ver, tsVersionName, pShow->pMeta->pSchemas[cols].bytes); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)ver, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + if (tsExpireTime <= 0) { + colDataAppendNULL(pColInfo, numOfRows); + } else { + colDataAppend(pColInfo, numOfRows, (const char *)&tsExpireTime, false); + } + sdbRelease(pSdb, pCluster); numOfRows++; } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index c7a416d444..ddb54a95ea 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -308,7 +308,8 @@ void mndGetDnodeData(SMnode *pMnode, SArray *pDnodeEps) { void *pIter = NULL; while (1) { SDnodeObj *pDnode = NULL; - pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pDnode); + ESdbStatus objStatus = 0; + pIter = sdbFetchAll(pSdb, SDB_DNODE, pIter, (void **)&pDnode, &objStatus, true); if (pIter == NULL) break; SDnodeEp dnodeEp = {0}; diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index 31f31a15ba..244e6058d4 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -293,7 +293,7 @@ static int32_t mndProcessCreateFuncReq(SRpcMsg *pReq) { goto _OVER; } - mInfo("func:%s, start to create", createReq.name); + mInfo("func:%s, start to create, size:%d", createReq.name, createReq.codeLen); if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_FUNC) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 9b3934c40c..7dcd287fb7 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -15,13 +15,13 @@ #define _DEFAULT_SOURCE #include "mndMnode.h" +#include "mndCluster.h" #include "mndDnode.h" #include "mndPrivilege.h" #include "mndShow.h" #include "mndSync.h" #include "mndTrans.h" #include "tmisce.h" -#include "mndCluster.h" #define MNODE_VER_NUMBER 1 #define MNODE_RESERVE_SIZE 64 @@ -181,9 +181,8 @@ _OVER: static int32_t mndMnodeActionInsert(SSdb *pSdb, SMnodeObj *pObj) { mTrace("mnode:%d, perform insert action, row:%p", pObj->id, pObj); - pObj->pDnode = sdbAcquire(pSdb, SDB_DNODE, &pObj->id); + pObj->pDnode = sdbAcquireNotReadyObj(pSdb, SDB_DNODE, &pObj->id); if (pObj->pDnode == NULL) { - terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; mError("mnode:%d, failed to perform insert action since %s", pObj->id, terrstr()); return -1; } @@ -748,7 +747,7 @@ static void mndReloadSyncConfig(SMnode *pMnode) { pNode->clusterId = mndGetClusterId(pMnode); pNode->nodePort = pObj->pDnode->port; tstrncpy(pNode->nodeFqdn, pObj->pDnode->fqdn, TSDB_FQDN_LEN); - (void)tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); + tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); mInfo("vgId:1, ep:%s:%u dnode:%d", pNode->nodeFqdn, pNode->nodePort, pNode->nodeId); if (pObj->pDnode->id == pMnode->selfDnodeId) { cfg.myIndex = cfg.replicaNum; diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index d7205e6497..90baf57c52 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -202,11 +202,13 @@ static SSdbRow *mndSmaActionDecode(SSdbRaw *pRaw) { _OVER: if (terrno != 0) { - mError("sma:%s, failed to decode from raw:%p since %s", pSma == NULL ? "null" : pSma->name, pRaw, terrstr()); - taosMemoryFreeClear(pSma->expr); - taosMemoryFreeClear(pSma->tagsFilter); - taosMemoryFreeClear(pSma->sql); - taosMemoryFreeClear(pSma->ast); + if (pSma != NULL) { + mError("sma:%s, failed to decode from raw:%p since %s", pSma->name, pRaw, terrstr()); + taosMemoryFreeClear(pSma->expr); + taosMemoryFreeClear(pSma->tagsFilter); + taosMemoryFreeClear(pSma->sql); + taosMemoryFreeClear(pSma->ast); + } taosMemoryFreeClear(pRow); return NULL; } diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 93c9192bed..6b675586e4 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -271,9 +271,11 @@ SSyncFSM *mndSyncMakeFsm(SMnode *pMnode) { int32_t mndInitSync(SMnode *pMnode) { SSyncMgmt *pMgmt = &pMnode->syncMgmt; taosThreadMutexInit(&pMgmt->lock, NULL); + taosThreadMutexLock(&pMgmt->lock); pMgmt->transId = 0; pMgmt->transSec = 0; pMgmt->transSeq = 0; + taosThreadMutexUnlock(&pMgmt->lock); SSyncInfo syncInfo = { .snapshotStrategy = SYNC_STRATEGY_STANDARD_SNAPSHOT, @@ -301,7 +303,7 @@ int32_t mndInitSync(SMnode *pMnode) { pNode->nodeId = pMgmt->replicas[i].id; pNode->nodePort = pMgmt->replicas[i].port; tstrncpy(pNode->nodeFqdn, pMgmt->replicas[i].fqdn, sizeof(pNode->nodeFqdn)); - (void)tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); + tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); mInfo("vgId:1, index:%d ep:%s:%u dnode:%d cluster:%" PRId64, i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId, pNode->clusterId); } @@ -369,6 +371,7 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) { if (pMgmt->transId != 0) { mError("trans:%d, can't be proposed since trans:%d already waiting for confirm", transId, pMgmt->transId); taosThreadMutexUnlock(&pMgmt->lock); + rpcFreeCont(req.pCont); terrno = TSDB_CODE_MND_LAST_TRANS_NOT_FINISHED; return terrno; } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 2550c68cfb..54ea9e7b24 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -1441,10 +1441,10 @@ static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, { SSdbRaw *pRaw = mndVgroupActionEncode(&newVg); - if (pRaw == NULL) return -1; + if (pRaw == NULL) goto _OVER; if (mndTransAppendCommitlog(pTrans, pRaw) != 0) { sdbFreeRaw(pRaw); - return -1; + goto _OVER; } (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); } diff --git a/source/dnode/mnode/sdb/inc/sdb.h b/source/dnode/mnode/sdb/inc/sdb.h index e799f08a17..5a44e4279f 100644 --- a/source/dnode/mnode/sdb/inc/sdb.h +++ b/source/dnode/mnode/sdb/inc/sdb.h @@ -291,6 +291,7 @@ int32_t sdbWriteWithoutFree(SSdb *pSdb, SSdbRaw *pRaw); * @return void* The object of the row. */ void *sdbAcquire(SSdb *pSdb, ESdbType type, const void *pKey); +void *sdbAcquireNotReadyObj(SSdb *pSdb, ESdbType type, const void *pKey); /** * @brief Release a row from sdb. diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index 9e830b83e6..c2d7a9757a 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -228,11 +228,12 @@ static int32_t sdbReadFileImp(SSdb *pSdb) { int32_t readLen = 0; int64_t ret = 0; char file[PATH_MAX] = {0}; + int32_t bufLen = TSDB_MAX_MSG_SIZE; snprintf(file, sizeof(file), "%s%ssdb.data", pSdb->currDir, TD_DIRSEP); mInfo("start to read sdb file:%s", file); - SSdbRaw *pRaw = taosMemoryMalloc(TSDB_MAX_MSG_SIZE + 100); + SSdbRaw *pRaw = taosMemoryMalloc(bufLen + 100); if (pRaw == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; mError("failed read sdb file since %s", terrstr()); @@ -275,14 +276,15 @@ static int32_t sdbReadFileImp(SSdb *pSdb) { } readLen = pRaw->dataLen + sizeof(int32_t); - if (readLen >= pRaw->dataLen) { - SSdbRaw *pNewRaw = taosMemoryMalloc(pRaw->dataLen + TSDB_MAX_MSG_SIZE); + if (readLen >= bufLen) { + bufLen = pRaw->dataLen * 2; + SSdbRaw *pNewRaw = taosMemoryMalloc(bufLen + 100); if (pNewRaw == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - mError("failed read sdb file since malloc new sdbRaw size:%d failed", pRaw->dataLen + TSDB_MAX_MSG_SIZE); + mError("failed read sdb file since malloc new sdbRaw size:%d failed", bufLen); goto _OVER; } - mInfo("malloc new sdbRaw size:%d, type:%d", pRaw->dataLen + TSDB_MAX_MSG_SIZE, pRaw->type); + mInfo("malloc new sdb raw size:%d, type:%d", bufLen, pRaw->type); memcpy(pNewRaw, pRaw, sizeof(SSdbRaw)); sdbFreeRaw(pRaw); pRaw = pNewRaw; diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index 32b34ea3a3..505dee3d87 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -270,7 +270,7 @@ int32_t sdbWrite(SSdb *pSdb, SSdbRaw *pRaw) { return code; } -void *sdbAcquire(SSdb *pSdb, ESdbType type, const void *pKey) { +void *sdbAcquireAll(SSdb *pSdb, ESdbType type, const void *pKey, bool onlyReady) { terrno = 0; SHashObj *hash = sdbGetHash(pSdb, type); @@ -306,10 +306,24 @@ void *sdbAcquire(SSdb *pSdb, ESdbType type, const void *pKey) { break; } + if (pRet == NULL) { + if (!onlyReady) { + terrno = 0; + atomic_add_fetch_32(&pRow->refCount, 1); + pRet = pRow->pObj; + sdbPrintOper(pSdb, pRow, "acquire"); + } + } + sdbUnLock(pSdb, type); return pRet; } +void *sdbAcquire(SSdb *pSdb, ESdbType type, const void *pKey) { return sdbAcquireAll(pSdb, type, pKey, true); } +void *sdbAcquireNotReadyObj(SSdb *pSdb, ESdbType type, const void *pKey) { + return sdbAcquireAll(pSdb, type, pKey, false); +} + static void sdbCheckRow(SSdb *pSdb, SSdbRow *pRow) { int32_t type = pRow->type; sdbWriteLock(pSdb, type); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 5ba44decb4..d8c379f476 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2353,32 +2353,33 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI SBlockData* pBlockData, SLastBlockReader* pLastBlockReader) { SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; + TSDBROW *pRow = NULL, *piRow = NULL; int64_t key = (pBlockData->nRow > 0 && (!pDumpInfo->allDumped)) ? pBlockData->aTSKEY[pDumpInfo->rowIndex] : INT64_MIN; - if (pBlockScanInfo->iter.hasVal && pBlockScanInfo->iiter.hasVal) { - return doMergeMultiLevelRows(pReader, pBlockScanInfo, pBlockData, pLastBlockReader); - } else { - TSDBROW *pRow = NULL, *piRow = NULL; - if (pBlockScanInfo->iter.hasVal) { - pRow = getValidMemRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader); - } - - if (pBlockScanInfo->iiter.hasVal) { - piRow = getValidMemRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader); - } - - // imem + file + last block - if (pBlockScanInfo->iiter.hasVal) { - return doMergeBufAndFileRows(pReader, pBlockScanInfo, piRow, &pBlockScanInfo->iiter, key, pLastBlockReader); - } - - // mem + file + last block - if (pBlockScanInfo->iter.hasVal) { - return doMergeBufAndFileRows(pReader, pBlockScanInfo, pRow, &pBlockScanInfo->iter, key, pLastBlockReader); - } - - // files data blocks + last block - return mergeFileBlockAndLastBlock(pReader, pLastBlockReader, key, pBlockScanInfo, pBlockData); + if (pBlockScanInfo->iter.hasVal) { + pRow = getValidMemRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader); } + + if (pBlockScanInfo->iiter.hasVal) { + piRow = getValidMemRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader); + } + + // two levels of mem-table does contain the valid rows + if (pRow != NULL && piRow != NULL) { + return doMergeMultiLevelRows(pReader, pBlockScanInfo, pBlockData, pLastBlockReader); + } + + // imem + file + last block + if (pBlockScanInfo->iiter.hasVal) { + return doMergeBufAndFileRows(pReader, pBlockScanInfo, piRow, &pBlockScanInfo->iiter, key, pLastBlockReader); + } + + // mem + file + last block + if (pBlockScanInfo->iter.hasVal) { + return doMergeBufAndFileRows(pReader, pBlockScanInfo, pRow, &pBlockScanInfo->iter, key, pLastBlockReader); + } + + // files data blocks + last block + return mergeFileBlockAndLastBlock(pReader, pLastBlockReader, key, pBlockScanInfo, pBlockData); } static int32_t loadNeighborIfOverlap(SFileDataBlockInfo* pBlockInfo, STableBlockScanInfo* pBlockScanInfo, @@ -2854,7 +2855,37 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { TSDBKEY keyInBuf = getCurrentKeyInBuf(pScanInfo, pReader); if (pBlockInfo == NULL) { // build data block from last data file - code = buildComposedDataBlock(pReader); + SBlockData* pBData = &pReader->status.fileBlockData; + tBlockDataReset(pBData); + + SSDataBlock* pResBlock = pReader->pResBlock; + tsdbDebug("load data in last block firstly, due to desc scan data, %s", pReader->idStr); + + int64_t st = taosGetTimestampUs(); + + while (1) { + bool hasBlockLData = hasDataInLastBlock(pLastBlockReader); + + // no data in last block and block, no need to proceed. + if (hasBlockLData == false) { + break; + } + + buildComposedDataBlockImpl(pReader, pScanInfo, &pReader->status.fileBlockData, pLastBlockReader); + if (pResBlock->info.rows >= pReader->capacity) { + break; + } + } + + double el = (taosGetTimestampUs() - st) / 1000.0; + updateComposedBlockInfo(pReader, el, pScanInfo); + + if (pResBlock->info.rows > 0) { + tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 + " rows:%d, elapsed time:%.2f ms %s", + pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey, + pResBlock->info.rows, el, pReader->idStr); + } } else if (fileBlockShouldLoad(pReader, pBlockInfo, pBlock, pScanInfo, keyInBuf, pLastBlockReader)) { code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData, pScanInfo->uid); if (code != TSDB_CODE_SUCCESS) { @@ -2873,10 +2904,38 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { // only return the rows in last block int64_t tsLast = getCurrentKeyInLastBlock(pLastBlockReader); ASSERT(tsLast >= pBlock->maxKey.ts); - tBlockDataReset(&pReader->status.fileBlockData); + SBlockData* pBData = &pReader->status.fileBlockData; + tBlockDataReset(pBData); + + SSDataBlock* pResBlock = pReader->pResBlock; tsdbDebug("load data in last block firstly, due to desc scan data, %s", pReader->idStr); - code = buildComposedDataBlock(pReader); + + int64_t st = taosGetTimestampUs(); + + while (1) { + bool hasBlockLData = hasDataInLastBlock(pLastBlockReader); + + // no data in last block and block, no need to proceed. + if (hasBlockLData == false) { + break; + } + + buildComposedDataBlockImpl(pReader, pScanInfo, &pReader->status.fileBlockData, pLastBlockReader); + if (pResBlock->info.rows >= pReader->capacity) { + break; + } + } + + double el = (taosGetTimestampUs() - st) / 1000.0; + updateComposedBlockInfo(pReader, el, pScanInfo); + + if (pResBlock->info.rows > 0) { + tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 + " rows:%d, elapsed time:%.2f ms %s", + pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey, + pResBlock->info.rows, el, pReader->idStr); + } } else { // whole block is required, return it directly SDataBlockInfo* pInfo = &pReader->pResBlock->info; pInfo->rows = pBlock->nRow; diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 1c92e922b0..f4ac4b7a97 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -637,7 +637,7 @@ SColVal *tsdbRowIterNext(STSDBRowIter *pIter) { } } else { ASSERT(0); - return NULL; // suppress error report by compiler + return NULL; // suppress error report by compiler } } @@ -683,6 +683,18 @@ int32_t tsdbRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW * } tsdbRowGetColVal(pRow, pTSchema, jCol++, pColVal); + if ((!COL_VAL_IS_NONE(pColVal)) && (!COL_VAL_IS_NULL(pColVal)) && IS_VAR_DATA_TYPE(pColVal->type)) { + uint8_t *pVal = pColVal->value.pData; + + pColVal->value.pData = NULL; + code = tRealloc(&pColVal->value.pData, pColVal->value.nData); + if (code) goto _exit; + + if (pColVal->value.nData) { + memcpy(pColVal->value.pData, pVal, pColVal->value.nData); + } + } + if (taosArrayPush(pMerger->pArray, pColVal) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; @@ -721,12 +733,35 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) if (key.version > pMerger->version) { if (!COL_VAL_IS_NONE(pColVal)) { - taosArraySet(pMerger->pArray, iCol, pColVal); + if ((!COL_VAL_IS_NULL(pColVal)) && IS_VAR_DATA_TYPE(pColVal->type)) { + SColVal *tColVal = taosArrayGet(pMerger->pArray, iCol); + code = tRealloc(&tColVal->value.pData, pColVal->value.nData); + if (code) return code; + + tColVal->value.nData = pColVal->value.nData; + if (pColVal->value.nData) { + memcpy(tColVal->value.pData, pColVal->value.pData, pColVal->value.nData); + } + tColVal->flag = 0; + } else { + taosArraySet(pMerger->pArray, iCol, pColVal); + } } } else if (key.version < pMerger->version) { SColVal *tColVal = (SColVal *)taosArrayGet(pMerger->pArray, iCol); if (COL_VAL_IS_NONE(tColVal) && !COL_VAL_IS_NONE(pColVal)) { - taosArraySet(pMerger->pArray, iCol, pColVal); + if ((!COL_VAL_IS_NULL(pColVal)) && IS_VAR_DATA_TYPE(pColVal->type)) { + code = tRealloc(&tColVal->value.pData, pColVal->value.nData); + if (code) return code; + + tColVal->value.nData = pColVal->value.nData; + if (pColVal->value.nData) { + memcpy(tColVal->value.pData, pColVal->value.pData, pColVal->value.nData); + } + tColVal->flag = 0; + } else { + taosArraySet(pMerger->pArray, iCol, pColVal); + } } } else { ASSERT(0 && "dup versions not allowed"); @@ -766,6 +801,18 @@ int32_t tsdbRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema // other for (int16_t iCol = 1; iCol < pTSchema->numOfCols; iCol++) { tsdbRowGetColVal(pRow, pTSchema, iCol, pColVal); + if ((!COL_VAL_IS_NONE(pColVal)) && (!COL_VAL_IS_NULL(pColVal)) && IS_VAR_DATA_TYPE(pColVal->type)) { + uint8_t *pVal = pColVal->value.pData; + + pColVal->value.pData = NULL; + code = tRealloc(&pColVal->value.pData, pColVal->value.nData); + if (code) goto _exit; + + if (pColVal->value.nData) { + memcpy(pColVal->value.pData, pVal, pColVal->value.nData); + } + } + if (taosArrayPush(pMerger->pArray, pColVal) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; @@ -776,7 +823,16 @@ _exit: return code; } -void tsdbRowMergerClear(SRowMerger *pMerger) { taosArrayDestroy(pMerger->pArray); } +void tsdbRowMergerClear(SRowMerger *pMerger) { + for (int32_t iCol = 1; iCol < pMerger->pTSchema->numOfCols; iCol++) { + SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol); + if (IS_VAR_DATA_TYPE(pTColVal->type)) { + tFree(pTColVal->value.pData); + } + } + + taosArrayDestroy(pMerger->pArray); +} int32_t tsdbRowMerge(SRowMerger *pMerger, TSDBROW *pRow) { int32_t code = 0; @@ -790,12 +846,47 @@ int32_t tsdbRowMerge(SRowMerger *pMerger, TSDBROW *pRow) { if (key.version > pMerger->version) { if (!COL_VAL_IS_NONE(pColVal)) { - taosArraySet(pMerger->pArray, iCol, pColVal); + if (IS_VAR_DATA_TYPE(pColVal->type)) { + SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol); + if (!COL_VAL_IS_NULL(pColVal)) { + code = tRealloc(&pTColVal->value.pData, pColVal->value.nData); + if (code) goto _exit; + + pTColVal->value.nData = pColVal->value.nData; + if (pTColVal->value.nData) { + memcpy(pTColVal->value.pData, pColVal->value.pData, pTColVal->value.nData); + } + pTColVal->flag = 0; + } else { + tFree(pTColVal->value.pData); + pTColVal->value.pData = NULL; + taosArraySet(pMerger->pArray, iCol, pColVal); + } + } else { + taosArraySet(pMerger->pArray, iCol, pColVal); + } } } else if (key.version < pMerger->version) { SColVal *tColVal = (SColVal *)taosArrayGet(pMerger->pArray, iCol); if (COL_VAL_IS_NONE(tColVal) && !COL_VAL_IS_NONE(pColVal)) { - taosArraySet(pMerger->pArray, iCol, pColVal); + if (IS_VAR_DATA_TYPE(pColVal->type)) { + if (!COL_VAL_IS_NULL(pColVal)) { + code = tRealloc(&tColVal->value.pData, pColVal->value.nData); + if (code) goto _exit; + + tColVal->value.nData = pColVal->value.nData; + if (tColVal->value.nData) { + memcpy(tColVal->value.pData, pColVal->value.pData, tColVal->value.nData); + } + tColVal->flag = 0; + } else { + tFree(tColVal->value.pData); + tColVal->value.pData = NULL; + taosArraySet(pMerger->pArray, iCol, pColVal); + } + } else { + taosArraySet(pMerger->pArray, iCol, pColVal); + } } } else { ASSERT(0); diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 63c05ce88c..6a6fbcff4d 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -451,8 +451,7 @@ static int vnodeCommitImpl(SCommitInfo *pInfo) { snprintf(dir, TSDB_FILENAME_LEN, "%s", pVnode->path); } - // walBeginSnapshot(pVnode->pWal, pVnode->state.applied); - syncBeginSnapshot(pVnode->sync, pVnode->state.applied); + syncBeginSnapshot(pVnode->sync, pInfo->info.state.committed); // commit each sub-system code = tsdbCommit(pVnode->pTsdb, pInfo); @@ -494,7 +493,6 @@ static int vnodeCommitImpl(SCommitInfo *pInfo) { return -1; } - // walEndSnapshot(pVnode->pWal); syncEndSnapshot(pVnode->sync); _exit: diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 52c1f45762..743ae17f2b 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -86,7 +86,7 @@ int32_t vnodeAlter(const char *path, SAlterVnodeReplicaReq *pReq, STfs *pTfs) { pNode->nodeId = pReq->replicas[i].id; pNode->nodePort = pReq->replicas[i].port; tstrncpy(pNode->nodeFqdn, pReq->replicas[i].fqdn, sizeof(pNode->nodeFqdn)); - (void)tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); + tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); vInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", pReq->vgId, i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId); } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index ef6733a4f2..fdfd81f52f 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -3222,7 +3222,9 @@ static void buildVnodeGroupedNtbTableCount(STableCountScanOperatorInfo* pInfo, S uint64_t groupId = calcGroupId(fullStbName, strlen(fullStbName)); pRes->info.id.groupId = groupId; int64_t ntbNum = metaGetNtbNum(pInfo->readHandle.meta); - fillTableCountScanDataBlock(pSupp, dbName, "", ntbNum, pRes); + if (ntbNum != 0) { + fillTableCountScanDataBlock(pSupp, dbName, "", ntbNum, pRes); + } } static void buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index ac32b54f56..05570eda2f 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -1918,6 +1918,13 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) { colDataAppend(pColInfo, 0, p, false); taosMemoryFree(p); + // make the valgrind happy that all memory buffer has been initialized already. + if (slotId != 0) { + SColumnInfoData* p1 = taosArrayGet(pBlock->pDataBlock, 0); + int64_t v = 0; + colDataAppendInt64(p1, 0, &v); + } + pBlock->info.rows = 1; pOperator->status = OP_EXEC_DONE; return pBlock; diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 03be1ee6f2..661e9f97b7 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -251,7 +251,8 @@ static int32_t sortComparInit(SMsortComparParam* pParam, SArray* pSources, int32 if (pHandle->pBuf == NULL) { if (!osTempSpaceAvailable()) { code = TSDB_CODE_NO_AVAIL_DISK; - qError("Sort compare init failed since %s, %s", terrstr(code), pHandle->idStr); + terrno = code; + qError("Sort compare init failed since %s, %s", tstrerror(code), pHandle->idStr); return code; } @@ -259,6 +260,7 @@ static int32_t sortComparInit(SMsortComparParam* pParam, SArray* pSources, int32 "sortComparInit", tsTempDir); dBufSetPrintInfo(pHandle->pBuf); if (code != TSDB_CODE_SUCCESS) { + terrno = code; return code; } } @@ -282,6 +284,7 @@ static int32_t sortComparInit(SMsortComparParam* pParam, SArray* pSources, int32 code = blockDataFromBuf(pSource->src.pBlock, pPage); if (code != TSDB_CODE_SUCCESS) { + terrno = code; return code; } diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 6b6db16725..224537c1f1 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -1654,7 +1654,9 @@ int32_t percentileFunction(SqlFunctionCtx* pCtx) { int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SVariant* pVal = &pCtx->param[1].param; + int32_t code = 0; double v = 0; + GET_TYPED_DATA(v, double, pVal->nType, &pVal->i); SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); @@ -1662,14 +1664,14 @@ int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { tMemBucket* pMemBucket = ppInfo->pMemBucket; if (pMemBucket != NULL && pMemBucket->total > 0) { // check for null - int32_t code = getPercentile(pMemBucket, v, &ppInfo->result); - if (code != TSDB_CODE_SUCCESS) { - tMemBucketDestroy(pMemBucket); - return code; - } + code = getPercentile(pMemBucket, v, &ppInfo->result); } tMemBucketDestroy(pMemBucket); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + return functionFinalize(pCtx, pBlock); } @@ -2636,7 +2638,7 @@ static int32_t doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv, int32_t v = *(int32_t*)pv; int64_t delta = factor * (v - pDiffInfo->prev.i64); // direct previous may be null if (delta < 0 && pDiffInfo->ignoreNegative) { - colDataSetNull_f(pOutput->nullbitmap, pos); + colDataSetNull_f_s(pOutput, pos); } else { colDataAppendInt64(pOutput, pos, &delta); } @@ -2649,7 +2651,7 @@ static int32_t doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv, int8_t v = *(int8_t*)pv; int64_t delta = factor * (v - pDiffInfo->prev.i64); // direct previous may be null if (delta < 0 && pDiffInfo->ignoreNegative) { - colDataSetNull_f(pOutput->nullbitmap, pos); + colDataSetNull_f_s(pOutput, pos); } else { colDataAppendInt64(pOutput, pos, &delta); } @@ -2660,7 +2662,7 @@ static int32_t doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv, int16_t v = *(int16_t*)pv; int64_t delta = factor * (v - pDiffInfo->prev.i64); // direct previous may be null if (delta < 0 && pDiffInfo->ignoreNegative) { - colDataSetNull_f(pOutput->nullbitmap, pos); + colDataSetNull_f_s(pOutput, pos); } else { colDataAppendInt64(pOutput, pos, &delta); } @@ -2672,7 +2674,7 @@ static int32_t doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv, int64_t v = *(int64_t*)pv; int64_t delta = factor * (v - pDiffInfo->prev.i64); // direct previous may be null if (delta < 0 && pDiffInfo->ignoreNegative) { - colDataSetNull_f(pOutput->nullbitmap, pos); + colDataSetNull_f_s(pOutput, pos); } else { colDataAppendInt64(pOutput, pos, &delta); } @@ -2683,7 +2685,7 @@ static int32_t doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv, float v = *(float*)pv; double delta = factor * (v - pDiffInfo->prev.d64); // direct previous may be null if ((delta < 0 && pDiffInfo->ignoreNegative) || isinf(delta) || isnan(delta)) { // check for overflow - colDataSetNull_f(pOutput->nullbitmap, pos); + colDataSetNull_f_s(pOutput, pos); } else { colDataAppendDouble(pOutput, pos, &delta); } @@ -2694,7 +2696,7 @@ static int32_t doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv, double v = *(double*)pv; double delta = factor * (v - pDiffInfo->prev.d64); // direct previous may be null if ((delta < 0 && pDiffInfo->ignoreNegative) || isinf(delta) || isnan(delta)) { // check for overflow - colDataSetNull_f(pOutput->nullbitmap, pos); + colDataSetNull_f_s(pOutput, pos); } else { colDataAppendDouble(pOutput, pos, &delta); } @@ -2729,7 +2731,7 @@ int32_t diffFunction(SqlFunctionCtx* pCtx) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { if (pDiffInfo->includeNull) { - colDataSetNull_f(pOutput->nullbitmap, pos); + colDataSetNull_f_s(pOutput, pos); numOfElems += 1; } @@ -2767,8 +2769,7 @@ int32_t diffFunction(SqlFunctionCtx* pCtx) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { if (pDiffInfo->includeNull) { - colDataSetNull_f(pOutput->nullbitmap, pos); - + colDataSetNull_f_s(pOutput, pos); numOfElems += 1; } continue; diff --git a/source/libs/function/src/tpercentile.c b/source/libs/function/src/tpercentile.c index 19be8120f1..6577858ee6 100644 --- a/source/libs/function/src/tpercentile.c +++ b/source/libs/function/src/tpercentile.c @@ -99,6 +99,7 @@ static void resetPosInfo(SSlotInfo *pInfo) { int32_t findOnlyResult(tMemBucket *pMemBucket, double *result) { ASSERT(pMemBucket->total == 1); + terrno = 0; for (int32_t i = 0; i < pMemBucket->numOfSlots; ++i) { tMemBucketSlot *pSlot = &pMemBucket->pSlots[i]; diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index de660ae958..74d555af77 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3181,6 +3181,7 @@ bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, SColumnInfoData *pRe void *colData = colDataGetData(pData, i); if (colData == NULL || colDataIsNull_s(pData, i)) { all = false; + p[i] = 0; continue; } diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 6793430923..7e08e195c1 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -32,11 +32,9 @@ typedef struct SyncRequestVoteReply SyncRequestVoteReply; typedef struct SyncAppendEntries SyncAppendEntries; typedef struct SyncAppendEntriesReply SyncAppendEntriesReply; typedef struct SSyncEnv SSyncEnv; -typedef struct SRaftStore SRaftStore; typedef struct SVotesGranted SVotesGranted; typedef struct SVotesRespond SVotesRespond; typedef struct SSyncIndexMgr SSyncIndexMgr; -typedef struct SRaftCfg SRaftCfg; typedef struct SSyncRespMgr SSyncRespMgr; typedef struct SSyncSnapshotSender SSyncSnapshotSender; typedef struct SSyncSnapshotReceiver SSyncSnapshotReceiver; @@ -70,6 +68,11 @@ typedef struct SRaftId { SyncGroupId vgId; } SRaftId; +typedef struct SRaftStore { + SyncTerm currentTerm; + SRaftId voteFor; +} SRaftStore; + typedef struct SSyncHbTimerData { int64_t syncNodeRid; SSyncTimer* pTimer; @@ -112,8 +115,8 @@ typedef struct SSyncNode { // sync io SSyncLogBuffer* pLogBuf; - SWal* pWal; - const SMsgCb* msgcb; + SWal* pWal; + const SMsgCb* msgcb; int32_t (*syncSendMSg)(const SEpSet* pEpSet, SRpcMsg* pMsg); int32_t (*syncEqMsg)(const SMsgCb* msgcb, SRpcMsg* pMsg); int32_t (*syncEqCtrlMsg)(const SMsgCb* msgcb, SRpcMsg* pMsg); @@ -139,8 +142,8 @@ typedef struct SSyncNode { int64_t rid; // tla+ server vars - ESyncState state; - SRaftStore* pRaftStore; + ESyncState state; + SRaftStore raftStore; // tla+ candidate vars SVotesGranted* pVotesGranted; @@ -229,7 +232,7 @@ int32_t syncNodeStartStandBy(SSyncNode* pSyncNode); void syncNodeClose(SSyncNode* pSyncNode); void syncNodePreClose(SSyncNode* pSyncNode); void syncNodePostClose(SSyncNode* pSyncNode); -int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak, int64_t *seq); +int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak, int64_t* seq); int32_t syncNodeRestore(SSyncNode* pSyncNode); void syncHbTimerDataFree(SSyncHbTimerData* pData); diff --git a/source/libs/sync/inc/syncMessage.h b/source/libs/sync/inc/syncMessage.h index 49486bc12d..c3566c7c82 100644 --- a/source/libs/sync/inc/syncMessage.h +++ b/source/libs/sync/inc/syncMessage.h @@ -258,8 +258,8 @@ int32_t syncBuildRequestVote(SRpcMsg* pMsg, int32_t vgId); int32_t syncBuildRequestVoteReply(SRpcMsg* pMsg, int32_t vgId); int32_t syncBuildAppendEntries(SRpcMsg* pMsg, int32_t dataLen, int32_t vgId); int32_t syncBuildAppendEntriesReply(SRpcMsg* pMsg, int32_t vgId); -int32_t syncBuildAppendEntriesFromRaftLog(SSyncNode* pNode, SSyncRaftEntry* pEntry, SyncTerm prevLogTerm, - SRpcMsg* pRpcMsg); +int32_t syncBuildAppendEntriesFromRaftEntry(SSyncNode* pNode, SSyncRaftEntry* pEntry, SyncTerm prevLogTerm, + SRpcMsg* pRpcMsg); int32_t syncBuildHeartbeat(SRpcMsg* pMsg, int32_t vgId); int32_t syncBuildHeartbeatReply(SRpcMsg* pMsg, int32_t vgId); int32_t syncBuildPreSnapshot(SRpcMsg* pMsg, int32_t vgId); diff --git a/source/libs/sync/inc/syncPipeline.h b/source/libs/sync/inc/syncPipeline.h index 55cb0d7db6..504a9f0bd7 100644 --- a/source/libs/sync/inc/syncPipeline.h +++ b/source/libs/sync/inc/syncPipeline.h @@ -78,14 +78,14 @@ static FORCE_INLINE int32_t syncLogGetNextRetryBackoff(SSyncLogReplMgr* pMgr) { SyncTerm syncLogReplMgrGetPrevLogTerm(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index); int32_t syncLogReplMgrReplicateOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode); -int32_t syncLogBufferReplicateOneTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index, SyncTerm* pTerm, - SRaftId* pDestId, bool* pBarrier); -int32_t syncLogReplMgrReplicateAttemptedOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode); -int32_t syncLogReplMgrReplicateProbeOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index); +int32_t syncLogReplMgrReplicateOneTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index, SyncTerm* pTerm, + SRaftId* pDestId, bool* pBarrier); +int32_t syncLogReplMgrReplicateAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode); +int32_t syncLogReplMgrReplicateProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index); int32_t syncLogReplMgrProcessReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg); -int32_t syncLogReplMgrProcessReplyInRecoveryMode(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg); -int32_t syncLogReplMgrProcessReplyInNormalMode(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg); +int32_t syncLogReplMgrProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg); +int32_t syncLogReplMgrProcessReplyAsNormal(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg); int32_t syncLogReplMgrProcessHeartbeatReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncHeartbeatReply* pMsg); int32_t syncLogReplMgrRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode); diff --git a/source/libs/sync/inc/syncRaftStore.h b/source/libs/sync/inc/syncRaftStore.h index bb6405f6b2..21a8fc64a8 100644 --- a/source/libs/sync/inc/syncRaftStore.h +++ b/source/libs/sync/inc/syncRaftStore.h @@ -24,27 +24,16 @@ extern "C" { #define RAFT_STORE_BLOCK_SIZE 512 #define RAFT_STORE_PATH_LEN (TSDB_FILENAME_LEN * 2) +#define EMPTY_RAFT_ID ((SRaftId){.addr = 0, .vgId = 0}) -#define EMPTY_RAFT_ID ((SRaftId){.addr = 0, .vgId = 0}) +int32_t raftStoreReadFile(SSyncNode *pNode); +int32_t raftStoreWriteFile(SSyncNode *pNode); -typedef struct SRaftStore { - SyncTerm currentTerm; - SRaftId voteFor; - TdFilePtr pFile; - char path[RAFT_STORE_PATH_LEN]; -} SRaftStore; - -SRaftStore *raftStoreOpen(const char *path); -int32_t raftStoreClose(SRaftStore *pRaftStore); -int32_t raftStorePersist(SRaftStore *pRaftStore); -int32_t raftStoreSerialize(SRaftStore *pRaftStore, char *buf, size_t len); -int32_t raftStoreDeserialize(SRaftStore *pRaftStore, char *buf, size_t len); - -bool raftStoreHasVoted(SRaftStore *pRaftStore); -void raftStoreVote(SRaftStore *pRaftStore, SRaftId *pRaftId); -void raftStoreClearVote(SRaftStore *pRaftStore); -void raftStoreNextTerm(SRaftStore *pRaftStore); -void raftStoreSetTerm(SRaftStore *pRaftStore, SyncTerm term); +bool raftStoreHasVoted(SSyncNode *pNode); +void raftStoreVote(SSyncNode *pNode, SRaftId *pRaftId); +void raftStoreClearVote(SSyncNode *pNode); +void raftStoreNextTerm(SSyncNode *pNode); +void raftStoreSetTerm(SSyncNode *pNode, SyncTerm term); #ifdef __cplusplus } diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 026ebdb37c..e77a8d4be3 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -128,7 +128,7 @@ int32_t syncNodeFollowerCommit(SSyncNode* ths, SyncIndex newCommitIndex) { return 0; } -SSyncRaftEntry* syncLogAppendEntriesToRaftEntry(const SyncAppendEntries* pMsg) { +SSyncRaftEntry* syncBuildRaftEntryFromAppendEntries(const SyncAppendEntries* pMsg) { SSyncRaftEntry* pEntry = taosMemoryMalloc(pMsg->dataLen); if (pEntry == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -159,17 +159,17 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, const SRpcMsg* pRpcMsg) { // prepare response msg pReply->srcId = ths->myRaftId; pReply->destId = pMsg->srcId; - pReply->term = ths->pRaftStore->currentTerm; + pReply->term = ths->raftStore.currentTerm; pReply->success = false; pReply->matchIndex = SYNC_INDEX_INVALID; pReply->lastSendIndex = pMsg->prevLogIndex + 1; pReply->startTime = ths->startTime; - if (pMsg->term < ths->pRaftStore->currentTerm) { + if (pMsg->term < ths->raftStore.currentTerm) { goto _SEND_RESPONSE; } - if (pMsg->term > ths->pRaftStore->currentTerm) { + if (pMsg->term > ths->raftStore.currentTerm) { pReply->term = pMsg->term; } @@ -182,7 +182,7 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, const SRpcMsg* pRpcMsg) { goto _IGNORE; } - SSyncRaftEntry* pEntry = syncLogAppendEntriesToRaftEntry(pMsg); + SSyncRaftEntry* pEntry = syncBuildRaftEntryFromAppendEntries(pMsg); if (pEntry == NULL) { sError("vgId:%d, failed to get raft entry from append entries since %s", ths->vgId, terrstr()); @@ -253,19 +253,19 @@ int32_t syncNodeOnAppendEntriesOld(SSyncNode* ths, const SRpcMsg* pRpcMsg) { SyncAppendEntriesReply* pReply = rpcRsp.pCont; pReply->srcId = ths->myRaftId; pReply->destId = pMsg->srcId; - pReply->term = ths->pRaftStore->currentTerm; + pReply->term = ths->raftStore.currentTerm; pReply->success = false; // pReply->matchIndex = ths->pLogStore->syncLogLastIndex(ths->pLogStore); pReply->matchIndex = SYNC_INDEX_INVALID; pReply->lastSendIndex = pMsg->prevLogIndex + 1; pReply->startTime = ths->startTime; - if (pMsg->term < ths->pRaftStore->currentTerm) { + if (pMsg->term < ths->raftStore.currentTerm) { syncLogRecvAppendEntries(ths, pMsg, "reject, small term"); goto _SEND_RESPONSE; } - if (pMsg->term > ths->pRaftStore->currentTerm) { + if (pMsg->term > ths->raftStore.currentTerm) { pReply->term = pMsg->term; } diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index b83be2bebb..8157a5a14f 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -50,19 +50,19 @@ int32_t syncNodeOnAppendEntriesReply(SSyncNode* ths, const SRpcMsg* pRpcMsg) { } // drop stale response - if (pMsg->term < ths->pRaftStore->currentTerm) { + if (pMsg->term < ths->raftStore.currentTerm) { syncLogRecvAppendEntriesReply(ths, pMsg, "drop stale response"); return 0; } if (ths->state == TAOS_SYNC_STATE_LEADER) { - if (pMsg->term > ths->pRaftStore->currentTerm) { + if (pMsg->term > ths->raftStore.currentTerm) { syncLogRecvAppendEntriesReply(ths, pMsg, "error term"); syncNodeStepDown(ths, pMsg->term); return -1; } - ASSERT(pMsg->term == ths->pRaftStore->currentTerm); + ASSERT(pMsg->term == ths->raftStore.currentTerm); sTrace("vgId:%d, received append entries reply. srcId:0x%016" PRIx64 ", term:%" PRId64 ", matchIndex:%" PRId64 "", pMsg->vgId, pMsg->srcId.addr, pMsg->term, pMsg->matchIndex); @@ -100,19 +100,19 @@ int32_t syncNodeOnAppendEntriesReplyOld(SSyncNode* ths, SyncAppendEntriesReply* } // drop stale response - if (pMsg->term < ths->pRaftStore->currentTerm) { + if (pMsg->term < ths->raftStore.currentTerm) { syncLogRecvAppendEntriesReply(ths, pMsg, "drop stale response"); return 0; } if (ths->state == TAOS_SYNC_STATE_LEADER) { - if (pMsg->term > ths->pRaftStore->currentTerm) { + if (pMsg->term > ths->raftStore.currentTerm) { syncLogRecvAppendEntriesReply(ths, pMsg, "error term"); syncNodeStepDown(ths, pMsg->term); return -1; } - ASSERT(pMsg->term == ths->pRaftStore->currentTerm); + ASSERT(pMsg->term == ths->raftStore.currentTerm); if (pMsg->success) { SyncIndex oldMatchIndex = syncIndexMgrGetIndex(ths->pMatchIndex, &(pMsg->srcId)); diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index 152fddb7e6..286cf4daf5 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -133,7 +133,7 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { } } // cannot commit, even if quorum agree. need check term! - if (pEntry->term <= pSyncNode->pRaftStore->currentTerm) { + if (pEntry->term <= pSyncNode->raftStore.currentTerm) { // update commit index newCommitIndex = index; @@ -329,7 +329,7 @@ int64_t syncNodeCheckCommitIndex(SSyncNode* ths, SyncIndex indexLikely) { SyncIndex commitIndex = indexLikely; syncNodeUpdateCommitIndex(ths, commitIndex); sTrace("vgId:%d, agreed upon. role:%d, term:%" PRId64 ", index: %" PRId64 "", ths->vgId, ths->state, - ths->pRaftStore->currentTerm, commitIndex); + ths->raftStore.currentTerm, commitIndex); } return ths->commitIndex; } diff --git a/source/libs/sync/src/syncElection.c b/source/libs/sync/src/syncElection.c index bcc95c5f10..cd3ffc33e3 100644 --- a/source/libs/sync/src/syncElection.c +++ b/source/libs/sync/src/syncElection.c @@ -48,7 +48,7 @@ static int32_t syncNodeRequestVotePeers(SSyncNode* pNode) { SyncRequestVote* pMsg = rpcMsg.pCont; pMsg->srcId = pNode->myRaftId; pMsg->destId = pNode->peersId[i]; - pMsg->term = pNode->pRaftStore->currentTerm; + pMsg->term = pNode->raftStore.currentTerm; ret = syncNodeGetLastIndexTerm(pNode, &pMsg->lastLogIndex, &pMsg->lastLogTerm); ASSERT(ret == 0); @@ -75,10 +75,10 @@ int32_t syncNodeElect(SSyncNode* pSyncNode) { } // start election - raftStoreNextTerm(pSyncNode->pRaftStore); - raftStoreClearVote(pSyncNode->pRaftStore); - voteGrantedReset(pSyncNode->pVotesGranted, pSyncNode->pRaftStore->currentTerm); - votesRespondReset(pSyncNode->pVotesRespond, pSyncNode->pRaftStore->currentTerm); + raftStoreNextTerm(pSyncNode); + raftStoreClearVote(pSyncNode); + voteGrantedReset(pSyncNode->pVotesGranted, pSyncNode->raftStore.currentTerm); + votesRespondReset(pSyncNode->pVotesRespond, pSyncNode->raftStore.currentTerm); syncNodeVoteForSelf(pSyncNode); if (voteGrantedMajority(pSyncNode->pVotesGranted)) { diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index c2bf6dc837..02f9795cad 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -468,7 +468,7 @@ bool syncNodeIsReadyForRead(SSyncNode* pSyncNode) { } if (code == 0 && pEntry != NULL) { - if (pEntry->originalRpcType == TDMT_SYNC_NOOP && pEntry->term == pSyncNode->pRaftStore->currentTerm) { + if (pEntry->originalRpcType == TDMT_SYNC_NOOP && pEntry->term == pSyncNode->raftStore.currentTerm) { ready = true; } @@ -736,7 +736,7 @@ int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak, int64_ int32_t code = syncNodeOnClientRequest(pSyncNode, pMsg, &retIndex); if (code == 0) { pMsg->info.conn.applyIndex = retIndex; - pMsg->info.conn.applyTerm = pSyncNode->pRaftStore->currentTerm; + pMsg->info.conn.applyTerm = pSyncNode->raftStore.currentTerm; sTrace("vgId:%d, propose optimized msg, index:%" PRId64 " type:%s", pSyncNode->vgId, retIndex, TMSG_INFO(pMsg->msgType)); return 1; @@ -898,7 +898,7 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { sInfo("vgId:%d, start to open sync node, replica:%d selfIndex:%d", pSyncNode->vgId, pCfg->replicaNum, pCfg->myIndex); for (int32_t i = 0; i < pCfg->replicaNum; ++i) { SNodeInfo* pNode = &pCfg->nodeInfo[i]; - (void)tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); + tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); sInfo("vgId:%d, index:%d ep:%s:%u dnode:%d cluster:%" PRId64, pSyncNode->vgId, i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId, pNode->clusterId); } @@ -983,8 +983,7 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { // init TLA+ server vars pSyncNode->state = TAOS_SYNC_STATE_FOLLOWER; - pSyncNode->pRaftStore = raftStoreOpen(pSyncNode->raftStorePath); - if (pSyncNode->pRaftStore == NULL) { + if (raftStoreReadFile(pSyncNode) != 0) { sError("vgId:%d, failed to open raft store at path %s", pSyncNode->vgId, pSyncNode->raftStorePath); goto _error; } @@ -1184,7 +1183,7 @@ int32_t syncNodeRestore(SSyncNode* pSyncNode) { int32_t syncNodeStart(SSyncNode* pSyncNode) { // start raft if (pSyncNode->replicaNum == 1) { - raftStoreNextTerm(pSyncNode->pRaftStore); + raftStoreNextTerm(pSyncNode); syncNodeBecomeLeader(pSyncNode, "one replica start"); // Raft 3.6.2 Committing entries from previous terms @@ -1202,7 +1201,7 @@ int32_t syncNodeStart(SSyncNode* pSyncNode) { void syncNodeStartOld(SSyncNode* pSyncNode) { // start raft if (pSyncNode->replicaNum == 1) { - raftStoreNextTerm(pSyncNode->pRaftStore); + raftStoreNextTerm(pSyncNode); syncNodeBecomeLeader(pSyncNode, "one replica start"); // Raft 3.6.2 Committing entries from previous terms @@ -1288,10 +1287,6 @@ void syncNodeClose(SSyncNode* pSyncNode) { if (pSyncNode == NULL) return; sNInfo(pSyncNode, "sync close, node:%p", pSyncNode); - int32_t ret = raftStoreClose(pSyncNode->pRaftStore); - ASSERT(ret == 0); - pSyncNode->pRaftStore = NULL; - syncNodeLogReplMgrDestroy(pSyncNode); syncRespMgrDestroy(pSyncNode->pSyncRespMgr); pSyncNode->pSyncRespMgr = NULL; @@ -1708,45 +1703,44 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde _END: // log end config change - sNInfo(pSyncNode, "end do config change, from %d to %d", pSyncNode->vgId, oldConfig.replicaNum, - pNewConfig->replicaNum); + sNInfo(pSyncNode, "end do config change, from %d to %d", oldConfig.replicaNum, pNewConfig->replicaNum); } // raft state change -------------- void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term) { - if (term > pSyncNode->pRaftStore->currentTerm) { - raftStoreSetTerm(pSyncNode->pRaftStore, term); + if (term > pSyncNode->raftStore.currentTerm) { + raftStoreSetTerm(pSyncNode, term); char tmpBuf[64]; snprintf(tmpBuf, sizeof(tmpBuf), "update term to %" PRId64, term); syncNodeBecomeFollower(pSyncNode, tmpBuf); - raftStoreClearVote(pSyncNode->pRaftStore); + raftStoreClearVote(pSyncNode); } } void syncNodeUpdateTermWithoutStepDown(SSyncNode* pSyncNode, SyncTerm term) { - if (term > pSyncNode->pRaftStore->currentTerm) { - raftStoreSetTerm(pSyncNode->pRaftStore, term); + if (term > pSyncNode->raftStore.currentTerm) { + raftStoreSetTerm(pSyncNode, term); } } void syncNodeStepDown(SSyncNode* pSyncNode, SyncTerm newTerm) { - if (pSyncNode->pRaftStore->currentTerm > newTerm) { + if (pSyncNode->raftStore.currentTerm > newTerm) { sNTrace(pSyncNode, "step down, ignore, new-term:%" PRId64 ", current-term:%" PRId64, newTerm, - pSyncNode->pRaftStore->currentTerm); + pSyncNode->raftStore.currentTerm); return; } do { sNTrace(pSyncNode, "step down, new-term:%" PRId64 ", current-term:%" PRId64, newTerm, - pSyncNode->pRaftStore->currentTerm); + pSyncNode->raftStore.currentTerm); } while (0); - if (pSyncNode->pRaftStore->currentTerm < newTerm) { - raftStoreSetTerm(pSyncNode->pRaftStore, newTerm); + if (pSyncNode->raftStore.currentTerm < newTerm) { + raftStoreSetTerm(pSyncNode, newTerm); char tmpBuf[64]; snprintf(tmpBuf, sizeof(tmpBuf), "step down, update term to %" PRId64, newTerm); syncNodeBecomeFollower(pSyncNode, tmpBuf); - raftStoreClearVote(pSyncNode->pRaftStore); + raftStoreClearVote(pSyncNode); } else { if (pSyncNode->state != TAOS_SYNC_STATE_FOLLOWER) { @@ -1904,7 +1898,7 @@ void syncNodeCandidate2Leader(SSyncNode* pSyncNode) { SyncIndex lastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); ASSERT(lastIndex >= 0); sInfo("vgId:%d, become leader. term: %" PRId64 ", commit index: %" PRId64 ", last index: %" PRId64 "", - pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, lastIndex); + pSyncNode->vgId, pSyncNode->raftStore.currentTerm, pSyncNode->commitIndex, lastIndex); } void syncNodeCandidate2LeaderOld(SSyncNode* pSyncNode) { @@ -1937,7 +1931,7 @@ void syncNodeFollower2Candidate(SSyncNode* pSyncNode) { pSyncNode->state = TAOS_SYNC_STATE_CANDIDATE; SyncIndex lastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); sInfo("vgId:%d, become candidate from follower. term: %" PRId64 ", commit index: %" PRId64 ", last index: %" PRId64, - pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, lastIndex); + pSyncNode->vgId, pSyncNode->raftStore.currentTerm, pSyncNode->commitIndex, lastIndex); sNTrace(pSyncNode, "follower to candidate"); } @@ -1947,7 +1941,7 @@ void syncNodeLeader2Follower(SSyncNode* pSyncNode) { syncNodeBecomeFollower(pSyncNode, "leader to follower"); SyncIndex lastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); sInfo("vgId:%d, become follower from leader. term: %" PRId64 ", commit index: %" PRId64 ", last index: %" PRId64, - pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, lastIndex); + pSyncNode->vgId, pSyncNode->raftStore.currentTerm, pSyncNode->commitIndex, lastIndex); sNTrace(pSyncNode, "leader to follower"); } @@ -1957,7 +1951,7 @@ void syncNodeCandidate2Follower(SSyncNode* pSyncNode) { syncNodeBecomeFollower(pSyncNode, "candidate to follower"); SyncIndex lastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); sInfo("vgId:%d, become follower from candidate. term: %" PRId64 ", commit index: %" PRId64 ", last index: %" PRId64, - pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, lastIndex); + pSyncNode->vgId, pSyncNode->raftStore.currentTerm, pSyncNode->commitIndex, lastIndex); sNTrace(pSyncNode, "candidate to follower"); } @@ -1965,15 +1959,15 @@ void syncNodeCandidate2Follower(SSyncNode* pSyncNode) { // just called by syncNodeVoteForSelf // need assert void syncNodeVoteForTerm(SSyncNode* pSyncNode, SyncTerm term, SRaftId* pRaftId) { - ASSERT(term == pSyncNode->pRaftStore->currentTerm); - ASSERT(!raftStoreHasVoted(pSyncNode->pRaftStore)); + ASSERT(term == pSyncNode->raftStore.currentTerm); + ASSERT(!raftStoreHasVoted(pSyncNode)); - raftStoreVote(pSyncNode->pRaftStore, pRaftId); + raftStoreVote(pSyncNode, pRaftId); } // simulate get vote from outside void syncNodeVoteForSelf(SSyncNode* pSyncNode) { - syncNodeVoteForTerm(pSyncNode, pSyncNode->pRaftStore->currentTerm, &pSyncNode->myRaftId); + syncNodeVoteForTerm(pSyncNode, pSyncNode->raftStore.currentTerm, &pSyncNode->myRaftId); SRpcMsg rpcMsg = {0}; int32_t ret = syncBuildRequestVoteReply(&rpcMsg, pSyncNode->vgId); @@ -1982,7 +1976,7 @@ void syncNodeVoteForSelf(SSyncNode* pSyncNode) { SyncRequestVoteReply* pMsg = rpcMsg.pCont; pMsg->srcId = pSyncNode->myRaftId; pMsg->destId = pSyncNode->myRaftId; - pMsg->term = pSyncNode->pRaftStore->currentTerm; + pMsg->term = pSyncNode->raftStore.currentTerm; pMsg->voteGranted = true; voteGrantedVote(pSyncNode->pVotesGranted, pMsg); @@ -2272,13 +2266,6 @@ static void syncNodeEqPeerHeartbeatTimer(void* param, void* tmrId) { return; } - if (pSyncNode->pRaftStore == NULL) { - syncNodeRelease(pSyncNode); - syncHbTimerDataRelease(pData); - sError("vgId:%d, hb timer raft store already stop", pSyncNode->vgId); - return; - } - // sTrace("vgId:%d, eq peer hb timer", pSyncNode->vgId); if (pSyncNode->replicaNum > 1) { @@ -2302,7 +2289,7 @@ static void syncNodeEqPeerHeartbeatTimer(void* param, void* tmrId) { SyncHeartbeat* pSyncMsg = rpcMsg.pCont; pSyncMsg->srcId = pSyncNode->myRaftId; pSyncMsg->destId = pData->destId; - pSyncMsg->term = pSyncNode->pRaftStore->currentTerm; + pSyncMsg->term = pSyncNode->raftStore.currentTerm; pSyncMsg->commitIndex = pSyncNode->commitIndex; pSyncMsg->minMatchIndex = syncMinMatchIndex(pSyncNode); pSyncMsg->privateTerm = 0; @@ -2348,7 +2335,7 @@ static int32_t syncNodeEqNoop(SSyncNode* pNode) { } SyncIndex index = pNode->pLogStore->syncLogWriteIndex(pNode->pLogStore); - SyncTerm term = pNode->pRaftStore->currentTerm; + SyncTerm term = pNode->raftStore.currentTerm; SSyncRaftEntry* pEntry = syncEntryBuildNoop(term, index, pNode->vgId); if (pEntry == NULL) return -1; @@ -2394,8 +2381,7 @@ int32_t syncNodeAppend(SSyncNode* ths, SSyncRaftEntry* pEntry) { if (syncLogBufferAppend(ths->pLogBuf, ths, pEntry) < 0) { sError("vgId:%d, failed to enqueue sync log buffer, index:%" PRId64, ths->vgId, pEntry->index); terrno = TSDB_CODE_SYN_BUFFER_FULL; - (void)syncLogFsmExecute(ths, ths->pFsm, ths->state, ths->pRaftStore->currentTerm, pEntry, - TSDB_CODE_SYN_BUFFER_FULL); + (void)syncLogFsmExecute(ths, ths->pFsm, ths->state, ths->raftStore.currentTerm, pEntry, TSDB_CODE_SYN_BUFFER_FULL); syncEntryDestroy(pEntry); return -1; } @@ -2468,7 +2454,7 @@ bool syncNodeSnapshotRecving(SSyncNode* pSyncNode) { static int32_t syncNodeAppendNoop(SSyncNode* ths) { SyncIndex index = syncLogBufferGetEndIndex(ths->pLogBuf); - SyncTerm term = ths->pRaftStore->currentTerm; + SyncTerm term = ths->raftStore.currentTerm; SSyncRaftEntry* pEntry = syncEntryBuildNoop(term, index, ths->vgId); if (pEntry == NULL) { @@ -2484,7 +2470,7 @@ static int32_t syncNodeAppendNoopOld(SSyncNode* ths) { int32_t ret = 0; SyncIndex index = ths->pLogStore->syncLogWriteIndex(ths->pLogStore); - SyncTerm term = ths->pRaftStore->currentTerm; + SyncTerm term = ths->raftStore.currentTerm; SSyncRaftEntry* pEntry = syncEntryBuildNoop(term, index, ths->vgId); ASSERT(pEntry != NULL); @@ -2526,12 +2512,12 @@ int32_t syncNodeOnHeartbeat(SSyncNode* ths, const SRpcMsg* pRpcMsg) { SyncHeartbeatReply* pMsgReply = rpcMsg.pCont; pMsgReply->destId = pMsg->srcId; pMsgReply->srcId = ths->myRaftId; - pMsgReply->term = ths->pRaftStore->currentTerm; + pMsgReply->term = ths->raftStore.currentTerm; pMsgReply->privateTerm = 8864; // magic number pMsgReply->startTime = ths->startTime; pMsgReply->timeStamp = tsMs; - if (pMsg->term == ths->pRaftStore->currentTerm && ths->state != TAOS_SYNC_STATE_LEADER) { + if (pMsg->term == ths->raftStore.currentTerm && ths->state != TAOS_SYNC_STATE_LEADER) { syncIndexMgrSetRecvTime(ths->pNextIndex, &(pMsg->srcId), tsMs); syncNodeResetElectTimer(ths); @@ -2560,7 +2546,7 @@ int32_t syncNodeOnHeartbeat(SSyncNode* ths, const SRpcMsg* pRpcMsg) { } } - if (pMsg->term >= ths->pRaftStore->currentTerm && ths->state != TAOS_SYNC_STATE_FOLLOWER) { + if (pMsg->term >= ths->raftStore.currentTerm && ths->state != TAOS_SYNC_STATE_FOLLOWER) { // syncNodeStepDown(ths, pMsg->term); SRpcMsg rpcMsgLocalCmd = {0}; (void)syncBuildLocalCmd(&rpcMsgLocalCmd, ths->vgId); @@ -2687,7 +2673,7 @@ int32_t syncNodeOnClientRequest(SSyncNode* ths, SRpcMsg* pMsg, SyncIndex* pRetIn int32_t code = 0; SyncIndex index = syncLogBufferGetEndIndex(ths->pLogBuf); - SyncTerm term = ths->pRaftStore->currentTerm; + SyncTerm term = ths->raftStore.currentTerm; SSyncRaftEntry* pEntry = NULL; if (pMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) { pEntry = syncEntryBuildFromClientRequest(pMsg->pCont, term, index); @@ -2721,7 +2707,7 @@ int32_t syncNodeOnClientRequestOld(SSyncNode* ths, SRpcMsg* pMsg, SyncIndex* pRe int32_t code = 0; SyncIndex index = ths->pLogStore->syncLogWriteIndex(ths->pLogStore); - SyncTerm term = ths->pRaftStore->currentTerm; + SyncTerm term = ths->raftStore.currentTerm; SSyncRaftEntry* pEntry; if (pMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) { @@ -2755,7 +2741,7 @@ int32_t syncNodeOnClientRequestOld(SSyncNode* ths, SRpcMsg* pMsg, SyncIndex* pRe .state = ths->state, .seqNum = pEntry->seqNum, .term = pEntry->term, - .currentTerm = ths->pRaftStore->currentTerm, + .currentTerm = ths->raftStore.currentTerm, .flag = 0, }; ths->pFsm->FpCommitCb(ths->pFsm, pMsg, &cbMeta); @@ -2833,7 +2819,7 @@ int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* p return 0; } - if (pEntry->term < ths->pRaftStore->currentTerm) { + if (pEntry->term < ths->raftStore.currentTerm) { sNTrace(ths, "little term:%" PRId64 ", can not do leader transfer", pEntry->term); return 0; } @@ -2871,7 +2857,7 @@ int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* p if (ths->pFsm->FpLeaderTransferCb != NULL) { SFsmCbMeta cbMeta = { .code = 0, - .currentTerm = ths->pRaftStore->currentTerm, + .currentTerm = ths->raftStore.currentTerm, .flag = 0, .index = pEntry->index, .lastConfigIndex = syncNodeGetSnapshotConfigIndex(ths, pEntry->index), @@ -2987,7 +2973,7 @@ int32_t syncNodeDoCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endInde .state = ths->state, .seqNum = pEntry->seqNum, .term = pEntry->term, - .currentTerm = ths->pRaftStore->currentTerm, + .currentTerm = ths->raftStore.currentTerm, .flag = flag, }; diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index 467b4e2219..7d534c671e 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -154,8 +154,8 @@ int32_t syncBuildAppendEntriesReply(SRpcMsg* pMsg, int32_t vgId) { return 0; } -int32_t syncBuildAppendEntriesFromRaftLog(SSyncNode* pNode, SSyncRaftEntry* pEntry, SyncTerm prevLogTerm, - SRpcMsg* pRpcMsg) { +int32_t syncBuildAppendEntriesFromRaftEntry(SSyncNode* pNode, SSyncRaftEntry* pEntry, SyncTerm prevLogTerm, + SRpcMsg* pRpcMsg) { uint32_t dataLen = pEntry->bytes; uint32_t bytes = sizeof(SyncAppendEntries) + dataLen; pRpcMsg->contLen = bytes; @@ -176,7 +176,7 @@ int32_t syncBuildAppendEntriesFromRaftLog(SSyncNode* pNode, SSyncRaftEntry* pEnt pMsg->prevLogTerm = prevLogTerm; pMsg->vgId = pNode->vgId; pMsg->srcId = pNode->myRaftId; - pMsg->term = pNode->pRaftStore->currentTerm; + pMsg->term = pNode->raftStore.currentTerm; pMsg->commitIndex = pNode->commitIndex; pMsg->privateTerm = 0; return 0; diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index f878044bca..b61fc2e90d 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -285,9 +285,9 @@ SyncTerm syncLogBufferGetLastMatchTerm(SSyncLogBuffer* pBuf) { int32_t syncLogBufferAccept(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEntry* pEntry, SyncTerm prevTerm) { taosThreadMutexLock(&pBuf->mutex); syncLogBufferValidate(pBuf); - int32_t ret = -1; - SyncIndex index = pEntry->index; - SyncIndex prevIndex = pEntry->index - 1; + int32_t ret = -1; + SyncIndex index = pEntry->index; + SyncIndex prevIndex = pEntry->index - 1; SyncTerm lastMatchTerm = syncLogBufferGetLastMatchTermWithoutLock(pBuf); SSyncRaftEntry* pExist = NULL; bool inBuf = true; @@ -509,7 +509,7 @@ int32_t syncLogBufferCommit(SSyncLogBuffer* pBuf, SSyncNode* pNode, int64_t comm SSyncLogStore* pLogStore = pNode->pLogStore; SSyncFSM* pFsm = pNode->pFsm; ESyncState role = pNode->state; - SyncTerm term = pNode->pRaftStore->currentTerm; + SyncTerm term = pNode->raftStore.currentTerm; SyncGroupId vgId = pNode->vgId; int32_t ret = -1; int64_t upperIndex = TMIN(commitIndex, pBuf->matchIndex); @@ -571,7 +571,7 @@ int32_t syncLogBufferCommit(SSyncLogBuffer* pBuf, SSyncNode* pNode, int64_t comm _out: // mark as restored if needed if (!pNode->restoreFinish && pBuf->commitIndex >= pNode->commitIndex && pEntry != NULL && - pNode->pRaftStore->currentTerm <= pEntry->term) { + pNode->raftStore.currentTerm <= pEntry->term) { pNode->pFsm->FpRestoreFinishCb(pNode->pFsm); pNode->restoreFinish = true; sInfo("vgId:%d, restore finished. log buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")", pNode->vgId, @@ -614,9 +614,9 @@ int32_t syncLogReplMgrRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { return -1; } - int32_t ret = -1; - bool retried = false; - int64_t retryWaitMs = syncLogGetRetryBackoffTimeMs(pMgr); + int32_t ret = -1; + bool retried = false; + int64_t retryWaitMs = syncLogGetRetryBackoffTimeMs(pMgr); int64_t nowMs = taosGetMonoTimestampMs(); int count = 0; int64_t firstIndex = -1; @@ -642,7 +642,7 @@ int32_t syncLogReplMgrRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { } bool barrier = false; - if (syncLogBufferReplicateOneTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) { + if (syncLogReplMgrReplicateOneTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) { sError("vgId:%d, failed to replicate sync log entry since %s. index: %" PRId64 ", dest: %" PRIx64 "", pNode->vgId, terrstr(), index, pDestId->addr); goto _out; @@ -674,8 +674,7 @@ _out: return ret; } -int32_t syncLogReplMgrProcessReplyInRecoveryMode(SSyncLogReplMgr* pMgr, SSyncNode* pNode, - SyncAppendEntriesReply* pMsg) { +int32_t syncLogReplMgrProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg) { SSyncLogBuffer* pBuf = pNode->pLogBuf; SRaftId destId = pMsg->srcId; ASSERT(pMgr->restored == false); @@ -750,7 +749,7 @@ int32_t syncLogReplMgrProcessReplyInRecoveryMode(SSyncLogReplMgr* pMgr, SSyncNod // attempt to replicate the raft log at index (void)syncLogReplMgrReset(pMgr); - return syncLogReplMgrReplicateProbeOnce(pMgr, pNode, index); + return syncLogReplMgrReplicateProbe(pMgr, pNode, index); } int32_t syncLogReplMgrProcessHeartbeatReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncHeartbeatReply* pMsg) { @@ -778,9 +777,9 @@ int32_t syncLogReplMgrProcessReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, Sync } if (pMgr->restored) { - (void)syncLogReplMgrProcessReplyInNormalMode(pMgr, pNode, pMsg); + (void)syncLogReplMgrProcessReplyAsNormal(pMgr, pNode, pMsg); } else { - (void)syncLogReplMgrProcessReplyInRecoveryMode(pMgr, pNode, pMsg); + (void)syncLogReplMgrProcessReplyAsRecovery(pMgr, pNode, pMsg); } taosThreadMutexUnlock(&pBuf->mutex); return 0; @@ -788,14 +787,14 @@ int32_t syncLogReplMgrProcessReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, Sync int32_t syncLogReplMgrReplicateOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { if (pMgr->restored) { - (void)syncLogReplMgrReplicateAttemptedOnce(pMgr, pNode); + (void)syncLogReplMgrReplicateAttempt(pMgr, pNode); } else { - (void)syncLogReplMgrReplicateProbeOnce(pMgr, pNode, pNode->pLogBuf->matchIndex); + (void)syncLogReplMgrReplicateProbe(pMgr, pNode, pNode->pLogBuf->matchIndex); } return 0; } -int32_t syncLogReplMgrReplicateProbeOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index) { +int32_t syncLogReplMgrReplicateProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index) { ASSERT(!pMgr->restored); ASSERT(pMgr->startIndex >= 0); int64_t retryMaxWaitMs = syncGetRetryMaxWaitMs(); @@ -807,10 +806,10 @@ int32_t syncLogReplMgrReplicateProbeOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode } (void)syncLogReplMgrReset(pMgr); - SRaftId* pDestId = &pNode->replicasId[pMgr->peerId]; - bool barrier = false; - SyncTerm term = -1; - if (syncLogBufferReplicateOneTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) { + SRaftId* pDestId = &pNode->replicasId[pMgr->peerId]; + bool barrier = false; + SyncTerm term = -1; + if (syncLogReplMgrReplicateOneTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) { sError("vgId:%d, failed to replicate log entry since %s. index: %" PRId64 ", dest: 0x%016" PRIx64 "", pNode->vgId, terrstr(), index, pDestId->addr); return -1; @@ -833,14 +832,14 @@ int32_t syncLogReplMgrReplicateProbeOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode return 0; } -int32_t syncLogReplMgrReplicateAttemptedOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { +int32_t syncLogReplMgrReplicateAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { ASSERT(pMgr->restored); - SRaftId* pDestId = &pNode->replicasId[pMgr->peerId]; - int32_t batchSize = TMAX(1, pMgr->size >> (4 + pMgr->retryBackoff)); - int32_t count = 0; - int64_t nowMs = taosGetMonoTimestampMs(); - int64_t limit = pMgr->size >> 1; + SRaftId* pDestId = &pNode->replicasId[pMgr->peerId]; + int32_t batchSize = TMAX(1, pMgr->size >> (4 + pMgr->retryBackoff)); + int32_t count = 0; + int64_t nowMs = taosGetMonoTimestampMs(); + int64_t limit = pMgr->size >> 1; SyncTerm term = -1; SyncIndex firstIndex = -1; @@ -854,7 +853,8 @@ int32_t syncLogReplMgrReplicateAttemptedOnce(SSyncLogReplMgr* pMgr, SSyncNode* p int64_t pos = index % pMgr->size; SRaftId* pDestId = &pNode->replicasId[pMgr->peerId]; bool barrier = false; - if (syncLogBufferReplicateOneTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) { + SyncTerm term = -1; + if (syncLogReplMgrReplicateOneTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) { sError("vgId:%d, failed to replicate log entry since %s. index: %" PRId64 ", dest: 0x%016" PRIx64 "", pNode->vgId, terrstr(), index, pDestId->addr); return -1; @@ -888,16 +888,16 @@ int32_t syncLogReplMgrReplicateAttemptedOnce(SSyncLogReplMgr* pMgr, SSyncNode* p return 0; } -int32_t syncLogReplMgrProcessReplyInNormalMode(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg) { +int32_t syncLogReplMgrProcessReplyAsNormal(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg) { ASSERT(pMgr->restored == true); if (pMgr->startIndex <= pMsg->lastSendIndex && pMsg->lastSendIndex < pMgr->endIndex) { - if (pMgr->startIndex < pMgr->matchIndex && pMgr->retryBackoff > 0) { - int64_t firstSentMs = pMgr->states[pMgr->startIndex % pMgr->size].timeMs; - int64_t lastSentMs = pMgr->states[(pMgr->endIndex - 1) % pMgr->size].timeMs; - int64_t timeDiffMs = lastSentMs - firstSentMs; - if (timeDiffMs > 0 && timeDiffMs < (SYNC_LOG_REPL_RETRY_WAIT_MS << (pMgr->retryBackoff - 1))) { - pMgr->retryBackoff -= 1; - } + if (pMgr->startIndex < pMgr->matchIndex && pMgr->retryBackoff > 0) { + int64_t firstSentMs = pMgr->states[pMgr->startIndex % pMgr->size].timeMs; + int64_t lastSentMs = pMgr->states[(pMgr->endIndex - 1) % pMgr->size].timeMs; + int64_t timeDiffMs = lastSentMs - firstSentMs; + if (timeDiffMs > 0 && timeDiffMs < (SYNC_LOG_REPL_RETRY_WAIT_MS << (pMgr->retryBackoff - 1))) { + pMgr->retryBackoff -= 1; + } } pMgr->states[pMsg->lastSendIndex % pMgr->size].acked = true; pMgr->matchIndex = TMAX(pMgr->matchIndex, pMsg->matchIndex); @@ -907,7 +907,7 @@ int32_t syncLogReplMgrProcessReplyInNormalMode(SSyncLogReplMgr* pMgr, SSyncNode* pMgr->startIndex = pMgr->matchIndex; } - return syncLogReplMgrReplicateAttemptedOnce(pMgr, pNode); + return syncLogReplMgrReplicateAttempt(pMgr, pNode); } SSyncLogReplMgr* syncLogReplMgrCreate() { @@ -1101,12 +1101,11 @@ SSyncRaftEntry* syncLogBufferGetOneEntry(SSyncLogBuffer* pBuf, SSyncNode* pNode, return pEntry; } -int32_t syncLogBufferReplicateOneTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index, SyncTerm* pTerm, - SRaftId* pDestId, bool* pBarrier) { +int32_t syncLogReplMgrReplicateOneTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index, SyncTerm* pTerm, + SRaftId* pDestId, bool* pBarrier) { SSyncRaftEntry* pEntry = NULL; SRpcMsg msgOut = {0}; bool inBuf = false; - int32_t ret = -1; SyncTerm prevLogTerm = -1; SSyncLogBuffer* pBuf = pNode->pLogBuf; @@ -1132,14 +1131,13 @@ int32_t syncLogBufferReplicateOneTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, Syn } if (pTerm) *pTerm = pEntry->term; - int32_t code = syncBuildAppendEntriesFromRaftLog(pNode, pEntry, prevLogTerm, &msgOut); + int32_t code = syncBuildAppendEntriesFromRaftEntry(pNode, pEntry, prevLogTerm, &msgOut); if (code < 0) { sError("vgId:%d, failed to get append entries for index:%" PRId64 "", pNode->vgId, index); goto _err; } (void)syncNodeSendAppendEntries(pNode, pDestId, &msgOut); - ret = 0; sTrace("vgId:%d, replicate one msg index: %" PRId64 " term: %" PRId64 " prevterm: %" PRId64 " to dest: 0x%016" PRIx64, pNode->vgId, pEntry->index, pEntry->term, prevLogTerm, pDestId->addr); diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 03c3fe154d..ca6d3c314f 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -192,6 +192,8 @@ SyncTerm raftLogLastTerm(struct SSyncLogStore* pLogStore) { return SYNC_TERM_INVALID; } +static inline bool raftLogForceSync(SSyncRaftEntry* pEntry) { return (pEntry->originalRpcType == TDMT_VND_COMMIT); } + static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; @@ -219,9 +221,8 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr ASSERT(pEntry->index == index); - if (pEntry->originalRpcType == TDMT_VND_COMMIT) { - walFsync(pWal, true); - } + bool forceSync = raftLogForceSync(pEntry); + walFsync(pWal, forceSync); sNTrace(pData->pSyncNode, "write index:%" PRId64 ", type:%s, origin type:%s, elapsed:%" PRId64, pEntry->index, TMSG_INFO(pEntry->msgType), TMSG_INFO(pEntry->originalRpcType), tsElapsed); diff --git a/source/libs/sync/src/syncRaftStore.c b/source/libs/sync/src/syncRaftStore.c index b19cda2a44..197d1463fd 100644 --- a/source/libs/sync/src/syncRaftStore.c +++ b/source/libs/sync/src/syncRaftStore.c @@ -16,156 +16,161 @@ #define _DEFAULT_SOURCE #include "syncRaftStore.h" #include "syncUtil.h" +#include "tjson.h" -// private function -static int32_t raftStoreInit(SRaftStore *pRaftStore); -static bool raftStoreFileExist(char *path); +static int32_t raftStoreDecode(const SJson *pJson, SRaftStore *pStore) { + int32_t code = 0; -// public function -SRaftStore *raftStoreOpen(const char *path) { - int32_t ret; + tjsonGetNumberValue(pJson, "current_term", pStore->currentTerm, code); + if (code < 0) return -1; + tjsonGetNumberValue(pJson, "vote_for_addr", pStore->voteFor.addr, code); + if (code < 0) return -1; + tjsonGetInt32ValueFromDouble(pJson, "vote_for_vgid", pStore->voteFor.vgId, code); + if (code < 0) return -1; - SRaftStore *pRaftStore = taosMemoryCalloc(1, sizeof(SRaftStore)); - if (pRaftStore == NULL) { + return 0; +} + +int32_t raftStoreReadFile(SSyncNode *pNode) { + int32_t code = -1; + TdFilePtr pFile = NULL; + char *pData = NULL; + SJson *pJson = NULL; + const char *file = pNode->raftStorePath; + SRaftStore *pStore = &pNode->raftStore; + + if (taosStatFile(file, NULL, NULL) < 0) { + sInfo("vgId:%d, raft store file:%s not exist, use default value", pNode->vgId, file); + pStore->currentTerm = 0; + pStore->voteFor.addr = 0; + pStore->voteFor.vgId = 0; + return raftStoreWriteFile(pNode); + } + + pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + sError("vgId:%d, failed to open raft store file:%s since %s", pNode->vgId, file, terrstr()); + goto _OVER; + } + + int64_t size = 0; + if (taosFStatFile(pFile, &size, NULL) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + sError("vgId:%d, failed to fstat raft store file:%s since %s", pNode->vgId, file, terrstr()); + goto _OVER; + } + + pData = taosMemoryMalloc(size + 1); + if (pData == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + goto _OVER; } - snprintf(pRaftStore->path, sizeof(pRaftStore->path), "%s", path); - if (!raftStoreFileExist(pRaftStore->path)) { - ret = raftStoreInit(pRaftStore); - ASSERT(ret == 0); + if (taosReadFile(pFile, pData, size) != size) { + terrno = TAOS_SYSTEM_ERROR(errno); + sError("vgId:%d, failed to read raft store file:%s since %s", pNode->vgId, file, terrstr()); + goto _OVER; } - char storeBuf[RAFT_STORE_BLOCK_SIZE] = {0}; - pRaftStore->pFile = taosOpenFile(path, TD_FILE_READ | TD_FILE_WRITE); - ASSERT(pRaftStore->pFile != NULL); + pData[size] = '\0'; - int len = taosReadFile(pRaftStore->pFile, storeBuf, RAFT_STORE_BLOCK_SIZE); - ASSERT(len > 0); + pJson = tjsonParse(pData); + if (pJson == NULL) { + terrno = TSDB_CODE_INVALID_JSON_FORMAT; + goto _OVER; + } - ret = raftStoreDeserialize(pRaftStore, storeBuf, len); - ASSERT(ret == 0); + if (raftStoreDecode(pJson, pStore) < 0) { + terrno = TSDB_CODE_INVALID_JSON_FORMAT; + goto _OVER; + } - return pRaftStore; + code = 0; + sInfo("vgId:%d, succceed to read raft store file %s", pNode->vgId, file); + +_OVER: + if (pData != NULL) taosMemoryFree(pData); + if (pJson != NULL) cJSON_Delete(pJson); + if (pFile != NULL) taosCloseFile(&pFile); + + if (code != 0) { + sError("vgId:%d, failed to read raft store file:%s since %s", pNode->vgId, file, terrstr()); + } + return code; } -static int32_t raftStoreInit(SRaftStore *pRaftStore) { - ASSERT(pRaftStore != NULL); - - pRaftStore->pFile = taosOpenFile(pRaftStore->path, TD_FILE_CREATE | TD_FILE_WRITE); - ASSERT(pRaftStore->pFile != NULL); - - pRaftStore->currentTerm = 0; - pRaftStore->voteFor.addr = 0; - pRaftStore->voteFor.vgId = 0; - - int32_t ret = raftStorePersist(pRaftStore); - ASSERT(ret == 0); - - taosCloseFile(&pRaftStore->pFile); +static int32_t raftStoreEncode(SJson *pJson, SRaftStore *pStore) { + if (tjsonAddIntegerToObject(pJson, "current_term", pStore->currentTerm) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "vote_for_addr", pStore->voteFor.addr) < 0) return -1; + if (tjsonAddDoubleToObject(pJson, "vote_for_vgid", pStore->voteFor.vgId) < 0) return -1; return 0; } -int32_t raftStoreClose(SRaftStore *pRaftStore) { - if (pRaftStore == NULL) return 0; +int32_t raftStoreWriteFile(SSyncNode *pNode) { + int32_t code = -1; + char *buffer = NULL; + SJson *pJson = NULL; + TdFilePtr pFile = NULL; + const char *realfile = pNode->raftStorePath; + SRaftStore *pStore = &pNode->raftStore; + char file[PATH_MAX] = {0}; + snprintf(file, sizeof(file), "%s.bak", realfile); - taosCloseFile(&pRaftStore->pFile); - taosMemoryFree(pRaftStore); - pRaftStore = NULL; - return 0; + terrno = TSDB_CODE_OUT_OF_MEMORY; + pJson = tjsonCreateObject(); + if (pJson == NULL) goto _OVER; + if (raftStoreEncode(pJson, pStore) != 0) goto _OVER; + buffer = tjsonToString(pJson); + if (buffer == NULL) goto _OVER; + terrno = 0; + + pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) goto _OVER; + + int32_t len = strlen(buffer); + if (taosWriteFile(pFile, buffer, len) <= 0) goto _OVER; + if (taosFsyncFile(pFile) < 0) goto _OVER; + + taosCloseFile(&pFile); + if (taosRenameFile(file, realfile) != 0) goto _OVER; + + code = 0; + sInfo("vgId:%d, succeed to write raft store file:%s, len:%d", pNode->vgId, realfile, len); + +_OVER: + if (pJson != NULL) tjsonDelete(pJson); + if (buffer != NULL) taosMemoryFree(buffer); + if (pFile != NULL) taosCloseFile(&pFile); + + if (code != 0) { + if (terrno == 0) terrno = TAOS_SYSTEM_ERROR(errno); + sError("vgId:%d, failed to write raft store file:%s since %s", pNode->vgId, realfile, terrstr()); + } + return code; } -int32_t raftStorePersist(SRaftStore *pRaftStore) { - ASSERT(pRaftStore != NULL); - - int32_t ret; - char storeBuf[RAFT_STORE_BLOCK_SIZE] = {0}; - ret = raftStoreSerialize(pRaftStore, storeBuf, sizeof(storeBuf)); - ASSERT(ret == 0); - - taosLSeekFile(pRaftStore->pFile, 0, SEEK_SET); - - ret = taosWriteFile(pRaftStore->pFile, storeBuf, sizeof(storeBuf)); - ASSERT(ret == RAFT_STORE_BLOCK_SIZE); - - taosFsyncFile(pRaftStore->pFile); - return 0; -} - -static bool raftStoreFileExist(char *path) { - bool b = taosStatFile(path, NULL, NULL) >= 0; - return b; -} - -int32_t raftStoreSerialize(SRaftStore *pRaftStore, char *buf, size_t len) { - ASSERT(pRaftStore != NULL); - - cJSON *pRoot = cJSON_CreateObject(); - - char u64Buf[128] = {0}; - snprintf(u64Buf, sizeof(u64Buf), "%" PRIu64 "", pRaftStore->currentTerm); - cJSON_AddStringToObject(pRoot, "current_term", u64Buf); - - snprintf(u64Buf, sizeof(u64Buf), "%" PRIu64 "", pRaftStore->voteFor.addr); - cJSON_AddStringToObject(pRoot, "vote_for_addr", u64Buf); - - cJSON_AddNumberToObject(pRoot, "vote_for_vgid", pRaftStore->voteFor.vgId); - - char *serialized = cJSON_Print(pRoot); - int len2 = strlen(serialized); - ASSERT(len2 < len); - memset(buf, 0, len); - snprintf(buf, len, "%s", serialized); - taosMemoryFree(serialized); - - cJSON_Delete(pRoot); - return 0; -} - -int32_t raftStoreDeserialize(SRaftStore *pRaftStore, char *buf, size_t len) { - ASSERT(pRaftStore != NULL); - - ASSERT(len > 0 && len <= RAFT_STORE_BLOCK_SIZE); - cJSON *pRoot = cJSON_Parse(buf); - - cJSON *pCurrentTerm = cJSON_GetObjectItem(pRoot, "current_term"); - ASSERT(cJSON_IsString(pCurrentTerm)); - sscanf(pCurrentTerm->valuestring, "%" PRIu64 "", &(pRaftStore->currentTerm)); - - cJSON *pVoteForAddr = cJSON_GetObjectItem(pRoot, "vote_for_addr"); - ASSERT(cJSON_IsString(pVoteForAddr)); - sscanf(pVoteForAddr->valuestring, "%" PRIu64 "", &(pRaftStore->voteFor.addr)); - - cJSON *pVoteForVgid = cJSON_GetObjectItem(pRoot, "vote_for_vgid"); - pRaftStore->voteFor.vgId = pVoteForVgid->valueint; - - cJSON_Delete(pRoot); - return 0; -} - -bool raftStoreHasVoted(SRaftStore *pRaftStore) { - bool b = syncUtilEmptyId(&(pRaftStore->voteFor)); +bool raftStoreHasVoted(SSyncNode *pNode) { + bool b = syncUtilEmptyId(&pNode->raftStore.voteFor); return (!b); } -void raftStoreVote(SRaftStore *pRaftStore, SRaftId *pRaftId) { - ASSERT(!syncUtilEmptyId(pRaftId)); - pRaftStore->voteFor = *pRaftId; - raftStorePersist(pRaftStore); +void raftStoreVote(SSyncNode *pNode, SRaftId *pRaftId) { + pNode->raftStore.voteFor = *pRaftId; + (void)raftStoreWriteFile(pNode); } -void raftStoreClearVote(SRaftStore *pRaftStore) { - pRaftStore->voteFor = EMPTY_RAFT_ID; - raftStorePersist(pRaftStore); +void raftStoreClearVote(SSyncNode *pNode) { + pNode->raftStore.voteFor = EMPTY_RAFT_ID; + (void)raftStoreWriteFile(pNode); } -void raftStoreNextTerm(SRaftStore *pRaftStore) { - ++(pRaftStore->currentTerm); - raftStorePersist(pRaftStore); +void raftStoreNextTerm(SSyncNode *pNode) { + pNode->raftStore.currentTerm++; + (void)raftStoreWriteFile(pNode); } -void raftStoreSetTerm(SRaftStore *pRaftStore, SyncTerm term) { - pRaftStore->currentTerm = term; - raftStorePersist(pRaftStore); +void raftStoreSetTerm(SSyncNode *pNode, SyncTerm term) { + pNode->raftStore.currentTerm = term; + (void)raftStoreWriteFile(pNode); } diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index e3058768f8..1aa476e84e 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -122,7 +122,7 @@ int32_t syncNodeReplicateOne(SSyncNode* pSyncNode, SRaftId* pDestId, bool snapsh ASSERT(pMsg != NULL); pMsg->srcId = pSyncNode->myRaftId; pMsg->destId = *pDestId; - pMsg->term = pSyncNode->pRaftStore->currentTerm; + pMsg->term = pSyncNode->raftStore.currentTerm; pMsg->prevLogIndex = preLogIndex; pMsg->prevLogTerm = preLogTerm; pMsg->commitIndex = pSyncNode->commitIndex; @@ -245,7 +245,7 @@ int32_t syncNodeHeartbeatPeers(SSyncNode* pSyncNode) { SyncHeartbeat* pSyncMsg = rpcMsg.pCont; pSyncMsg->srcId = pSyncNode->myRaftId; pSyncMsg->destId = pSyncNode->peersId[i]; - pSyncMsg->term = pSyncNode->pRaftStore->currentTerm; + pSyncMsg->term = pSyncNode->raftStore.currentTerm; pSyncMsg->commitIndex = pSyncNode->commitIndex; pSyncMsg->minMatchIndex = syncMinMatchIndex(pSyncNode); pSyncMsg->privateTerm = 0; diff --git a/source/libs/sync/src/syncRequestVote.c b/source/libs/sync/src/syncRequestVote.c index 773befe1e4..069ea2ea88 100644 --- a/source/libs/sync/src/syncRequestVote.c +++ b/source/libs/sync/src/syncRequestVote.c @@ -44,21 +44,12 @@ // /\ UNCHANGED <> // -static bool syncNodeOnRequestVoteLogOK(SSyncNode* pSyncNode, SyncRequestVote* pMsg) { - SyncTerm myLastTerm = syncNodeGetLastTerm(pSyncNode); - SyncIndex myLastIndex = syncNodeGetLastIndex(pSyncNode); - - if (pMsg->lastLogIndex < pSyncNode->commitIndex) { - sNTrace(pSyncNode, - "logok:0, {my-lterm:%" PRIu64 ", my-lindex:%" PRId64 ", recv-lterm:%" PRIu64 ", recv-lindex:%" PRId64 - ", recv-term:%" PRIu64 "}", - myLastTerm, myLastIndex, pMsg->lastLogTerm, pMsg->lastLogIndex, pMsg->term); - - return false; - } +static bool syncNodeOnRequestVoteLogOK(SSyncNode* ths, SyncRequestVote* pMsg) { + SyncTerm myLastTerm = syncNodeGetLastTerm(ths); + SyncIndex myLastIndex = syncNodeGetLastIndex(ths); if (myLastTerm == SYNC_TERM_INVALID) { - sNTrace(pSyncNode, + sNTrace(ths, "logok:0, {my-lterm:%" PRIu64 ", my-lindex:%" PRId64 ", recv-lterm:%" PRIu64 ", recv-lindex:%" PRId64 ", recv-term:%" PRIu64 "}", myLastTerm, myLastIndex, pMsg->lastLogTerm, pMsg->lastLogIndex, pMsg->term); @@ -66,22 +57,29 @@ static bool syncNodeOnRequestVoteLogOK(SSyncNode* pSyncNode, SyncRequestVote* pM } if (pMsg->lastLogTerm > myLastTerm) { - sNTrace(pSyncNode, + sNTrace(ths, "logok:1, {my-lterm:%" PRIu64 ", my-lindex:%" PRId64 ", recv-lterm:%" PRIu64 ", recv-lindex:%" PRId64 ", recv-term:%" PRIu64 "}", myLastTerm, myLastIndex, pMsg->lastLogTerm, pMsg->lastLogIndex, pMsg->term); + + if (pMsg->lastLogIndex < ths->commitIndex) { + sNWarn(ths, + "logok:1, commit rollback required. {my-lterm:%" PRIu64 ", my-lindex:%" PRId64 ", recv-lterm:%" PRIu64 + ", recv-lindex:%" PRId64 ", recv-term:%" PRIu64 "}", + myLastTerm, myLastIndex, pMsg->lastLogTerm, pMsg->lastLogIndex, pMsg->term); + } return true; } if (pMsg->lastLogTerm == myLastTerm && pMsg->lastLogIndex >= myLastIndex) { - sNTrace(pSyncNode, + sNTrace(ths, "logok:1, {my-lterm:%" PRIu64 ", my-lindex:%" PRId64 ", recv-lterm:%" PRIu64 ", recv-lindex:%" PRId64 ", recv-term:%" PRIu64 "}", myLastTerm, myLastIndex, pMsg->lastLogTerm, pMsg->lastLogIndex, pMsg->term); return true; } - sNTrace(pSyncNode, + sNTrace(ths, "logok:0, {my-lterm:%" PRIu64 ", my-lindex:%" PRId64 ", recv-lterm:%" PRIu64 ", recv-lindex:%" PRId64 ", recv-term:%" PRIu64 "}", myLastTerm, myLastIndex, pMsg->lastLogTerm, pMsg->lastLogIndex, pMsg->term); @@ -93,7 +91,7 @@ int32_t syncNodeOnRequestVote(SSyncNode* ths, const SRpcMsg* pRpcMsg) { SyncRequestVote* pMsg = pRpcMsg->pCont; // if already drop replica, do not process - if (!syncNodeInRaftGroup(ths, &(pMsg->srcId))) { + if (!syncNodeInRaftGroup(ths, &pMsg->srcId)) { syncLogRecvRequestVote(ths, pMsg, -1, "not in my config"); return -1; } @@ -101,21 +99,21 @@ int32_t syncNodeOnRequestVote(SSyncNode* ths, const SRpcMsg* pRpcMsg) { bool logOK = syncNodeOnRequestVoteLogOK(ths, pMsg); // maybe update term - if (pMsg->term > ths->pRaftStore->currentTerm) { + if (pMsg->term > ths->raftStore.currentTerm) { syncNodeStepDown(ths, pMsg->term); // syncNodeUpdateTerm(ths, pMsg->term); } - ASSERT(pMsg->term <= ths->pRaftStore->currentTerm); + ASSERT(pMsg->term <= ths->raftStore.currentTerm); - bool grant = (pMsg->term == ths->pRaftStore->currentTerm) && logOK && - ((!raftStoreHasVoted(ths->pRaftStore)) || (syncUtilSameId(&(ths->pRaftStore->voteFor), &(pMsg->srcId)))); + bool grant = (pMsg->term == ths->raftStore.currentTerm) && logOK && + ((!raftStoreHasVoted(ths)) || (syncUtilSameId(&ths->raftStore.voteFor, &pMsg->srcId))); if (grant) { // maybe has already voted for pMsg->srcId // vote again, no harm - raftStoreVote(ths->pRaftStore, &(pMsg->srcId)); + raftStoreVote(ths, &(pMsg->srcId)); // candidate ? - syncNodeStepDown(ths, ths->pRaftStore->currentTerm); + syncNodeStepDown(ths, ths->raftStore.currentTerm); // forbid elect for this round syncNodeResetElectTimer(ths); @@ -129,7 +127,7 @@ int32_t syncNodeOnRequestVote(SSyncNode* ths, const SRpcMsg* pRpcMsg) { SyncRequestVoteReply* pReply = rpcMsg.pCont; pReply->srcId = ths->myRaftId; pReply->destId = pMsg->srcId; - pReply->term = ths->pRaftStore->currentTerm; + pReply->term = ths->raftStore.currentTerm; pReply->voteGranted = grant; // trace log @@ -137,4 +135,4 @@ int32_t syncNodeOnRequestVote(SSyncNode* ths, const SRpcMsg* pRpcMsg) { syncLogSendRequestVoteReply(ths, pReply, ""); syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); return 0; -} \ No newline at end of file +} diff --git a/source/libs/sync/src/syncRequestVoteReply.c b/source/libs/sync/src/syncRequestVoteReply.c index 563f475070..a0d6cbf597 100644 --- a/source/libs/sync/src/syncRequestVoteReply.c +++ b/source/libs/sync/src/syncRequestVoteReply.c @@ -49,25 +49,25 @@ int32_t syncNodeOnRequestVoteReply(SSyncNode* ths, const SRpcMsg* pRpcMsg) { } // drop stale response - if (pMsg->term < ths->pRaftStore->currentTerm) { + if (pMsg->term < ths->raftStore.currentTerm) { syncLogRecvRequestVoteReply(ths, pMsg, "drop stale response"); return -1; } - // ASSERT(!(pMsg->term > ths->pRaftStore->currentTerm)); + // ASSERT(!(pMsg->term > ths->raftStore.currentTerm)); // no need this code, because if I receive reply.term, then I must have sent for that term. - // if (pMsg->term > ths->pRaftStore->currentTerm) { + // if (pMsg->term > ths->raftStore.currentTerm) { // syncNodeUpdateTerm(ths, pMsg->term); // } - if (pMsg->term > ths->pRaftStore->currentTerm) { + if (pMsg->term > ths->raftStore.currentTerm) { syncLogRecvRequestVoteReply(ths, pMsg, "error term"); syncNodeStepDown(ths, pMsg->term); return -1; } syncLogRecvRequestVoteReply(ths, pMsg, ""); - ASSERT(pMsg->term == ths->pRaftStore->currentTerm); + ASSERT(pMsg->term == ths->raftStore.currentTerm); // This tallies votes even when the current state is not Candidate, // but they won't be looked at, so it doesn't matter. diff --git a/source/libs/sync/src/syncRespMgr.c b/source/libs/sync/src/syncRespMgr.c index b55aae4c76..9373eccaef 100644 --- a/source/libs/sync/src/syncRespMgr.c +++ b/source/libs/sync/src/syncRespMgr.c @@ -143,7 +143,7 @@ static void syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl, bool rsp) { .state = pNode->state, .seqNum = *pSeqNum, .term = SYNC_TERM_INVALID, - .currentTerm = pNode->pRaftStore->currentTerm, + .currentTerm = pNode->raftStore.currentTerm, .flag = 0, }; diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index defb7402f4..880c76e4dd 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -43,7 +43,7 @@ SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaI pSender->sendingMS = SYNC_SNAPSHOT_RETRY_MS; pSender->pSyncNode = pSyncNode; pSender->replicaIndex = replicaIndex; - pSender->term = pSyncNode->pRaftStore->currentTerm; + pSender->term = pSyncNode->raftStore.currentTerm; pSender->startTime = 0; pSender->endTime = 0; pSender->pSyncNode->pFsm->FpGetSnapshotInfo(pSender->pSyncNode->pFsm, &pSender->snapshot); @@ -90,7 +90,7 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender) { memset(&pSender->lastConfig, 0, sizeof(pSender->lastConfig)); pSender->sendingMS = 0; - pSender->term = pSender->pSyncNode->pRaftStore->currentTerm; + pSender->term = pSender->pSyncNode->raftStore.currentTerm; pSender->startTime = taosGetTimestampMs(); pSender->lastSendTime = pSender->startTime; pSender->finish = false; @@ -105,7 +105,7 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender) { SyncSnapshotSend *pMsg = rpcMsg.pCont; pMsg->srcId = pSender->pSyncNode->myRaftId; pMsg->destId = pSender->pSyncNode->replicasId[pSender->replicaIndex]; - pMsg->term = pSender->pSyncNode->pRaftStore->currentTerm; + pMsg->term = pSender->pSyncNode->raftStore.currentTerm; pMsg->beginIndex = pSender->snapshotParam.start; pMsg->lastIndex = pSender->snapshot.lastApplyIndex; pMsg->lastTerm = pSender->snapshot.lastApplyTerm; @@ -185,7 +185,7 @@ static int32_t snapshotSend(SSyncSnapshotSender *pSender) { SyncSnapshotSend *pMsg = rpcMsg.pCont; pMsg->srcId = pSender->pSyncNode->myRaftId; pMsg->destId = pSender->pSyncNode->replicasId[pSender->replicaIndex]; - pMsg->term = pSender->pSyncNode->pRaftStore->currentTerm; + pMsg->term = pSender->pSyncNode->raftStore.currentTerm; pMsg->beginIndex = pSender->snapshotParam.start; pMsg->lastIndex = pSender->snapshot.lastApplyIndex; pMsg->lastTerm = pSender->snapshot.lastApplyTerm; @@ -226,7 +226,7 @@ int32_t snapshotReSend(SSyncSnapshotSender *pSender) { SyncSnapshotSend *pMsg = rpcMsg.pCont; pMsg->srcId = pSender->pSyncNode->myRaftId; pMsg->destId = pSender->pSyncNode->replicasId[pSender->replicaIndex]; - pMsg->term = pSender->pSyncNode->pRaftStore->currentTerm; + pMsg->term = pSender->pSyncNode->raftStore.currentTerm; pMsg->beginIndex = pSender->snapshotParam.start; pMsg->lastIndex = pSender->snapshot.lastApplyIndex; pMsg->lastTerm = pSender->snapshot.lastApplyTerm; @@ -314,7 +314,7 @@ SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId from pReceiver->pWriter = NULL; pReceiver->pSyncNode = pSyncNode; pReceiver->fromId = fromId; - pReceiver->term = pSyncNode->pRaftStore->currentTerm; + pReceiver->term = pSyncNode->raftStore.currentTerm; pReceiver->snapshot.data = NULL; pReceiver->snapshot.lastApplyIndex = SYNC_INDEX_INVALID; pReceiver->snapshot.lastApplyTerm = 0; @@ -380,7 +380,7 @@ void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *p pReceiver->start = true; pReceiver->ack = SYNC_SNAPSHOT_SEQ_PRE_SNAPSHOT; - pReceiver->term = pReceiver->pSyncNode->pRaftStore->currentTerm; + pReceiver->term = pReceiver->pSyncNode->raftStore.currentTerm; pReceiver->fromId = pPreMsg->srcId; pReceiver->startTime = pPreMsg->startTime; @@ -437,9 +437,9 @@ static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnap } // maybe update term - if (pReceiver->snapshot.lastApplyTerm > pReceiver->pSyncNode->pRaftStore->currentTerm) { - pReceiver->pSyncNode->pRaftStore->currentTerm = pReceiver->snapshot.lastApplyTerm; - raftStorePersist(pReceiver->pSyncNode->pRaftStore); + if (pReceiver->snapshot.lastApplyTerm > pReceiver->pSyncNode->raftStore.currentTerm) { + pReceiver->pSyncNode->raftStore.currentTerm = pReceiver->snapshot.lastApplyTerm; + (void)raftStoreWriteFile(pReceiver->pSyncNode); } // stop writer, apply data @@ -592,7 +592,7 @@ _SEND_REPLY: SyncSnapshotRsp *pRspMsg = rpcMsg.pCont; pRspMsg->srcId = pSyncNode->myRaftId; pRspMsg->destId = pMsg->srcId; - pRspMsg->term = pSyncNode->pRaftStore->currentTerm; + pRspMsg->term = pSyncNode->raftStore.currentTerm; pRspMsg->lastIndex = pMsg->lastIndex; pRspMsg->lastTerm = pMsg->lastTerm; pRspMsg->startTime = pReceiver->startTime; @@ -648,7 +648,7 @@ _SEND_REPLY: SyncSnapshotRsp *pRspMsg = rpcMsg.pCont; pRspMsg->srcId = pSyncNode->myRaftId; pRspMsg->destId = pMsg->srcId; - pRspMsg->term = pSyncNode->pRaftStore->currentTerm; + pRspMsg->term = pSyncNode->raftStore.currentTerm; pRspMsg->lastIndex = pMsg->lastIndex; pRspMsg->lastTerm = pMsg->lastTerm; pRspMsg->startTime = pReceiver->startTime; @@ -698,7 +698,7 @@ static int32_t syncNodeOnSnapshotReceive(SSyncNode *pSyncNode, SyncSnapshotSend SyncSnapshotRsp *pRspMsg = rpcMsg.pCont; pRspMsg->srcId = pSyncNode->myRaftId; pRspMsg->destId = pMsg->srcId; - pRspMsg->term = pSyncNode->pRaftStore->currentTerm; + pRspMsg->term = pSyncNode->raftStore.currentTerm; pRspMsg->lastIndex = pMsg->lastIndex; pRspMsg->lastTerm = pMsg->lastTerm; pRspMsg->startTime = pReceiver->startTime; @@ -745,7 +745,7 @@ static int32_t syncNodeOnSnapshotEnd(SSyncNode *pSyncNode, SyncSnapshotSend *pMs SyncSnapshotRsp *pRspMsg = rpcMsg.pCont; pRspMsg->srcId = pSyncNode->myRaftId; pRspMsg->destId = pMsg->srcId; - pRspMsg->term = pSyncNode->pRaftStore->currentTerm; + pRspMsg->term = pSyncNode->raftStore.currentTerm; pRspMsg->lastIndex = pMsg->lastIndex; pRspMsg->lastTerm = pMsg->lastTerm; pRspMsg->startTime = pReceiver->startTime; @@ -794,13 +794,13 @@ int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, const SRpcMsg *pRpcMsg) { return -1; } - if (pMsg->term < pSyncNode->pRaftStore->currentTerm) { + if (pMsg->term < pSyncNode->raftStore.currentTerm) { syncLogRecvSyncSnapshotSend(pSyncNode, pMsg, "reject since small term"); terrno = TSDB_CODE_SYN_INTERNAL_ERROR; return -1; } - if (pMsg->term > pSyncNode->pRaftStore->currentTerm) { + if (pMsg->term > pSyncNode->raftStore.currentTerm) { syncNodeStepDown(pSyncNode, pMsg->term); } syncNodeResetElectTimer(pSyncNode); @@ -808,7 +808,7 @@ int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, const SRpcMsg *pRpcMsg) { // state, term, seq/ack int32_t code = 0; if (pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER) { - if (pMsg->term == pSyncNode->pRaftStore->currentTerm) { + if (pMsg->term == pSyncNode->raftStore.currentTerm) { if (pMsg->seq == SYNC_SNAPSHOT_SEQ_PRE_SNAPSHOT) { syncLogRecvSyncSnapshotSend(pSyncNode, pMsg, "process seq pre-snapshot"); code = syncNodeOnSnapshotPre(pSyncNode, pMsg); @@ -892,7 +892,7 @@ static int32_t syncNodeOnSnapshotPreRsp(SSyncNode *pSyncNode, SSyncSnapshotSende SyncSnapshotSend *pSendMsg = rpcMsg.pCont; pSendMsg->srcId = pSender->pSyncNode->myRaftId; pSendMsg->destId = pSender->pSyncNode->replicasId[pSender->replicaIndex]; - pSendMsg->term = pSender->pSyncNode->pRaftStore->currentTerm; + pSendMsg->term = pSender->pSyncNode->raftStore.currentTerm; pSendMsg->beginIndex = pSender->snapshotParam.start; pSendMsg->lastIndex = pSender->snapshot.lastApplyIndex; pSendMsg->lastTerm = pSender->snapshot.lastApplyTerm; @@ -951,10 +951,10 @@ int32_t syncNodeOnSnapshotRsp(SSyncNode *pSyncNode, const SRpcMsg *pRpcMsg) { goto _ERROR; } - if (pMsg->term != pSyncNode->pRaftStore->currentTerm) { + if (pMsg->term != pSyncNode->raftStore.currentTerm) { syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "snapshot sender and receiver term not match"); sSError(pSender, "snapshot sender term not equal, msg term:%" PRId64 " currentTerm:%" PRId64, pMsg->term, - pSyncNode->pRaftStore->currentTerm); + pSyncNode->raftStore.currentTerm); terrno = TSDB_CODE_SYN_INTERNAL_ERROR; goto _ERROR; } diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index e4a65837f7..b246d9a79d 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -158,8 +158,8 @@ static void syncPeerState2Str(SSyncNode* pSyncNode, char* buf, int32_t bufLen) { } void syncPrintNodeLog(const char* flags, ELogLevel level, int32_t dflag, SSyncNode* pNode, const char* format, ...) { - if (pNode == NULL || pNode->pRaftStore == NULL || pNode->pLogStore == NULL) return; - int64_t currentTerm = pNode->pRaftStore->currentTerm; + if (pNode == NULL || pNode->pLogStore == NULL) return; + int64_t currentTerm = pNode->raftStore.currentTerm; // save error code, otherwise it will be overwritten int32_t errCode = terrno; @@ -228,7 +228,7 @@ void syncPrintNodeLog(const char* flags, ELogLevel level, int32_t dflag, SSyncNo void syncPrintSnapshotSenderLog(const char* flags, ELogLevel level, int32_t dflag, SSyncSnapshotSender* pSender, const char* format, ...) { SSyncNode* pNode = pSender->pSyncNode; - if (pNode == NULL || pNode->pRaftStore == NULL || pNode->pLogStore == NULL) return; + if (pNode == NULL || pNode->pLogStore == NULL) return; SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; if (pNode->pFsm != NULL && pNode->pFsm->FpGetSnapshotInfo != NULL) { @@ -264,7 +264,7 @@ void syncPrintSnapshotSenderLog(const char* flags, ELogLevel level, int32_t dfla pNode->vgId, eventLog, syncStr(pNode->state), pSender, pSender->snapshotParam.start, pSender->snapshotParam.end, pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, pSender->seq, pSender->ack, pSender->finish, pSender->replicaIndex, - DID(&pNode->replicasId[pSender->replicaIndex]), pNode->pRaftStore->currentTerm, pNode->commitIndex, + DID(&pNode->replicasId[pSender->replicaIndex]), pNode->raftStore.currentTerm, pNode->commitIndex, logBeginIndex, logLastIndex, pNode->minMatchIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm, pNode->raftCfg.isStandBy, pNode->raftCfg.snapshotStrategy, pNode->raftCfg.batchSize, pNode->replicaNum, pNode->raftCfg.lastConfigIndex, pNode->changing, pNode->restoreFinish, syncNodeDynamicQuorum(pNode), @@ -274,7 +274,7 @@ void syncPrintSnapshotSenderLog(const char* flags, ELogLevel level, int32_t dfla void syncPrintSnapshotReceiverLog(const char* flags, ELogLevel level, int32_t dflag, SSyncSnapshotReceiver* pReceiver, const char* format, ...) { SSyncNode* pNode = pReceiver->pSyncNode; - if (pNode == NULL || pNode->pRaftStore == NULL || pNode->pLogStore == NULL) return; + if (pNode == NULL || pNode->pLogStore == NULL) return; SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; if (pNode->pFsm != NULL && pNode->pFsm->FpGetSnapshotInfo != NULL) { @@ -311,7 +311,7 @@ void syncPrintSnapshotReceiverLog(const char* flags, ELogLevel level, int32_t df pNode->vgId, eventLog, syncStr(pNode->state), pReceiver, pReceiver->start, pReceiver->ack, pReceiver->term, pReceiver->startTime, DID(&pReceiver->fromId), pReceiver->snapshotParam.start, pReceiver->snapshotParam.end, pReceiver->snapshot.lastApplyIndex, pReceiver->snapshot.lastApplyTerm, - pReceiver->snapshot.lastConfigIndex, pNode->pRaftStore->currentTerm, pNode->commitIndex, logBeginIndex, + pReceiver->snapshot.lastConfigIndex, pNode->raftStore.currentTerm, pNode->commitIndex, logBeginIndex, logLastIndex, pNode->minMatchIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm, pNode->raftCfg.isStandBy, pNode->raftCfg.snapshotStrategy, pNode->raftCfg.batchSize, pNode->replicaNum, pNode->raftCfg.lastConfigIndex, pNode->changing, pNode->restoreFinish, syncNodeDynamicQuorum(pNode), diff --git a/source/libs/sync/test/syncLocalCmdTest.cpp b/source/libs/sync/test/syncLocalCmdTest.cpp index 8003cce7cc..2c839d0acb 100644 --- a/source/libs/sync/test/syncLocalCmdTest.cpp +++ b/source/libs/sync/test/syncLocalCmdTest.cpp @@ -16,8 +16,8 @@ SyncLocalCmd *createMsg() { pMsg->srcId.vgId = 100; pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678); pMsg->destId.vgId = 100; - pMsg->sdNewTerm = 123; - pMsg->fcIndex = 456; + // pMsg->sdNewTerm = 123; + // pMsg->fcIndex = 456; pMsg->cmd = SYNC_LOCAL_CMD_STEP_DOWN; return pMsg; diff --git a/source/libs/sync/test/syncRaftStoreTest.cpp b/source/libs/sync/test/syncRaftStoreTest.cpp index 87798a7d80..a8022184ef 100644 --- a/source/libs/sync/test/syncRaftStoreTest.cpp +++ b/source/libs/sync/test/syncRaftStoreTest.cpp @@ -33,35 +33,35 @@ int main() { initRaftId(); - SRaftStore* pRaftStore = raftStoreOpen("./test_raft_store.json"); - assert(pRaftStore != NULL); - raftStoreLog2((char*)"==raftStoreOpen==", pRaftStore); + // SRaftStore* pRaftStore = raftStoreOpen("./test_raft_store.json"); + // assert(pRaftStore != NULL); + // raftStoreLog2((char*)"==raftStoreOpen==", pRaftStore); - raftStoreSetTerm(pRaftStore, 100); - raftStoreLog2((char*)"==raftStoreSetTerm==", pRaftStore); + // raftStoreSetTerm(pRaftStore, 100); + // raftStoreLog2((char*)"==raftStoreSetTerm==", pRaftStore); - raftStoreVote(pRaftStore, &ids[0]); - raftStoreLog2((char*)"==raftStoreVote==", pRaftStore); + // raftStoreVote(pRaftStore, &ids[0]); + // raftStoreLog2((char*)"==raftStoreVote==", pRaftStore); - raftStoreClearVote(pRaftStore); - raftStoreLog2((char*)"==raftStoreClearVote==", pRaftStore); + // raftStoreClearVote(pRaftStore); + // raftStoreLog2((char*)"==raftStoreClearVote==", pRaftStore); - raftStoreVote(pRaftStore, &ids[1]); - raftStoreLog2((char*)"==raftStoreVote==", pRaftStore); + // raftStoreVote(pRaftStore, &ids[1]); + // raftStoreLog2((char*)"==raftStoreVote==", pRaftStore); - raftStoreNextTerm(pRaftStore); - raftStoreLog2((char*)"==raftStoreNextTerm==", pRaftStore); + // raftStoreNextTerm(pRaftStore); + // raftStoreLog2((char*)"==raftStoreNextTerm==", pRaftStore); - raftStoreNextTerm(pRaftStore); - raftStoreLog2((char*)"==raftStoreNextTerm==", pRaftStore); + // raftStoreNextTerm(pRaftStore); + // raftStoreLog2((char*)"==raftStoreNextTerm==", pRaftStore); - raftStoreNextTerm(pRaftStore); - raftStoreLog2((char*)"==raftStoreNextTerm==", pRaftStore); + // raftStoreNextTerm(pRaftStore); + // raftStoreLog2((char*)"==raftStoreNextTerm==", pRaftStore); - raftStoreNextTerm(pRaftStore); - raftStoreLog2((char*)"==raftStoreNextTerm==", pRaftStore); + // raftStoreNextTerm(pRaftStore); + // raftStoreLog2((char*)"==raftStoreNextTerm==", pRaftStore); - raftStoreClose(pRaftStore); + // raftStoreClose(pRaftStore); return 0; } diff --git a/source/libs/sync/test/syncSnapshotReceiverTest.cpp b/source/libs/sync/test/syncSnapshotReceiverTest.cpp index 49b06a7d1b..1fca04a1ad 100644 --- a/source/libs/sync/test/syncSnapshotReceiverTest.cpp +++ b/source/libs/sync/test/syncSnapshotReceiverTest.cpp @@ -29,7 +29,7 @@ int32_t SnapshotDoWrite(struct SSyncFSM* pFsm, void* pWriter, void* pBuf, int32_ SSyncSnapshotReceiver* createReceiver() { SSyncNode* pSyncNode = (SSyncNode*)taosMemoryMalloc(sizeof(*pSyncNode)); - pSyncNode->pRaftStore = (SRaftStore*)taosMemoryMalloc(sizeof(*(pSyncNode->pRaftStore))); + // pSyncNode->pRaftStore = (SRaftStore*)taosMemoryMalloc(sizeof(*(pSyncNode->pRaftStore))); pSyncNode->pFsm = (SSyncFSM*)taosMemoryMalloc(sizeof(*(pSyncNode->pFsm))); #if 0 diff --git a/source/libs/sync/test/syncSnapshotSenderTest.cpp b/source/libs/sync/test/syncSnapshotSenderTest.cpp index bb697d541a..a1768c2ce5 100644 --- a/source/libs/sync/test/syncSnapshotSenderTest.cpp +++ b/source/libs/sync/test/syncSnapshotSenderTest.cpp @@ -29,7 +29,7 @@ int32_t SnapshotDoWrite(struct SSyncFSM* pFsm, void* pWriter, void* pBuf, int32_ SSyncSnapshotSender* createSender() { SSyncNode* pSyncNode = (SSyncNode*)taosMemoryMalloc(sizeof(*pSyncNode)); - pSyncNode->pRaftStore = (SRaftStore*)taosMemoryMalloc(sizeof(*(pSyncNode->pRaftStore))); + // pSyncNode->pRaftStore = (SRaftStore*)taosMemoryMalloc(sizeof(*(pSyncNode->pRaftStore))); pSyncNode->pFsm = (SSyncFSM*)taosMemoryMalloc(sizeof(*(pSyncNode->pFsm))); #if 0 diff --git a/source/libs/sync/test/sync_test_lib/src/syncMainDebug.c b/source/libs/sync/test/sync_test_lib/src/syncMainDebug.c index f1db2f0204..1dbf4fb4fb 100644 --- a/source/libs/sync/test/sync_test_lib/src/syncMainDebug.c +++ b/source/libs/sync/test/sync_test_lib/src/syncMainDebug.c @@ -80,7 +80,7 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) { // tla+ server vars cJSON_AddNumberToObject(pRoot, "state", pSyncNode->state); cJSON_AddStringToObject(pRoot, "state_str", syncStr(pSyncNode->state)); - cJSON_AddItemToObject(pRoot, "pRaftStore", raftStore2Json(pSyncNode->pRaftStore)); + // cJSON_AddItemToObject(pRoot, "pRaftStore", raftStore2Json(&pSyncNode.raftStore)); // tla+ candidate vars cJSON_AddItemToObject(pRoot, "pVotesGranted", voteGranted2Json(pSyncNode->pVotesGranted)); @@ -199,7 +199,7 @@ inline char* syncNode2SimpleStr(const SSyncNode* pSyncNode) { ", sby:%d, " "r-num:%d, " "lcfg:%" PRId64 ", chging:%d, rsto:%d", - pSyncNode->vgId, syncStr(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, + pSyncNode->vgId, syncStr(pSyncNode->state), pSyncNode->raftStore.currentTerm, pSyncNode->commitIndex, logBeginIndex, logLastIndex, snapshot.lastApplyIndex, pSyncNode->raftCfg.isStandBy, pSyncNode->replicaNum, pSyncNode->raftCfg.lastConfigIndex, pSyncNode->changing, pSyncNode->restoreFinish); diff --git a/source/libs/sync/test/sync_test_lib/src/syncMessageDebug.c b/source/libs/sync/test/sync_test_lib/src/syncMessageDebug.c index ae83bf9ead..5f011ffe69 100644 --- a/source/libs/sync/test/sync_test_lib/src/syncMessageDebug.c +++ b/source/libs/sync/test/sync_test_lib/src/syncMessageDebug.c @@ -2858,11 +2858,11 @@ cJSON* syncLocalCmd2Json(const SyncLocalCmd* pMsg) { cJSON_AddNumberToObject(pRoot, "cmd", pMsg->cmd); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->sdNewTerm); - cJSON_AddStringToObject(pRoot, "sd-new-term", u64buf); + // snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->sdNewTerm); + // cJSON_AddStringToObject(pRoot, "sd-new-term", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->fcIndex); - cJSON_AddStringToObject(pRoot, "fc-index", u64buf); + // snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->fcIndex); + // cJSON_AddStringToObject(pRoot, "fc-index", u64buf); } cJSON* pJson = cJSON_CreateObject(); diff --git a/source/libs/sync/test/sync_test_lib/src/syncRaftStoreDebug.c b/source/libs/sync/test/sync_test_lib/src/syncRaftStoreDebug.c index c462b3275d..f6cd381e54 100644 --- a/source/libs/sync/test/sync_test_lib/src/syncRaftStoreDebug.c +++ b/source/libs/sync/test/sync_test_lib/src/syncRaftStoreDebug.c @@ -41,8 +41,8 @@ cJSON *raftStore2Json(SRaftStore *pRaftStore) { cJSON_AddNumberToObject(pVoteFor, "vgId", pRaftStore->voteFor.vgId); cJSON_AddItemToObject(pRoot, "voteFor", pVoteFor); - int hasVoted = raftStoreHasVoted(pRaftStore); - cJSON_AddNumberToObject(pRoot, "hasVoted", hasVoted); + // int hasVoted = raftStoreHasVoted(pRaftStore); + // cJSON_AddNumberToObject(pRoot, "hasVoted", hasVoted); } cJSON *pJson = cJSON_CreateObject(); diff --git a/source/libs/sync/test/sync_test_lib/src/syncSnapshotDebug.c b/source/libs/sync/test/sync_test_lib/src/syncSnapshotDebug.c index f1237e5282..d8740de16a 100644 --- a/source/libs/sync/test/sync_test_lib/src/syncSnapshotDebug.c +++ b/source/libs/sync/test/sync_test_lib/src/syncSnapshotDebug.c @@ -137,7 +137,7 @@ int32_t syncNodeOnPreSnapshot(SSyncNode *ths, SyncPreSnapshot *pMsg) { SyncPreSnapshotReply *pMsgReply = syncPreSnapshotReplyBuild(ths->vgId); pMsgReply->srcId = ths->myRaftId; pMsgReply->destId = pMsg->srcId; - pMsgReply->term = ths->pRaftStore->currentTerm; + pMsgReply->term = ths->raftStore.currentTerm; SSyncLogStoreData *pData = ths->pLogStore->data; SWal *pWal = pData->pWal; diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 734bac5c87..4cd6a39bfe 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -1065,11 +1065,11 @@ static int tdbBtreeEncodePayload(SPage *pPage, SCell *pCell, int nHeader, const } else { int nLeftKey = kLen; // pack partial key and nextPgno - memcpy(pCell + nHeader, pKey, nLocal - 4); - nLeft -= nLocal - 4; - nLeftKey -= nLocal - 4; + memcpy(pCell + nHeader, pKey, nLocal - nHeader - sizeof(pgno)); + nLeft -= nLocal - nHeader - sizeof(pgno); + nLeftKey -= nLocal - nHeader - sizeof(pgno); - memcpy(pCell + nHeader + nLocal - 4, &pgno, sizeof(pgno)); + memcpy(pCell + nLocal - sizeof(pgno), &pgno, sizeof(pgno)); int lastKeyPageSpace = 0; // pack left key & val to ovpages @@ -1089,9 +1089,12 @@ static int tdbBtreeEncodePayload(SPage *pPage, SCell *pCell, int nHeader, const if (lastKeyPage) { if (lastKeyPageSpace >= vLen) { - memcpy(pBuf + kLen - nLeftKey, pVal, vLen); + if (vLen > 0) { + memcpy(pBuf + kLen - nLeftKey, pVal, vLen); + + nLeft -= vLen; + } - nLeft -= vLen; pgno = 0; } else { memcpy(pBuf + kLen - nLeftKey, pVal, lastKeyPageSpace); @@ -1113,7 +1116,7 @@ static int tdbBtreeEncodePayload(SPage *pPage, SCell *pCell, int nHeader, const } } - memcpy(pBuf + kLen - nLeft, &pgno, sizeof(pgno)); + memcpy(pBuf + bytes, &pgno, sizeof(pgno)); ret = tdbPageInsertCell(ofp, 0, pBuf, bytes + sizeof(pgno), 0); if (ret < 0) { @@ -1327,11 +1330,11 @@ static int tdbBtreeDecodePayload(SPage *pPage, const SCell *pCell, int nHeader, } TDB_CELLDECODER_SET_FREE_KEY(pDecoder); - memcpy(pDecoder->pKey, pCell + nHeader, nLocal - 4); - nLeft -= nLocal - 4; - nLeftKey -= nLocal - 4; + memcpy(pDecoder->pKey, pCell + nHeader, nLocal - nHeader - sizeof(pgno)); + nLeft -= nLocal - nHeader - sizeof(pgno); + nLeftKey -= nLocal - nHeader - sizeof(pgno); - memcpy(&pgno, pCell + nHeader + nLocal - 4, sizeof(pgno)); + memcpy(&pgno, pCell + nLocal - sizeof(pgno), sizeof(pgno)); int lastKeyPageSpace = 0; // load left key & val to ovpages @@ -1357,9 +1360,11 @@ static int tdbBtreeDecodePayload(SPage *pPage, const SCell *pCell, int nHeader, if (lastKeyPage) { if (lastKeyPageSpace >= vLen) { - pDecoder->pVal = ofpCell + kLen - nLeftKey; + if (vLen > 0) { + pDecoder->pVal = ofpCell + kLen - nLeftKey; - nLeft -= vLen; + nLeft -= vLen; + } pgno = 0; } else { // read partial val to local diff --git a/source/libs/transport/src/tmsgcb.c b/source/libs/transport/src/tmsgcb.c index 4131619ed9..eea0d87684 100644 --- a/source/libs/transport/src/tmsgcb.c +++ b/source/libs/transport/src/tmsgcb.c @@ -59,6 +59,6 @@ void tmsgReleaseHandle(SRpcHandleInfo* pHandle, int8_t type) { (*defaultMsgCb.re void tmsgReportStartup(const char* name, const char* desc) { (*defaultMsgCb.reportStartupFp)(name, desc); } -int32_t tmsgUpdateDnodeInfo(int32_t* dnodeId, int64_t* clusterId, char* fqdn, uint16_t* port) { - return (*defaultMsgCb.updateDnodeInfoFp)(defaultMsgCb.data, dnodeId, clusterId, fqdn, port); +void tmsgUpdateDnodeInfo(int32_t* dnodeId, int64_t* clusterId, char* fqdn, uint16_t* port) { + (*defaultMsgCb.updateDnodeInfoFp)(defaultMsgCb.data, dnodeId, clusterId, fqdn, port); } diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index f49c9a53af..0562dc10ce 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -613,16 +613,13 @@ int32_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in } void walFsync(SWal *pWal, bool forceFsync) { + taosThreadMutexLock(&pWal->mutex); if (forceFsync || (pWal->cfg.level == TAOS_WAL_FSYNC && pWal->cfg.fsyncPeriod == 0)) { - wTrace("vgId:%d, fileId:%" PRId64 ".idx, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal)); - if (taosFsyncFile(pWal->pIdxFile) < 0) { - wError("vgId:%d, file:%" PRId64 ".idx, fsync failed since %s", pWal->cfg.vgId, walGetCurFileFirstVer(pWal), - strerror(errno)); - } wTrace("vgId:%d, fileId:%" PRId64 ".log, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal)); if (taosFsyncFile(pWal->pLogFile) < 0) { wError("vgId:%d, file:%" PRId64 ".log, fsync failed since %s", pWal->cfg.vgId, walGetCurFileFirstVer(pWal), strerror(errno)); } } + taosThreadMutexUnlock(&pWal->mutex); } diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index a9a84c1860..5581931178 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -227,6 +227,7 @@ STaosQueue *tAutoQWorkerAllocQueue(SAutoQWorkerPool *pool, void *ahandle, FItem uError("worker:%s:%d failed to create", pool->name, curWorkerNum); taosMemoryFree(worker); taosCloseQueue(queue); + taosThreadMutexUnlock(&pool->mutex); terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } diff --git a/tests/develop-test/2-query/table_count_scan.py b/tests/develop-test/2-query/table_count_scan.py new file mode 100644 index 0000000000..1ef65bfc67 --- /dev/null +++ b/tests/develop-test/2-query/table_count_scan.py @@ -0,0 +1,238 @@ +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import tdDnodes +from math import inf + +class TDTestCase: + def caseDescription(self): + ''' + case1: [TD-21890] table count scan test case + ''' + return + + def init(self, conn, logSql, replicaVer=1): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), False) + self._conn = conn + + def restartTaosd(self, index=1, dbname="db"): + tdDnodes.stop(index) + tdDnodes.startWithoutSleep(index) + tdSql.execute(f"use tbl_count") + + def run(self): + print("running {}".format(__file__)) + tdSql.execute("drop database if exists tbl_count") + tdSql.execute("create database if not exists tbl_count") + tdSql.execute('use tbl_count') + tdSql.execute('create table stb1 (ts timestamp, c1 bool, c2 tinyint, c3 smallint, c4 int, c5 bigint, c6 float, c7 double, c8 binary(10), c9 nchar(10), c10 tinyint unsigned, c11 smallint unsigned, c12 int unsigned, c13 bigint unsigned) TAGS(t1 int, t2 binary(10), t3 double);') + + tdSql.execute("create table tb1 using stb1 tags(1,'1',1.0);") + + tdSql.execute("create table tb2 using stb1 tags(2,'2',2.0);") + + tdSql.execute("create table tb3 using stb1 tags(3,'3',3.0);") + + tdSql.execute('insert into tb1 values (\'2021-11-11 09:00:00\',true,1,1,1,1,1,1,"123","1234",1,1,1,1);') + + tdSql.execute("insert into tb1 values ('2021-11-11 09:00:01',true,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);") + + tdSql.execute('insert into tb1 values (\'2021-11-11 09:00:02\',true,2,NULL,2,NULL,2,NULL,"234",NULL,2,NULL,2,NULL);') + + tdSql.execute('insert into tb1 values (\'2021-11-11 09:00:03\',false,NULL,3,NULL,3,NULL,3,NULL,"3456",NULL,3,NULL,3);') + + tdSql.execute('insert into tb1 values (\'2021-11-11 09:00:04\',true,4,4,4,4,4,4,"456","4567",4,4,4,4);') + + tdSql.execute('insert into tb1 values (\'2021-11-11 09:00:05\',true,127,32767,2147483647,9223372036854775807,3.402823466e+38,1.79769e+308,"567","5678",254,65534,4294967294,9223372036854775807);') + + tdSql.execute('insert into tb1 values (\'2021-11-11 09:00:06\',true,-127,-32767,-2147483647,-9223372036854775807,-3.402823466e+38,-1.79769e+308,"678","6789",0,0,0,0);') + + tdSql.execute('insert into tb2 values (\'2021-11-11 09:00:00\',true,1,1,1,1,1,1,"111","1111",1,1,1,1);') + + tdSql.execute('insert into tb2 values (\'2021-11-11 09:00:01\',true,2,2,2,2,2,2,"222","2222",2,2,2,2);') + + tdSql.execute('insert into tb2 values (\'2021-11-11 09:00:02\',true,3,3,2,3,3,3,"333","3333",3,3,3,3);') + + tdSql.execute('insert into tb2 values (\'2021-11-11 09:00:03\',false,4,4,4,4,4,4,"444","4444",4,4,4,4);') + + tdSql.execute('insert into tb2 values (\'2021-11-11 09:00:04\',true,5,5,5,5,5,5,"555","5555",5,5,5,5);') + + tdSql.execute('insert into tb2 values (\'2021-11-11 09:00:05\',true,6,6,6,6,6,6,"666","6666",6,6,6,6);') + + tdSql.execute('insert into tb2 values (\'2021-11-11 09:00:06\',true,7,7,7,7,7,7,"777","7777",7,7,7,7);') + + tdSql.query('select count(*),db_name, stable_name from information_schema.ins_tables group by db_name, stable_name;') + tdSql.checkRows(3) + tdSql.checkData(0, 0, 23) + tdSql.checkData(0, 1, 'information_schema') + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 3) + tdSql.checkData(1, 1, 'tbl_count') + tdSql.checkData(1, 2, 'stb1') + tdSql.checkData(2, 0, 5) + tdSql.checkData(2, 1, 'performance_schema') + tdSql.checkData(2, 2, None) + + tdSql.query('select count(1),db_name, stable_name from information_schema.ins_tables group by db_name, stable_name;') + tdSql.checkRows(3) + tdSql.checkData(0, 0, 23) + tdSql.checkData(0, 1, 'information_schema') + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 5) + tdSql.checkData(1, 1, 'performance_schema') + tdSql.checkData(1, 2, None) + tdSql.checkData(2, 0, 3) + tdSql.checkData(2, 1, 'tbl_count') + tdSql.checkData(2, 2, 'stb1') + + tdSql.query('select count(1),db_name from information_schema.ins_tables group by db_name') + tdSql.checkRows(3) + tdSql.checkData(0, 0, 5) + tdSql.checkData(0, 1, 'performance_schema') + tdSql.checkData(1, 0, 3) + tdSql.checkData(1, 1, 'tbl_count') + tdSql.checkData(2, 0, 23) + tdSql.checkData(2, 1, 'information_schema') + + tdSql.query("select count(*) from information_schema.ins_tables where db_name='tbl_count'") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 3) + + tdSql.query('select count(*) from information_schema.ins_tables where db_name=\'tbl_count\' and stable_name="stb1";') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 3) + + tdSql.query('select count(*) from information_schema.ins_tables') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 31) + + + tdSql.execute('create table stba (ts timestamp, c1 bool, c2 tinyint, c3 smallint, c4 int, c5 bigint, c6 float, c7 double, c8 binary(10), c9 nchar(10), c10 tinyint unsigned, c11 smallint unsigned, c12 int unsigned, c13 bigint unsigned) TAGS(t1 int, t2 binary(10), t3 double);') + + tdSql.execute("create table tba1 using stba tags(1,'1',1.0);") + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:00\',true, 1,1,1,1,1,1,"111","1111",1,1,1,1);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:01\',true, 2,2,2,2,2,2,"222","2222",2,2,2,2);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:02\',true, 3,3,2,3,3,3,"333","3333",3,3,3,3);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:03\',false,4,4,4,4,4,4,"444","4444",4,4,4,4);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:04\',true, 5,5,5,5,5,5,"555","5555",5,5,5,5);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:05\',true, 6,6,6,6,6,6,"666","6666",6,6,6,6);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:06\',true, 7,7,7,7,7,7,"777","7777",7,7,7,7);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:07\',true, 8,8,8,8,8,8,"888","8888",8,8,8,8);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:08\',true, 9,9,9,9,9,9,"999","9999",9,9,9,9);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:09\',true, 0,0,0,0,0,0,"000","0000",0,0,0,0);') + + self.restartTaosd(1, dbname='tbl_count') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:10\',true, 1,1,1,1,1,1,"111","1111",1,1,1,1);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:11\',true, 2,2,2,2,2,2,"222","2222",2,2,2,2);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:12\',true, 3,3,2,3,3,3,"333","3333",3,3,3,3);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:13\',false,4,4,4,4,4,4,"444","4444",4,4,4,4);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:14\',true, 5,5,5,5,5,5,"555","5555",5,5,5,5);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:15\',true, 6,6,6,6,6,6,"666","6666",6,6,6,6);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:16\',true, 7,7,7,7,7,7,"777","7777",7,7,7,7);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:17\',true, 8,8,8,8,8,8,"888","8888",8,8,8,8);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:18\',true, 9,9,9,9,9,9,"999","9999",9,9,9,9);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:19\',true, 0,0,0,0,0,0,"000","0000",0,0,0,0);') + + self.restartTaosd(1, dbname='tbl_count') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:20\',true, 1,1,1,1,1,1,"111","1111",1,1,1,1);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:21\',true, 2,2,2,2,2,2,"222","2222",2,2,2,2);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:22\',true, 3,3,2,3,3,3,"333","3333",3,3,3,3);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:23\',false,4,4,4,4,4,4,"444","4444",4,4,4,4);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:24\',true, 5,5,5,5,5,5,"555","5555",5,5,5,5);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:25\',true, 6,6,6,6,6,6,"666","6666",6,6,6,6);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:26\',true, 7,7,7,7,7,7,"777","7777",7,7,7,7);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:27\',true, 8,8,8,8,8,8,"888","8888",8,8,8,8);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:28\',true, 9,9,9,9,9,9,"999","9999",9,9,9,9);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:29\',true, 0,0,0,0,0,0,"000","0000",0,0,0,0);') + + tdSql.query('select count(*),db_name, stable_name from information_schema.ins_tables group by db_name, stable_name;') + tdSql.checkRows(4) + tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 1, 'tbl_count') + tdSql.checkData(0, 2, 'stba') + tdSql.checkData(1, 0, 23) + tdSql.checkData(1, 1, 'information_schema') + tdSql.checkData(1, 2, None) + tdSql.checkData(2, 0, 3) + tdSql.checkData(2, 1, 'tbl_count') + tdSql.checkData(2, 2, 'stb1') + tdSql.checkData(3, 0, 5) + tdSql.checkData(3, 1, 'performance_schema') + tdSql.checkData(3, 2, None) + + tdSql.query('select count(1),db_name, stable_name from information_schema.ins_tables group by db_name, stable_name;') + tdSql.checkRows(4) + tdSql.checkData(0, 0, 23) + tdSql.checkData(0, 1, 'information_schema') + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 5) + tdSql.checkData(1, 1, 'performance_schema') + tdSql.checkData(1, 2, None) + tdSql.checkData(2, 0, 1) + tdSql.checkData(2, 1, 'tbl_count') + tdSql.checkData(2, 2, 'stba') + tdSql.checkData(3, 0, 3) + tdSql.checkData(3, 1, 'tbl_count') + tdSql.checkData(3, 2, 'stb1') + + tdSql.query('select count(1),db_name from information_schema.ins_tables group by db_name') + tdSql.checkRows(3) + tdSql.checkData(0, 0, 5) + tdSql.checkData(0, 1, 'performance_schema') + tdSql.checkData(1, 0, 4) + tdSql.checkData(1, 1, 'tbl_count') + tdSql.checkData(2, 0, 23) + tdSql.checkData(2, 1, 'information_schema') + + tdSql.query("select count(*) from information_schema.ins_tables where db_name='tbl_count'") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 4) + + tdSql.query('select count(*) from information_schema.ins_tables where db_name=\'tbl_count\' and stable_name="stb1";') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 3) + + tdSql.query('select count(*) from information_schema.ins_tables') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 32) + + + tdSql.execute('drop database tbl_count') + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index c2810dbffc..00bad876a1 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -448,6 +448,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/database_pre_suf.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/InsertFuturets.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/information_schema.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/and_or_for_byte.py @@ -675,7 +676,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 -n 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 +#,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRecreateMnode.py -N 5 -M 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeStopFollowerLeader.py -N 5 -M 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3 @@ -1046,11 +1047,17 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_null_none.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_null_none.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_null_none.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/out_of_order.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/out_of_order.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/out_of_order.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/out_of_order.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/out_of_order.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/blockSMA.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-21561.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-20582.py #develop test +,,n,develop-test,python3 ./test.py -f 2-query/table_count_scan.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/auto_create_table_json.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/custom_col_tag.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/default_json.py diff --git a/tests/script/sh/checkAsan.sh b/tests/script/sh/checkAsan.sh index 7df17b22da..7225722791 100755 --- a/tests/script/sh/checkAsan.sh +++ b/tests/script/sh/checkAsan.sh @@ -39,7 +39,7 @@ python_error=`cat ${LOG_DIR}/*.info | grep -w "stack" | wc -l` # /root/TDengine/source/libs/scalar/src/sclvector.c:1075:66: runtime error: signed integer overflow: 9223372034707292160 + 1668838476672 cannot be represented in type 'long int' # /root/TDengine/source/common/src/tdataformat.c:1876:7: runtime error: signed integer overflow: 8252423483843671206 + 2406154664059062870 cannot be represented in type 'long int' -runtime_error=`cat ${LOG_DIR}/*.asan | grep "runtime error" | grep -v "trees.c:873" | grep -v "sclfunc.c.*outside the range of representable values of type"| grep -v "signed integer overflow" | wc -l` +runtime_error=`cat ${LOG_DIR}/*.asan | grep "runtime error" | grep -v "trees.c:873" | grep -v "sclfunc.c.*outside the range of representable values of type"| grep -v "signed integer overflow" |grep -v "strerror.c"| grep -v "asan_malloc_linux.cc" |wc -l` echo -e "\033[44;32;1m"asan error_num: $error_num"\033[0m" echo -e "\033[44;32;1m"asan memory_leak: $memory_leak"\033[0m" diff --git a/tests/script/tsim/parser/alter1.sim b/tests/script/tsim/parser/alter1.sim index 369419dcd9..cf9da46fba 100644 --- a/tests/script/tsim/parser/alter1.sim +++ b/tests/script/tsim/parser/alter1.sim @@ -88,6 +88,7 @@ sql insert into car1 values (now, 1, 1,1 ) (now +1s, 2,2,2) car2 values (now, 1, sql select c1+speed from stb where c1 > 0 if $rows != 3 then + print $rows , expect 3 return -1 endi diff --git a/tests/script/tsim/parser/regressiontest.sim b/tests/script/tsim/parser/regressiontest.sim index 98cb0248a1..1b127155cb 100644 --- a/tests/script/tsim/parser/regressiontest.sim +++ b/tests/script/tsim/parser/regressiontest.sim @@ -58,4 +58,9 @@ if $data40 != @18-09-17 09:06:49.600@ then return -1 endi +sql select * from $tb order by ts desc; +if $rows != 8198 then + return -1 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/system-test/0-others/information_schema.py b/tests/system-test/0-others/information_schema.py new file mode 100644 index 0000000000..1b82fa6e64 --- /dev/null +++ b/tests/system-test/0-others/information_schema.py @@ -0,0 +1,113 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + + +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.setsql = TDSetSql() + self.dbname = 'db' + self.stbname = 'stb' + self.binary_length = 20 # the length of binary for column_dict + self.nchar_length = 20 # the length of nchar for column_dict + self.ts = 1537146000000 + self.column_dict = { + 'ts' : 'timestamp', + 'col1': 'tinyint', + 'col2': 'smallint', + 'col3': 'int', + 'col4': 'bigint', + 'col5': 'tinyint unsigned', + 'col6': 'smallint unsigned', + 'col7': 'int unsigned', + 'col8': 'bigint unsigned', + 'col9': 'float', + 'col10': 'double', + 'col11': 'bool', + 'col12': f'binary({self.binary_length})', + 'col13': f'nchar({self.nchar_length})' + } + self.tbnum = 20 + self.rowNum = 10 + self.tag_dict = { + 't0':'int' + } + self.tag_values = [ + f'1' + ] + self.binary_str = 'taosdata' + self.nchar_str = '涛思数据' + self.ins_list = ['ins_dnodes','ins_mnodes','ins_modules','ins_qnodes','ins_snodes','ins_cluster','ins_databases','ins_functions',\ + 'ins_indexes','ins_stables','ins_tables','ins_tags','ins_users','ins_grants','ins_vgroups','ins_configs','ins_dnode_variables',\ + 'ins_topics','ins_subscriptions','ins_streams','ins_stream_tasks','ins_vnodes','ins_user_privileges'] + self.perf_list = ['perf_connections','perf_queries','perf_consumers','perf_trans','perf_apps'] + def insert_data(self,column_dict,tbname,row_num): + insert_sql = self.setsql.set_insertsql(column_dict,tbname,self.binary_str,self.nchar_str) + for i in range(row_num): + insert_list = [] + self.setsql.insert_values(column_dict,i,insert_sql,insert_list,self.ts) + def prepare_data(self): + tdSql.execute(f"create database if not exists {self.dbname} vgroups 2") + tdSql.execute(f'use {self.dbname}') + tdSql.execute(self.setsql.set_create_stable_sql(self.stbname,self.column_dict,self.tag_dict)) + for i in range(self.tbnum): + tdSql.execute(f"create table {self.stbname}_{i} using {self.stbname} tags({self.tag_values[0]})") + self.insert_data(self.column_dict,f'{self.stbname}_{i}',self.rowNum) + def count_check(self): + tdSql.query('select count(*) from information_schema.ins_tables') + tdSql.checkEqual(tdSql.queryResult[0][0],self.tbnum+len(self.ins_list)+len(self.perf_list)) + tdSql.query(f'select count(*) from information_schema.ins_tables where db_name = "{self.dbname}"') + tdSql.checkEqual(tdSql.queryResult[0][0],self.tbnum) + tdSql.query(f'select count(*) from information_schema.ins_tables where db_name = "{self.dbname}" and stable_name = "{self.stbname}"') + tdSql.checkEqual(tdSql.queryResult[0][0],self.tbnum) + tdSql.execute('create database db1') + tdSql.execute('create table stb1 (ts timestamp,c0 int) tags(t0 int)') + tdSql.execute('create table tb1 using stb1 tags(1)') + tdSql.query(f'select db_name, stable_name, count(*) from information_schema.ins_tables group by db_name, stable_name') + for i in tdSql.queryResult: + if i[0].lower() == 'information_schema': + tdSql.checkEqual(i[2],len(self.ins_list)) + elif i[0].lower() == self.dbname and i[1] == self.stbname: + tdSql.checkEqual(i[2],self.tbnum) + elif i[0].lower() == self.dbname and i[1] == 'stb1': + tdSql.checkEqual(i[2],1) + elif i[0].lower() == 'performance_schema': + tdSql.checkEqual(i[2],len(self.perf_list)) + tdSql.execute('create table db1.ntb (ts timestamp,c0 int)') + tdSql.query(f'select db_name, count(*) from information_schema.ins_tables group by db_name') + print(tdSql.queryResult) + for i in tdSql.queryResult: + if i[0].lower() == 'information_schema': + tdSql.checkEqual(i[1],len(self.ins_list)) + elif i[0].lower() == 'performance_schema': + tdSql.checkEqual(i[1],len(self.perf_list)) + elif i[0].lower() == self.dbname: + tdSql.checkEqual(i[1],self.tbnum+1) + def run(self): + self.prepare_data() + self.count_check() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/1-insert/time_range_wise.py b/tests/system-test/1-insert/time_range_wise.py index 3d5c9197d1..df1cc516c5 100644 --- a/tests/system-test/1-insert/time_range_wise.py +++ b/tests/system-test/1-insert/time_range_wise.py @@ -600,6 +600,11 @@ class TDTestCase: tdLog.printNoPrefix("==========step4:after wal, all check again ") self.all_test() + # add for TS-2440 + for i in range(self.rows): + tdSql.execute("drop database if exists db3 ") + tdSql.execute("create database db3 retentions 1s:4m,2s:8m,3s:12m") + def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") diff --git a/tests/system-test/2-query/insert_null_none.py b/tests/system-test/2-query/insert_null_none.py index cf5636fb1f..4304dee89e 100755 --- a/tests/system-test/2-query/insert_null_none.py +++ b/tests/system-test/2-query/insert_null_none.py @@ -24,7 +24,7 @@ from util.dnodes import tdDnodes from util.dnodes import * class TDTestCase: - updatecfgDict = {'maxSQLLength':1048576,'debugFlag': 143 ,"querySmaOptimize":1} + updatecfgDict = {'maxSQLLength':1048576,'debugFlag': 131 ,"querySmaOptimize":1} def init(self, conn, logSql, replicaVar): tdLog.debug("start to execute %s" % __file__) diff --git a/tests/system-test/2-query/nestedQuery.py b/tests/system-test/2-query/nestedQuery.py index 034ab8dcdc..6557aad05f 100755 --- a/tests/system-test/2-query/nestedQuery.py +++ b/tests/system-test/2-query/nestedQuery.py @@ -24,9 +24,9 @@ from util.dnodes import tdDnodes from util.dnodes import * class TDTestCase: - updatecfgDict = {'maxSQLLength':1048576,'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , - "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, - "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143} + updatecfgDict = {'maxSQLLength':1048576,'debugFlag': 131 ,"cDebugFlag":131,"uDebugFlag":131 ,"rpcDebugFlag":131 , "tmrDebugFlag":131 , + "jniDebugFlag":131 ,"simDebugFlag":131,"dDebugFlag":131, "dDebugFlag":131,"vDebugFlag":131,"mDebugFlag":131,"qDebugFlag":131, + "wDebugFlag":131,"sDebugFlag":131,"tsdbDebugFlag":131,"tqDebugFlag":131 ,"fsDebugFlag":131 ,"fnDebugFlag":131} def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) @@ -851,6 +851,7 @@ class TDTestCase: tdLog.info("========mark==%s==="% mark); try: tdSql.query(sql,queryTimes=1) + self.explain_sql(sql) except: tdLog.info("sql is not support :=====%s; " %sql) tdSql.error(sql) @@ -4995,9 +4996,7 @@ class TDTestCase: sql += "%s ;" % random.choice(self.limit_u_where) tdLog.info(sql) tdLog.info(len(sql)) - tdSql.query(sql) - self.cur1.execute(sql) - self.explain_sql(sql) + self.data_check(sql,mark='15-2') tdSql.query("select 15-2.2 from stable_1;") for i in range(self.fornum): @@ -5013,9 +5012,7 @@ class TDTestCase: sql += "%s ;" % random.choice(self.limit_u_where) tdLog.info(sql) tdLog.info(len(sql)) - tdSql.query(sql) - self.cur1.execute(sql) - self.explain_sql(sql) + self.data_check(sql,mark='15-2.2') self.restartDnodes() tdSql.query("select 15-3 from stable_1;") @@ -5033,9 +5030,7 @@ class TDTestCase: sql += "%s " % random.choice(self.limit_where) tdLog.info(sql) tdLog.info(len(sql)) - tdSql.query(sql) - self.cur1.execute(sql) - self.explain_sql(sql) + self.data_check(sql,mark='15-3') tdSql.query("select 15-4 from stable_1;") for i in range(self.fornum): @@ -5052,9 +5047,7 @@ class TDTestCase: sql += "%s " % random.choice(self.limit_u_where) tdLog.info(sql) tdLog.info(len(sql)) - tdSql.query(sql) - self.cur1.execute(sql) - self.explain_sql(sql) + self.data_check(sql,mark='15-4') tdSql.query("select 15-4.2 from stable_1;") for i in range(self.fornum): @@ -5087,8 +5080,7 @@ class TDTestCase: tdLog.info(sql) tdLog.info(len(sql)) tdSql.query(sql) - self.cur1.execute(sql) - self.explain_sql(sql) + self.data_check(sql,mark='15-5') #16 select * from (select calc_aggregate_regulars as agg from regular_table where <\>\in\and\or order by limit offset ) #self.dropandcreateDB_random("%s" %db, 1) diff --git a/tests/system-test/2-query/out_of_order.py b/tests/system-test/2-query/out_of_order.py new file mode 100644 index 0000000000..5b52661bae --- /dev/null +++ b/tests/system-test/2-query/out_of_order.py @@ -0,0 +1,191 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +import random + +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql, replicaVar): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + def run_benchmark(self,dbname,tables,per_table_num,order,replica): + #O :Out of order + #A :Repliaca + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + binPath = buildPath+ "/build/bin/" + + os.system("%staosBenchmark -d %s -t %d -n %d -O %d -a %d -b float,double,nchar\(200\),binary\(50\) -T 50 -y " % (binPath,dbname,tables,per_table_num,order,replica)) + + def sql_base(self,dbname): + self.check_sub(dbname) + sql1 = "select count(*) from %s.meters" %dbname + self.sql_base_check(sql1,sql1) + + self.check_sub(dbname) + sql2 = "select count(ts) from %s.meters" %dbname + self.sql_base_check(sql1,sql2) + + self.check_sub(dbname) + sql2 = "select count(_c0) from %s.meters" %dbname + self.sql_base_check(sql1,sql2) + + self.check_sub(dbname) + sql2 = "select count(c0) from %s.meters" %dbname + self.sql_base_check(sql1,sql2) + + self.check_sub(dbname) + sql2 = "select count(c1) from %s.meters" %dbname + self.sql_base_check(sql1,sql2) + + self.check_sub(dbname) + sql2 = "select count(c2) from %s.meters" %dbname + self.sql_base_check(sql1,sql2) + + self.check_sub(dbname) + sql2 = "select count(c3) from %s.meters" %dbname + self.sql_base_check(sql1,sql2) + + self.check_sub(dbname) + sql2 = "select count(t0) from %s.meters" %dbname + self.sql_base_check(sql1,sql2) + + self.check_sub(dbname) + sql2 = "select count(t1) from %s.meters" %dbname + self.sql_base_check(sql1,sql2) + + + self.check_sub(dbname) + sql2 = "select count(ts) from (select * from %s.meters)" %dbname + self.sql_base_check(sql1,sql2) + + self.check_sub(dbname) + sql2 = "select count(_c0) from (select * from %s.meters)" %dbname + self.sql_base_check(sql1,sql2) + + self.check_sub(dbname) + sql2 = "select count(c0) from (select * from %s.meters)" %dbname + self.sql_base_check(sql1,sql2) + + self.check_sub(dbname) + sql2 = "select count(c1) from (select * from %s.meters)" %dbname + self.sql_base_check(sql1,sql2) + + self.check_sub(dbname) + sql2 = "select count(c2) from (select * from %s.meters)" %dbname + self.sql_base_check(sql1,sql2) + + self.check_sub(dbname) + sql2 = "select count(c3) from (select * from %s.meters)" %dbname + self.sql_base_check(sql1,sql2) + + self.check_sub(dbname) + sql2 = "select count(t0) from (select * from %s.meters)" %dbname + self.sql_base_check(sql1,sql2) + + self.check_sub(dbname) + sql2 = "select count(t1) from (select * from %s.meters)" %dbname + self.sql_base_check(sql1,sql2) + + + def sql_base_check(self,sql1,sql2): + tdSql.query(sql1) + sql1_result = tdSql.getData(0,0) + tdLog.info("sql:%s , result: %s" %(sql1,sql1_result)) + + tdSql.query(sql2) + sql2_result = tdSql.getData(0,0) + tdLog.info("sql:%s , result: %s" %(sql2,sql2_result)) + + if sql1_result==sql2_result: + tdLog.info(f"checkEqual success, sql1_result={sql1_result},sql2_result={sql2_result}") + else : + tdLog.exit(f"checkEqual error, sql1_result=={sql1_result},sql2_result={sql2_result}") + + def run_sql(self,dbname): + self.sql_base(dbname) + + tdSql.execute(" flush database %s;" %dbname) + + self.sql_base(dbname) + + def check_sub(self,dbname): + + sql = "select count(*) from (select distinct(tbname) from %s.meters)" %dbname + tdSql.query(sql) + num = tdSql.getData(0,0) + + for i in range(0,num): + sql1 = "select count(*) from %s.d%d" %(dbname,i) + tdSql.query(sql1) + sql1_result = tdSql.getData(0,0) + tdLog.info("sql:%s , result: %s" %(sql1,sql1_result)) + + def check_out_of_order(self,dbname,tables,per_table_num,order,replica): + self.run_benchmark(dbname,tables,per_table_num,order,replica) + print("sleep 10 seconds") + #time.sleep(10) + print("sleep 10 seconds finish") + + self.run_sql(dbname) + + def run(self): + startTime = time.time() + + #self.check_out_of_order('db1',10,random.randint(10000,50000),random.randint(1,10),1) + self.check_out_of_order('db1',random.randint(50,200),random.randint(10000,20000),random.randint(1,5),1) + + # self.check_out_of_order('db2',random.randint(50,200),random.randint(10000,50000),random.randint(5,50),1) + + # self.check_out_of_order('db3',random.randint(50,200),random.randint(10000,50000),random.randint(50,100),1) + + # self.check_out_of_order('db4',random.randint(50,200),random.randint(10000,50000),100,1) + + endTime = time.time() + print("total time %ds" % (endTime - startTime)) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/stablity.py b/tests/system-test/2-query/stablity.py index ff026bf120..5e4d5dcbaf 100755 --- a/tests/system-test/2-query/stablity.py +++ b/tests/system-test/2-query/stablity.py @@ -24,9 +24,9 @@ from util.dnodes import tdDnodes from util.dnodes import * class TDTestCase: - updatecfgDict = {'maxSQLLength':1048576,'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , - "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, - "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143} + updatecfgDict = {'maxSQLLength':1048576,'debugFlag': 131 ,"cDebugFlag":131,"uDebugFlag":131 ,"rpcDebugFlag":131 , "tmrDebugFlag":131 , + "jniDebugFlag":131 ,"simDebugFlag":131,"dDebugFlag":131, "dDebugFlag":131,"vDebugFlag":131,"mDebugFlag":131,"qDebugFlag":131, + "wDebugFlag":131,"sDebugFlag":131,"tsdbDebugFlag":131,"tqDebugFlag":131 ,"fsDebugFlag":131 ,"fnDebugFlag":131} def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) diff --git a/tests/system-test/6-cluster/5dnode3mnodeDrop.py b/tests/system-test/6-cluster/5dnode3mnodeDrop.py index de9207ddd8..9dd3c56805 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeDrop.py +++ b/tests/system-test/6-cluster/5dnode3mnodeDrop.py @@ -112,7 +112,8 @@ class TDTestCase: dnode_first_port = dnode.cfgDict["firstEp"].split(":")[-1] cmd = f" taos -h {dnode_first_host} -P {dnode_first_port} -s ' create dnode \"{dnode_id} \" ' ;" tdLog.debug(cmd) - os.system(cmd) + if os.system(cmd) != 0: + raise Exception("failed to execute system command. cmd: %s" % cmd) time.sleep(2) tdLog.info(" create cluster with %d dnode done! " %dnodes_nums) @@ -120,7 +121,7 @@ class TDTestCase: def check3mnode(self): count=0 while count < 10: - time.sleep(1) + time.sleep(0.1) tdSql.query("select * from information_schema.ins_mnodes;") if tdSql.checkRows(3) : tdLog.debug("mnode is three nodes") @@ -157,7 +158,7 @@ class TDTestCase: def check3mnode1off(self): count=0 while count < 10: - time.sleep(1) + time.sleep(0.1) tdSql.query("select * from information_schema.ins_mnodes;") if tdSql.checkRows(3) : tdLog.debug("mnode is three nodes") @@ -189,7 +190,7 @@ class TDTestCase: def check3mnode2off(self): count=0 while count < 40: - time.sleep(1) + time.sleep(0.1) tdSql.query("select * from information_schema.ins_mnodes;") if tdSql.checkRows(3) : tdLog.debug("mnode is three nodes") @@ -219,7 +220,7 @@ class TDTestCase: def check3mnode3off(self): count=0 while count < 10: - time.sleep(1) + time.sleep(0.1) tdSql.query("select * from information_schema.ins_mnodes;") if tdSql.checkRows(3) : tdLog.debug("mnode is three nodes") @@ -279,32 +280,47 @@ class TDTestCase: # drop follower of mnode dropcount =0 - while dropcount <= 10: + while dropcount <= 5: for i in range(1,3): tdLog.debug("drop mnode on dnode %d"%(i+1)) tdSql.execute("drop mnode on dnode %d"%(i+1)) tdSql.query("select * from information_schema.ins_mnodes;") count=0 while count<10: - time.sleep(1) + time.sleep(0.1) tdSql.query("select * from information_schema.ins_mnodes;") - if tdSql.checkRows(2): + if tdSql.queryRows == 2: tdLog.debug("drop mnode %d successfully"%(i+1)) break count+=1 + self.wait_for_transactions(100) + tdLog.debug("create mnode on dnode %d"%(i+1)) tdSql.execute("create mnode on dnode %d"%(i+1)) count=0 while count<10: - time.sleep(1) + time.sleep(0.1) tdSql.query("select * from information_schema.ins_mnodes;") - if tdSql.checkRows(3): - tdLog.debug("drop mnode %d successfully"%(i+1)) + if tdSql.queryRows == 3: + tdLog.debug("create mnode %d successfully"%(i+1)) break count+=1 + self.wait_for_transactions(100) dropcount+=1 self.check3mnode() + def wait_for_transactions(self, timeout): + count=0 + while count= timeout: + tdLog.debug("transactions not finished before timeout (%d secs)"%timeout) def getConnection(self, dnode): host = dnode.cfgDict["fqdn"] diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb.py b/tests/system-test/7-tmq/tmqConsFromTsdb.py index 9bb8c4cc0d..8ed4a6df97 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb.py @@ -130,7 +130,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) tdLog.exit("%d tmq consume rows error!"%consumerId) - tmqCom.checkFileContent(consumerId, queryString) + # tmqCom.checkFileContent(consumerId, queryString) time.sleep(10) for i in range(len(topicNameList)): diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 01e3246ca4..33fbbba1c1 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -19,8 +19,8 @@ #include #include #include "taos.h" -#include "types.h" #include "tlog.h" +#include "types.h" int smlProcess_influx_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -78,22 +78,22 @@ int smlProcess_telnet_Test() { pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); -// char *sql[4] = {0}; -// sql[0] = taosMemoryCalloc(1, 128); -// sql[1] = taosMemoryCalloc(1, 128); -// sql[2] = taosMemoryCalloc(1, 128); -// sql[3] = taosMemoryCalloc(1, 128); + // char *sql[4] = {0}; + // sql[0] = taosMemoryCalloc(1, 128); + // sql[1] = taosMemoryCalloc(1, 128); + // sql[2] = taosMemoryCalloc(1, 128); + // sql[3] = taosMemoryCalloc(1, 128); const char *sql1[] = {"sys.if.bytes.out 1479496100 1.3E0 host=web01 interface=eth0", - "sys.if.bytes.out 1479496101 1.3E1 interface=eth0 host=web01 ", - "sys.if.bytes.out 1479496102 1.3E3 network=tcp", - " sys.procs.running 1479496100 42 host=web01 "}; + "sys.if.bytes.out 1479496101 1.3E1 interface=eth0 host=web01 ", + "sys.if.bytes.out 1479496102 1.3E3 network=tcp", + " sys.procs.running 1479496100 42 host=web01 "}; -// for(int i = 0; i < 4; i++){ -// strncpy(sql[i], sql1[i], 128); -// } + // for(int i = 0; i < 4; i++){ + // strncpy(sql[i], sql1[i], 128); + // } -// pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_TELNET_PROTOCOL, -// TSDB_SML_TIMESTAMP_NANO_SECONDS); + // pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_TELNET_PROTOCOL, + // TSDB_SML_TIMESTAMP_NANO_SECONDS); pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); @@ -114,11 +114,12 @@ int smlProcess_json1_Test() { taos_free_result(pRes); const char *sql[] = { - "[{\"metric\":\"sys.cpu.nice\",\"timestamp\":0,\"value\":18,\"tags\":{\"host\":\"web01\",\"id\":\"t1\",\"dc\":\"lga\"}},{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344045,\"value\":9,\"tags\":{\"host\":\"web02\",\"dc\":\"lga\"}}]" - }; + "[{\"metric\":\"sys.cpu.nice\",\"timestamp\":0,\"value\":18,\"tags\":{\"host\":\"web01\",\"id\":\"t1\",\"dc\":" + "\"lga\"}},{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344045,\"value\":9,\"tags\":{\"host\":\"web02\",\"dc\":" + "\"lga\"}}]"}; char *sql1[1] = {0}; - for(int i = 0; i < 1; i++){ + for (int i = 0; i < 1; i++) { sql1[i] = taosMemoryCalloc(1, 1024); strncpy(sql1[i], sql[i], 1023); } @@ -126,23 +127,25 @@ int smlProcess_json1_Test() { pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); int code = taos_errno(pRes); - if(code != 0){ + if (code != 0) { printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); - }else{ + } else { printf("%s result:success\n", __FUNCTION__); } taos_free_result(pRes); - for(int i = 0; i < 1; i++){ + for (int i = 0; i < 1; i++) { taosMemoryFree(sql1[i]); } const char *sql2[] = { - "[{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344041,\"value\":13,\"tags\":{\"host\":\"web01\",\"dc\":\"lga\"}},{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344042,\"value\":9,\"tags\":{\"host\":\"web02\",\"dc\":\"lga\"}}]", + "[{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344041,\"value\":13,\"tags\":{\"host\":\"web01\",\"dc\":\"lga\"}" + "},{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344042,\"value\":9,\"tags\":{\"host\":\"web02\",\"dc\":\"lga\"}" + "}]", }; char *sql3[1] = {0}; - for(int i = 0; i < 1; i++){ + for (int i = 0; i < 1; i++) { sql3[i] = taosMemoryCalloc(1, 1024); strncpy(sql3[i], sql2[i], 1023); } @@ -150,14 +153,14 @@ int smlProcess_json1_Test() { pRes = taos_schemaless_insert(taos, (char **)sql3, sizeof(sql3) / sizeof(sql3[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); code = taos_errno(pRes); - if(code != 0){ + if (code != 0) { printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); - }else{ + } else { printf("%s result:success\n", __FUNCTION__); } taos_free_result(pRes); - for(int i = 0; i < 1; i++){ + for (int i = 0; i < 1; i++) { taosMemoryFree(sql3[i]); } @@ -176,10 +179,11 @@ int smlProcess_json2_Test() { taos_free_result(pRes); const char *sql[] = { - "{\"metric\":\"meter_current0\",\"timestamp\":{\"value\":1662344042,\"type\":\"s\"},\"value\":{\"value\":10.3,\"type\":\"i64\"},\"tags\":{\"groupid\":{\"value\":2,\"type\":\"bigint\"},\"location\":{\"value\":\"北京\",\"type\":\"binary\"},\"id\":\"d1001\"}}" - }; + "{\"metric\":\"meter_current0\",\"timestamp\":{\"value\":1662344042,\"type\":\"s\"},\"value\":{\"value\":10.3," + "\"type\":\"i64\"},\"tags\":{\"groupid\":{\"value\":2,\"type\":\"bigint\"},\"location\":{\"value\":\"北京\"," + "\"type\":\"binary\"},\"id\":\"d1001\"}}"}; char *sql1[1] = {0}; - for(int i = 0; i < 1; i++){ + for (int i = 0; i < 1; i++) { sql1[i] = taosMemoryCalloc(1, 1024); strncpy(sql1[i], sql[i], 1023); } @@ -187,15 +191,15 @@ int smlProcess_json2_Test() { pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); int code = taos_errno(pRes); - if(code != 0){ + if (code != 0) { printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); - }else{ + } else { printf("%s result:success\n", __FUNCTION__); } taos_free_result(pRes); taos_close(taos); - for(int i = 0; i < 1; i++){ + for (int i = 0; i < 1; i++) { taosMemoryFree(sql1[i]); } return code; @@ -211,10 +215,10 @@ int smlProcess_json3_Test() { taos_free_result(pRes); const char *sql[] = { - "[{\"metric\":\"sys.cpu.nice3\",\"timestamp\":0,\"value\":\"18\",\"tags\":{\"host\":\"web01\",\"id\":\"t1\",\"dc\":\"lga\"}}]" - }; + "[{\"metric\":\"sys.cpu.nice3\",\"timestamp\":0,\"value\":\"18\",\"tags\":{\"host\":\"web01\",\"id\":\"t1\"," + "\"dc\":\"lga\"}}]"}; char *sql1[1] = {0}; - for(int i = 0; i < 1; i++){ + for (int i = 0; i < 1; i++) { sql1[i] = taosMemoryCalloc(1, 1024); strncpy(sql1[i], sql[i], 1023); } @@ -222,15 +226,15 @@ int smlProcess_json3_Test() { pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); int code = taos_errno(pRes); - if(code != 0){ + if (code != 0) { printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); - }else{ + } else { printf("%s result:success\n", __FUNCTION__); } taos_free_result(pRes); taos_close(taos); - for(int i = 0; i < 1; i++){ + for (int i = 0; i < 1; i++) { taosMemoryFree(sql1[i]); } return code; @@ -298,7 +302,7 @@ int sml_16384_Test() { printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); taos_free_result(pRes); - if(code) return code; + if (code) return code; const char *sql1[] = { "qelhxo,id=pnnqhsa,t0=t,t1=127i8 c0=f,c1=127i8,c11=L\"ncharColValue\",c10=t 1626006833631000000", @@ -740,11 +744,26 @@ int sml_dup_time_Test() { taos_free_result(pRes); const char *sql[] = {//"test_ms,t0=t c0=f 1626006833641", - "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=f,c1=1i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"xcxvwjvf\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=T,c1=2i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"fixrzcuq\",c8=L\"ncharColValue\",c9=7u64 1626006834639000000", - "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=t,c1=3i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"iupzdqub\",c8=L\"ncharColValue\",c9=7u64 1626006835639000000", - "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=t,c1=4i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"yvvtzzof\",c8=L\"ncharColValue\",c9=7u64 1626006836639000000", - "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=t,c1=5i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"vbxpilkj\",c8=L\"ncharColValue\",c9=7u64 1626006837639000000"}; + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." + "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " + "c0=f,c1=1i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." + "123456789f64,c7=\"xcxvwjvf\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." + "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " + "c0=T,c1=2i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." + "123456789f64,c7=\"fixrzcuq\",c8=L\"ncharColValue\",c9=7u64 1626006834639000000", + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." + "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " + "c0=t,c1=3i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." + "123456789f64,c7=\"iupzdqub\",c8=L\"ncharColValue\",c9=7u64 1626006835639000000", + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." + "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " + "c0=t,c1=4i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." + "123456789f64,c7=\"yvvtzzof\",c8=L\"ncharColValue\",c9=7u64 1626006836639000000", + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." + "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " + "c0=t,c1=5i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." + "123456789f64,c7=\"vbxpilkj\",c8=L\"ncharColValue\",c9=7u64 1626006837639000000"}; pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); @@ -764,8 +783,10 @@ int sml_add_tag_col_Test() { taos_free_result(pRes); const char *sql[] = { - "macylr,t0=f,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"binaryColValue\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000" - }; + "macylr,t0=f,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " + "c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"binaryColValue\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000"}; pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); @@ -776,8 +797,10 @@ int sml_add_tag_col_Test() { if (code) return code; const char *sql1[] = { - "macylr,id=macylr_17875_1804,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\",t11=127i8,t10=L\"ncharTagValue\" c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c8=L\"ncharColValue\",c9=7u64,c11=L\"ncharColValue\",c10=f 1626006833639000000" - }; + "macylr,id=macylr_17875_1804,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=" + "22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\",t11=127i8,t10=L\"ncharTagValue\" " + "c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c8=" + "L\"ncharColValue\",c9=7u64,c11=L\"ncharColValue\",c10=f 1626006833639000000"}; pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_LINE_PROTOCOL, 0); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); @@ -814,26 +837,26 @@ int smlProcess_18784_Test() { ASSERT(fieldNum == 5); printf("fieldNum:%d\n", fieldNum); TAOS_ROW row = NULL; - int32_t rowIndex = 0; - while((row = taos_fetch_row(pRes)) != NULL) { - int64_t ts = *(int64_t*)row[0]; - int64_t used = *(int64_t*)row[1]; - int64_t total = *(int64_t*)row[2]; - int64_t freed = *(int64_t*)row[3]; - if(rowIndex == 0){ + int32_t rowIndex = 0; + while ((row = taos_fetch_row(pRes)) != NULL) { + int64_t ts = *(int64_t *)row[0]; + int64_t used = *(int64_t *)row[1]; + int64_t total = *(int64_t *)row[2]; + int64_t freed = *(int64_t *)row[3]; + if (rowIndex == 0) { ASSERT(ts == 1661943960000); ASSERT(used == 176059); ASSERT(total == 1081101176832); ASSERT(freed == 66932805); -// ASSERT_EQ(latitude, 24.5208); -// ASSERT_EQ(longitude, 28.09377); -// ASSERT_EQ(elevation, 428); -// ASSERT_EQ(velocity, 0); -// ASSERT_EQ(heading, 304); -// ASSERT_EQ(grade, 0); -// ASSERT_EQ(fuel_consumption, 25); - }else{ -// ASSERT(0); + // ASSERT_EQ(latitude, 24.5208); + // ASSERT_EQ(longitude, 28.09377); + // ASSERT_EQ(elevation, 428); + // ASSERT_EQ(velocity, 0); + // ASSERT_EQ(heading, 304); + // ASSERT_EQ(grade, 0); + // ASSERT_EQ(fuel_consumption, 25); + } else { + // ASSERT(0); } rowIndex++; } @@ -850,17 +873,21 @@ int sml_19221_Test() { taos_free_result(pRes); const char *sql[] = { - "qelhxo,id=pnnqhsa,t0=t,t1=127i8 c11=L\"ncharColValue\",c0=t,c1=127i8 1626006833632000000\nqelhxo,id=pnnhsa,t0=t,t1=127i8 c11=L\"ncharColValue\",c0=t,c1=127i8 1626006833633000000\n#comment\nqelhxo,id=pnqhsa,t0=t,t1=127i8 c11=L\"ncharColValue\",c0=t,c1=127i8 1626006833634000000", + "qelhxo,id=pnnqhsa,t0=t,t1=127i8 c11=L\"ncharColValue\",c0=t,c1=127i8 " + "1626006833632000000\nqelhxo,id=pnnhsa,t0=t,t1=127i8 c11=L\"ncharColValue\",c0=t,c1=127i8 " + "1626006833633000000\n#comment\nqelhxo,id=pnqhsa,t0=t,t1=127i8 c11=L\"ncharColValue\",c0=t,c1=127i8 " + "1626006833634000000", }; pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - char* tmp = (char*)taosMemoryCalloc(1024, 1); + char *tmp = (char *)taosMemoryCalloc(1024, 1); memcpy(tmp, sql[0], strlen(sql[0])); - *(char*)(tmp+44) = 0; + *(char *)(tmp + 44) = 0; int32_t totalRows = 0; - pRes = taos_schemaless_insert_raw(taos, tmp, strlen(sql[0]), &totalRows, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + pRes = taos_schemaless_insert_raw(taos, tmp, strlen(sql[0]), &totalRows, TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); ASSERT(totalRows == 3); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); @@ -875,20 +902,22 @@ int sml_19221_Test() { int sml_ts2164_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - TAOS_RES *pRes = taos_query(taos, "CREATE DATABASE IF NOT EXISTS line_test BUFFER 384 MINROWS 1000 PAGES 256 PRECISION 'ns'"); + TAOS_RES *pRes = + taos_query(taos, "CREATE DATABASE IF NOT EXISTS line_test BUFFER 384 MINROWS 1000 PAGES 256 PRECISION 'ns'"); taos_free_result(pRes); const char *sql[] = { -// "meters,location=la,groupid=ca current=11.8,voltage=221,phase=0.27", + // "meters,location=la,groupid=ca current=11.8,voltage=221,phase=0.27", "meters,location=la,groupid=ca current=11.8,voltage=221", "meters,location=la,groupid=ca current=11.8,voltage=221,phase=0.27", -// "meters,location=la,groupid=cb current=11.8,voltage=221,phase=0.27", + // "meters,location=la,groupid=cb current=11.8,voltage=221,phase=0.27", }; pRes = taos_query(taos, "use line_test"); taos_free_result(pRes); - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS); + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); @@ -905,30 +934,35 @@ int sml_ttl_Test() { taos_free_result(pRes); const char *sql[] = { - "meters,location=California.LosAngeles,groupid=2 current=11.8,voltage=221,phase=\"2022-02-0210:22:22\" 1626006833739000000", + "meters,location=California.LosAngeles,groupid=2 current=11.8,voltage=221,phase=\"2022-02-0210:22:22\" " + "1626006833739000000", }; const char *sql1[] = { - "meters,location=California.LosAngeles,groupid=2 current=11.8,voltage=221,phase=\"2022-02-0210:22:22\" 1626006833339000000", + "meters,location=California.LosAngeles,groupid=2 current=11.8,voltage=221,phase=\"2022-02-0210:22:22\" " + "1626006833339000000", }; pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - pRes = taos_schemaless_insert_ttl(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, 20); + pRes = taos_schemaless_insert_ttl(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS, 20); printf("%s result1:%s\n", __FUNCTION__, taos_errstr(pRes)); taos_free_result(pRes); - pRes = taos_schemaless_insert_ttl(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, 20); + pRes = taos_schemaless_insert_ttl(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS, 20); printf("%s result1:%s\n", __FUNCTION__, taos_errstr(pRes)); taos_free_result(pRes); - pRes = taos_query(taos, "select `ttl` from information_schema.ins_tables where table_name='t_be97833a0e1f523fcdaeb6291d6fdf27'"); + pRes = taos_query( + taos, "select `ttl` from information_schema.ins_tables where table_name='t_be97833a0e1f523fcdaeb6291d6fdf27'"); printf("%s result2:%s\n", __FUNCTION__, taos_errstr(pRes)); TAOS_ROW row = taos_fetch_row(pRes); - if(row != NULL && *row != NULL){ - int32_t ttl = *(int32_t*)row[0]; + if (row != NULL && *row != NULL) { + int32_t ttl = *(int32_t *)row[0]; ASSERT(ttl == 20); } @@ -939,40 +973,40 @@ int sml_ttl_Test() { return code; } -//char *str[] ={ -// "", -// "f64", -// "F64", -// "f32", -// "F32", -// "i", -// "I", -// "i64", -// "I64", -// "u", -// "U", -// "u64", -// "U64", -// "i32", -// "I32", -// "u32", -// "U32", -// "i16", -// "I16", -// "u16", -// "U16", -// "i8", -// "I8", -// "u8", -// "U8", -//}; -//uint8_t smlCalTypeSum(char* endptr, int32_t left){ -// uint8_t sum = 0; -// for(int i = 0; i < left; i++){ -// sum += endptr[i]; -// } -// return sum; -//} +// char *str[] ={ +// "", +// "f64", +// "F64", +// "f32", +// "F32", +// "i", +// "I", +// "i64", +// "I64", +// "u", +// "U", +// "u64", +// "U64", +// "i32", +// "I32", +// "u32", +// "U32", +// "i16", +// "I16", +// "u16", +// "U16", +// "i8", +// "I8", +// "u8", +// "U8", +// }; +// uint8_t smlCalTypeSum(char* endptr, int32_t left){ +// uint8_t sum = 0; +// for(int i = 0; i < left; i++){ +// sum += endptr[i]; +// } +// return sum; +// } int sml_ts2385_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -980,17 +1014,50 @@ int sml_ts2385_Test() { TAOS_RES *pRes = taos_query(taos, "CREATE DATABASE IF NOT EXISTS ts2385"); taos_free_result(pRes); - const char *sql[] ={ - "DataRTU,deviceId=2211230C94K0_1,dataModelName=DataRTU_2211230C94K0_1 s5=false,s18=false,k14=0,k2=0,k8=0,k10=0,s9=false,s19=false,k11=0,k13=0,s22=false,k15=0,m2=37.416671660000006,m8=600,m10=1532,m1=20.25,m13=0,s7=false,k7=0,m16=0,s17=false,k4=0,s11=false,s15=true,m7=600,m12=1490,s1=true,m14=0,s14=false,s16=true,k5=0,hex=\"7b3b00000001030301030200000000323231313233304339344b30002b01012a10028003000000070d05da025802580258025802580258045305fc05f505d200000000000000000afc7d\",k6=0,m3=600,s3=false,s24=false,k3=0,m6=600,m15=0,s12=false,k1=0,k16=0,s10=false,s21=false,k12=0,m5=600,s8=false,m4=600,m9=1107,s2=false,s13=false,s20=false,s23=false,k9=0,m11=1525,s4=false,s6=false 1672818929178749400", - "DataRTU,deviceId=2211230C94K0_1,dataModelName=DataRTU_2211230C94K0_1 k2=0,k11=0,m3=600,m12=1506,s17=false,m5=600,s11=false,s22=false,k6=0,m13=0,s16=true,k5=0,s21=false,m4=600,m7=600,s9=false,s10=false,s18=false,k7=0,m8=600,k1=0,hex=\"7b3a00000001030301030200000000323231313233304339344b30002b01012a10028003000000071105e8025802580258025802580258044905eb05ef05e200000000000000000afc7d\",m11=1519,m16=0,s19=false,s23=false,s24=false,s14=false,s6=false,k10=0,k15=0,k14=0,s2=false,s4=false,s8=false,s13=false,s15=true,s20=false,m2=38.000005040000005,s3=false,s7=false,k3=0,k8=0,k13=0,m6=600,m14=0,m15=0,k4=0,m1=20.450000000000003,m9=1097,s1=true,m10=1515,s5=false,s12=false,k9=0,k12=0,k16=0 1672818919126971000", - "DataRTU,deviceId=2211230C94K0_1,dataModelName=DataRTU_2211230C94K0_1 k7=0,k14=0,m3=600,m7=600,s5=false,k2=0,k3=0,k8=0,s3=false,s20=false,k15=0,m10=1482,s17=false,k1=0,k16=0,m15=0,s12=false,k9=0,m16=0,s11=false,m4=600,s10=false,s15=true,s24=false,m8=600,m13=0,s2=false,s18=false,k12=0,s14=false,s19=false,hex=\"7b3900000001030301030200000000323231313233304339344b30002b01012a10028003000000071505ef025802580258025802580258045005ca05b105d800000000000000000aa47d\",s1=true,s4=false,s7=false,s8=false,s13=false,m6=600,s6=false,s21=false,k11=0,m12=1496,m9=1104,s16=true,k5=0,s9=false,k10=0,k13=0,m2=38.291671730000004,s22=false,m5=600,m11=1457,m14=0,k4=0,m1=20.650000000000006,s23=false,k6=0 1672818909130866800", - "DataRTU,deviceId=2211230C94K0_1,dataModelName=DataRTU_2211230C94K0_1 m7=600,k4=0,k14=0,s22=false,k13=0,s2=false,m11=1510,m14=0,s4=false,s10=false,m1=21,m16=0,m13=0,s9=false,s13=false,s14=false,k10=0,m3=600,m9=1107,s18=false,s19=false,k2=0,hex=\"7b3600000001030301030200000000323231313233304339344b30002b01012a10028003000000071c0619025802580258025802580258045305dc05e6058d00000000000000000ad27d\",m2=40.04167187,m8=600,k7=0,k8=0,m10=1500,s23=false,k5=0,s11=false,s21=false,k9=0,m15=0,m12=1421,s1=true,s5=false,s8=false,m5=600,k16=0,k15=0,m6=600,s3=false,s6=false,s7=false,s15=true,s20=false,s24=false,k11=0,k1=0,k6=0,k12=0,m4=600,s16=true,s17=false,k3=0,s12=false 1672818879189483200", - "DataRTU,deviceId=2106070C11M0_2,dataModelName=DataRTU_2106070C11M0_2 m1=5691,k14=0,m6=0,s14=false,k8=0,s19=false,s20=false,k12=0,s17=false,k3=0,m8=0,s8=false,m7=0,s9=false,s4=false,s11=false,s13=false,s16=false,k5=0,k15=0,k16=0,s10=false,s23=false,s1=false,s2=false,s3=false,s12=false,s24=false,k2=0,k10=0,hex=\"7b1400000001030301030200000000323130363037304331314d30002b01022a080400000000000008af0c000000000000000000000000000000000000000000000000000000000ad47d\",m2=0,s7=false,s18=false,s21=false,m3=0,m5=0,k4=0,k11=0,m4=0,k1=0,k6=0,k13=0,s6=false,s15=false,s5=false,s22=false,k7=0,k9=0 1672818779549848800" - }; + const char *sql[] = { + "DataRTU,deviceId=2211230C94K0_1,dataModelName=DataRTU_2211230C94K0_1 " + "s5=false,s18=false,k14=0,k2=0,k8=0,k10=0,s9=false,s19=false,k11=0,k13=0,s22=false,k15=0,m2=37.416671660000006," + "m8=600,m10=1532,m1=20.25,m13=0,s7=false,k7=0,m16=0,s17=false,k4=0,s11=false,s15=true,m7=600,m12=1490,s1=true," + "m14=0,s14=false,s16=true,k5=0,hex=" + "\"7b3b00000001030301030200000000323231313233304339344b30002b01012a10028003000000070d05da025802580258025802580258" + "045305fc05f505d200000000000000000afc7d\",k6=0,m3=600,s3=false,s24=false,k3=0,m6=600,m15=0,s12=false,k1=0,k16=0," + "s10=false,s21=false,k12=0,m5=600,s8=false,m4=600,m9=1107,s2=false,s13=false,s20=false,s23=false,k9=0,m11=1525," + "s4=false,s6=false 1672818929178749400", + "DataRTU,deviceId=2211230C94K0_1,dataModelName=DataRTU_2211230C94K0_1 " + "k2=0,k11=0,m3=600,m12=1506,s17=false,m5=600,s11=false,s22=false,k6=0,m13=0,s16=true,k5=0,s21=false,m4=600,m7=" + "600,s9=false,s10=false,s18=false,k7=0,m8=600,k1=0,hex=" + "\"7b3a00000001030301030200000000323231313233304339344b30002b01012a10028003000000071105e8025802580258025802580258" + "044905eb05ef05e200000000000000000afc7d\",m11=1519,m16=0,s19=false,s23=false,s24=false,s14=false,s6=false,k10=0," + "k15=0,k14=0,s2=false,s4=false,s8=false,s13=false,s15=true,s20=false,m2=38.000005040000005,s3=false,s7=false,k3=" + "0,k8=0,k13=0,m6=600,m14=0,m15=0,k4=0,m1=20.450000000000003,m9=1097,s1=true,m10=1515,s5=false,s12=false,k9=0,k12=" + "0,k16=0 1672818919126971000", + "DataRTU,deviceId=2211230C94K0_1,dataModelName=DataRTU_2211230C94K0_1 " + "k7=0,k14=0,m3=600,m7=600,s5=false,k2=0,k3=0,k8=0,s3=false,s20=false,k15=0,m10=1482,s17=false,k1=0,k16=0,m15=0," + "s12=false,k9=0,m16=0,s11=false,m4=600,s10=false,s15=true,s24=false,m8=600,m13=0,s2=false,s18=false,k12=0,s14=" + "false,s19=false,hex=" + "\"7b3900000001030301030200000000323231313233304339344b30002b01012a10028003000000071505ef025802580258025802580258" + "045005ca05b105d800000000000000000aa47d\",s1=true,s4=false,s7=false,s8=false,s13=false,m6=600,s6=false,s21=false," + "k11=0,m12=1496,m9=1104,s16=true,k5=0,s9=false,k10=0,k13=0,m2=38.291671730000004,s22=false,m5=600,m11=1457,m14=0," + "k4=0,m1=20.650000000000006,s23=false,k6=0 1672818909130866800", + "DataRTU,deviceId=2211230C94K0_1,dataModelName=DataRTU_2211230C94K0_1 " + "m7=600,k4=0,k14=0,s22=false,k13=0,s2=false,m11=1510,m14=0,s4=false,s10=false,m1=21,m16=0,m13=0,s9=false,s13=" + "false,s14=false,k10=0,m3=600,m9=1107,s18=false,s19=false,k2=0,hex=" + "\"7b3600000001030301030200000000323231313233304339344b30002b01012a10028003000000071c0619025802580258025802580258" + "045305dc05e6058d00000000000000000ad27d\",m2=40.04167187,m8=600,k7=0,k8=0,m10=1500,s23=false,k5=0,s11=false,s21=" + "false,k9=0,m15=0,m12=1421,s1=true,s5=false,s8=false,m5=600,k16=0,k15=0,m6=600,s3=false,s6=false,s7=false,s15=" + "true,s20=false,s24=false,k11=0,k1=0,k6=0,k12=0,m4=600,s16=true,s17=false,k3=0,s12=false 1672818879189483200", + "DataRTU,deviceId=2106070C11M0_2,dataModelName=DataRTU_2106070C11M0_2 " + "m1=5691,k14=0,m6=0,s14=false,k8=0,s19=false,s20=false,k12=0,s17=false,k3=0,m8=0,s8=false,m7=0,s9=false,s4=false," + "s11=false,s13=false,s16=false,k5=0,k15=0,k16=0,s10=false,s23=false,s1=false,s2=false,s3=false,s12=false,s24=" + "false,k2=0,k10=0,hex=" + "\"7b1400000001030301030200000000323130363037304331314d30002b01022a080400000000000008af0c000000000000000000000000" + "000000000000000000000000000000000ad47d\",m2=0,s7=false,s18=false,s21=false,m3=0,m5=0,k4=0,k11=0,m4=0,k1=0,k6=0," + "k13=0,s6=false,s15=false,s5=false,s22=false,k7=0,k9=0 1672818779549848800"}; pRes = taos_query(taos, "use ts2385"); taos_free_result(pRes); - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); @@ -999,14 +1066,14 @@ int sml_ts2385_Test() { pRes = taos_query(taos, "select distinct tbname from `DataRTU` order by tbname"); printf("%s result2:%s\n", __FUNCTION__, taos_errstr(pRes)); - int num = 0; + int num = 0; TAOS_ROW row = NULL; - while((row = taos_fetch_row(pRes))){ - if(row[0] != NULL && num == 0){ + while ((row = taos_fetch_row(pRes))) { + if (row[0] != NULL && num == 0) { ASSERT(strncmp((char *)row[0], "DataRTU_2106070C11M0_2", sizeof("DataRTU_2106070C11M0_2") - 1) == 0); } - if(row[0] != NULL && num == 1){ + if (row[0] != NULL && num == 1) { ASSERT(strncmp((char *)row[0], "DataRTU_2211230C94K0_1", sizeof("DataRTU_2211230C94K0_1") - 1) == 0); } num++; @@ -1021,17 +1088,17 @@ int sml_ts2385_Test() { } int main(int argc, char *argv[]) { - if(argc == 2){ + if (argc == 2) { taos_options(TSDB_OPTION_CONFIGDIR, argv[1]); } int ret = 0; ret = sml_ts2385_Test(); ASSERT(!ret); -// for(int i = 0; i < sizeof(str)/sizeof(str[0]); i++){ -// printf("str:%s \t %d\n", str[i], smlCalTypeSum(str[i], strlen(str[i]))); -// } -// int ret = 0; + // for(int i = 0; i < sizeof(str)/sizeof(str[0]); i++){ + // printf("str:%s \t %d\n", str[i], smlCalTypeSum(str[i], strlen(str[i]))); + // } + // int ret = 0; ret = sml_ttl_Test(); ASSERT(!ret); ret = sml_ts2164_Test();