From 13d3d4c7fae393a9a12c5cf19af114f82c3a333e Mon Sep 17 00:00:00 2001 From: wpan Date: Fri, 16 Jul 2021 16:44:29 +0800 Subject: [PATCH 1/4] add set sub tbname --- src/client/src/tscPrepare.c | 60 ++++++++++++++++++++++++++++++++++++- src/inc/taos.h | 1 + 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 3a462c96d6..a695b49304 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -47,6 +47,7 @@ typedef struct SNormalStmt { typedef struct SMultiTbStmt { bool nameSet; bool tagSet; + bool subSet; uint64_t currentUid; char *sqlstr; uint32_t tbNum; @@ -1522,6 +1523,7 @@ int taos_stmt_prepare(TAOS_STMT* stmt, const char* sql, unsigned long length) { int taos_stmt_set_tbname_tags(TAOS_STMT* stmt, const char* name, TAOS_BIND* tags) { STscStmt* pStmt = (STscStmt*)stmt; + int32_t code = 0; if (stmt == NULL || pStmt->pSql == NULL || pStmt->taos == NULL) { STMT_RET(TSDB_CODE_TSC_DISCONNECTED); @@ -1566,6 +1568,52 @@ int taos_stmt_set_tbname_tags(TAOS_STMT* stmt, const char* name, TAOS_BIND* tags STMT_RET(TSDB_CODE_SUCCESS); } + if (pStmt->mtb.subSet && taosHashGetSize(pStmt->mtb.pTableHash) > 0) { + STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; + char sTableName[TSDB_TABLE_FNAME_LEN]; + strncpy(sTableName, pTableMeta->sTableName, sizeof(sTableName)); + + SStrToken tname = {0}; + tname.type = TK_STRING; + tname.z = (char *)name; + tname.n = strlen(name); + SName fullname = {0}; + tscSetTableFullName(&fullname, &tname, pSql); + code = tscGetTableMeta(pSql, pTableMetaInfo); + if (code != TSDB_CODE_SUCCESS) { + STMT_RET(code); + } + + pTableMeta = pTableMetaInfo->pTableMeta; + + if (strcmp(sTableName, pTableMeta->sTableName)) { + tscError("0x%"PRIx64" only tables belongs to one stable is allowed", pSql->self); + STMT_RET(TSDB_CODE_TSC_APP_ERROR); + } + + memcpy(&pTableMetaInfo->name, &fullname, sizeof(fullname)); + + STableDataBlocks* pBlock = NULL; + code = tscGetDataBlockFromList(pCmd->insertParam.pTableBlockHashList, pTableMeta->id.uid, TSDB_PAYLOAD_SIZE, sizeof(SSubmitBlk), + pTableMeta->tableInfo.rowSize, &pTableMetaInfo->name, pTableMeta, &pBlock, NULL); + if (code != TSDB_CODE_SUCCESS) { + STMT_RET(code); + } + + SSubmitBlk* blk = (SSubmitBlk*)pBlock->pData; + blk->numOfRows = 0; + + pStmt->mtb.currentUid = pTableMeta->id.uid; + pStmt->mtb.tbNum++; + + taosHashPut(pStmt->mtb.pTableBlockHashList, (void *)&pStmt->mtb.currentUid, sizeof(pStmt->mtb.currentUid), (void*)&pBlock, POINTER_BYTES); + taosHashPut(pStmt->mtb.pTableHash, name, strlen(name), (char*) &pTableMeta->id.uid, sizeof(pTableMeta->id.uid)); + + tscDebug("0x%"PRIx64" table:%s is prepared, uid:%" PRIx64, pSql->self, name, pStmt->mtb.currentUid); + STMT_RET(TSDB_CODE_SUCCESS); + } + if (pStmt->mtb.tagSet) { pStmt->mtb.tbname = tscReplaceStrToken(&pSql->sqlstr, &pStmt->mtb.tbname, name); } else { @@ -1594,7 +1642,7 @@ int taos_stmt_set_tbname_tags(TAOS_STMT* stmt, const char* name, TAOS_BIND* tags pCmd->insertParam.pTableBlockHashList = hashList; } - int32_t code = tsParseSql(pStmt->pSql, true); + code = tsParseSql(pStmt->pSql, true); if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { // wait for the callback function to post the semaphore tsem_wait(&pStmt->pSql->rspSem); @@ -1629,7 +1677,17 @@ int taos_stmt_set_tbname_tags(TAOS_STMT* stmt, const char* name, TAOS_BIND* tags } +int taos_stmt_set_sub_tbname(TAOS_STMT* stmt, const char* name) { + STscStmt* pStmt = (STscStmt*)stmt; + pStmt->mtb.subSet = true; + return taos_stmt_set_tbname_tags(stmt, name, NULL); +} + + + int taos_stmt_set_tbname(TAOS_STMT* stmt, const char* name) { + STscStmt* pStmt = (STscStmt*)stmt; + pStmt->mtb.subSet = false; return taos_stmt_set_tbname_tags(stmt, name, NULL); } diff --git a/src/inc/taos.h b/src/inc/taos.h index a62f387924..a76a590edb 100644 --- a/src/inc/taos.h +++ b/src/inc/taos.h @@ -114,6 +114,7 @@ TAOS_STMT *taos_stmt_init(TAOS *taos); int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length); int taos_stmt_set_tbname_tags(TAOS_STMT* stmt, const char* name, TAOS_BIND* tags); int taos_stmt_set_tbname(TAOS_STMT* stmt, const char* name); +int taos_stmt_set_sub_tbname(TAOS_STMT* stmt, const char* name); int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert); int taos_stmt_num_params(TAOS_STMT *stmt, int *nums); int taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes); From 950212fd35731df7f0bac13ab92ee1d5f4eb529c Mon Sep 17 00:00:00 2001 From: wpan Date: Mon, 19 Jul 2021 14:40:40 +0800 Subject: [PATCH 2/4] support bind for sub tables in the same stable --- src/client/inc/tscUtil.h | 1 + src/client/inc/tsclient.h | 3 +- src/client/src/tscParseInsert.c | 2 +- src/client/src/tscPrepare.c | 154 ++++++++++++++++++++++++++++---- src/client/src/tscUtil.c | 36 ++++---- src/common/src/tname.c | 2 +- 6 files changed, 160 insertions(+), 38 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index dbb31f4634..1fb00bc0d7 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -99,6 +99,7 @@ int32_t converToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *le int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOffset, SName* name, STableMeta* pTableMeta, STableDataBlocks** dataBlocks); void tscDestroyDataBlock(STableDataBlocks* pDataBlock, bool removeMeta); void tscSortRemoveDataBlockDupRows(STableDataBlocks* dataBuf); +int32_t tsSetBlockInfo(SSubmitBlk *pBlocks, const STableMeta *pTableMeta, int32_t numOfRows); void tscDestroyBoundColumnInfo(SParsedDataColInfo* pColInfo); void doRetrieveSubqueryData(SSchedMsg *pMsg); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index c0a1afda77..5f9da6e3ba 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -108,7 +108,8 @@ typedef struct STableDataBlocks { uint32_t size; STableMeta *pTableMeta; // the tableMeta of current table, the table meta will be used during submit, keep a ref to avoid to be removed from cache char *pData; - + bool cloned; + SParsedDataColInfo boundColumnInfo; // for parameter ('?') binding diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 26d9cf0e49..8d064ed583 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -643,7 +643,7 @@ int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int3 return TSDB_CODE_SUCCESS; } -static int32_t tsSetBlockInfo(SSubmitBlk *pBlocks, const STableMeta *pTableMeta, int32_t numOfRows) { +int32_t FORCE_INLINE tsSetBlockInfo(SSubmitBlk *pBlocks, const STableMeta *pTableMeta, int32_t numOfRows) { pBlocks->tid = pTableMeta->id.tid; pBlocks->uid = pTableMeta->id.uid; pBlocks->sversion = pTableMeta->sversion; diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index a695b49304..8305604382 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -55,6 +55,7 @@ typedef struct SMultiTbStmt { SStrToken stbname; SStrToken values; SArray *tags; + STableDataBlocks *lastBlock; SHashObj *pTableHash; SHashObj *pTableBlockHashList; // data block for each table } SMultiTbStmt; @@ -348,7 +349,7 @@ int32_t fillTablesColumnsNull(SSqlObj* pSql) { //////////////////////////////////////////////////////////////////////////////// // functions for insertion statement preparation -static int doBindParam(STableDataBlocks* pBlock, char* data, SParamInfo* param, TAOS_BIND* bind, int32_t colNum) { +static FORCE_INLINE int doBindParam(STableDataBlocks* pBlock, char* data, SParamInfo* param, TAOS_BIND* bind, int32_t colNum) { if (bind->is_null != NULL && *(bind->is_null)) { setNull(data + param->offset, param->type, param->bytes); return TSDB_CODE_SUCCESS; @@ -747,25 +748,25 @@ static int doBindParam(STableDataBlocks* pBlock, char* data, SParamInfo* param, case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_UTINYINT: - size = 1; + *(uint8_t *)(data + param->offset) = *(uint8_t *)bind->buffer; break; case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_USMALLINT: - size = 2; + *(uint16_t *)(data + param->offset) = *(uint16_t *)bind->buffer; break; case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_UINT: case TSDB_DATA_TYPE_FLOAT: - size = 4; + *(uint32_t *)(data + param->offset) = *(uint32_t *)bind->buffer; break; case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_UBIGINT: case TSDB_DATA_TYPE_DOUBLE: case TSDB_DATA_TYPE_TIMESTAMP: - size = 8; + *(uint64_t *)(data + param->offset) = *(uint64_t *)bind->buffer; break; case TSDB_DATA_TYPE_BINARY: @@ -791,7 +792,6 @@ static int doBindParam(STableDataBlocks* pBlock, char* data, SParamInfo* param, return TSDB_CODE_TSC_INVALID_VALUE; } - memcpy(data + param->offset, bind->buffer, size); if (param->offset == 0) { if (tsCheckTimestamp(pBlock, data + param->offset) != TSDB_CODE_SUCCESS) { tscError("invalid timestamp"); @@ -802,6 +802,101 @@ static int doBindParam(STableDataBlocks* pBlock, char* data, SParamInfo* param, return TSDB_CODE_SUCCESS; } +static int32_t insertStmtGenLastBlock(STableDataBlocks** lastBlock, STableDataBlocks* pBlock) { + *lastBlock = (STableDataBlocks*)malloc(sizeof(STableDataBlocks)); + memcpy(*lastBlock, pBlock, sizeof(STableDataBlocks)); + (*lastBlock)->cloned = true; + +#if 0 + void* tmp = malloc((*pBlock)->numOfAllocedParams * sizeof(SParamInfo)); + memcpy(tmp, (*pBlock)->params, (*pBlock)->numOfAllocedParams * sizeof(SParamInfo)); + (*pBlock)->params = tmp; +#endif + + (*lastBlock)->pData = NULL; + (*lastBlock)->ordered = true; + (*lastBlock)->prevTS = INT64_MIN; + (*lastBlock)->size = sizeof(SSubmitBlk); + (*lastBlock)->tsSource = -1; + +#if 0 + if ((*pBlock)->boundColumnInfo.boundedColumns) { + tmp = malloc((*pBlock)->boundColumnInfo.numOfCols * sizeof(int32_t)); + memcpy(tmp, (*pBlock)->boundColumnInfo.boundedColumns, (*pBlock)->boundColumnInfo.numOfCols * sizeof(int32_t)); + (*pBlock)->boundColumnInfo.boundedColumns = tmp; + } + + if ((*pBlock)->boundColumnInfo.cols) { + tmp = malloc((*pBlock)->boundColumnInfo.numOfCols * sizeof(SBoundColumn)); + memcpy(tmp, (*pBlock)->boundColumnInfo.cols, (*pBlock)->boundColumnInfo.numOfCols * sizeof(SBoundColumn)); + (*pBlock)->boundColumnInfo.cols = tmp; + } +#endif + + return TSDB_CODE_SUCCESS; +} + + +static int32_t insertStmtGenBlock(STscStmt* pStmt, STableDataBlocks** pBlock, STableMeta* pTableMeta, SName* name) { + int32_t code = 0; + + if (pStmt->mtb.lastBlock == NULL) { + tscError("no previous data block"); + return TSDB_CODE_TSC_APP_ERROR; + } + + int32_t msize = tscGetTableMetaSize(pTableMeta); + int32_t tsize = sizeof(STableDataBlocks) + msize; + + void *t = malloc(tsize); + *pBlock = t; + + //*pBlock = (STableDataBlocks*)malloc(sizeof(STableDataBlocks)); + memcpy(*pBlock, pStmt->mtb.lastBlock, sizeof(STableDataBlocks)); + +#if 0 + void* tmp = malloc((*pBlock)->numOfAllocedParams * sizeof(SParamInfo)); + memcpy(tmp, (*pBlock)->params, (*pBlock)->numOfAllocedParams * sizeof(SParamInfo)); + (*pBlock)->params = tmp; +#endif + + t = (char *)t + sizeof(STableDataBlocks); + (*pBlock)->pTableMeta = t; + memcpy((*pBlock)->pTableMeta, pTableMeta, msize); + + (*pBlock)->pData = malloc((*pBlock)->nAllocSize); + //(*pBlock)->pTableMeta = tscTableMetaDup(pTableMeta); + //(*pBlock)->pTableMeta = pTableMeta; + + (*pBlock)->vgId = (*pBlock)->pTableMeta->vgId; + +#if 0 + if ((*pBlock)->boundColumnInfo.boundedColumns) { + tmp = malloc((*pBlock)->boundColumnInfo.numOfCols * sizeof(int32_t)); + memcpy(tmp, (*pBlock)->boundColumnInfo.boundedColumns, (*pBlock)->boundColumnInfo.numOfCols * sizeof(int32_t)); + (*pBlock)->boundColumnInfo.boundedColumns = tmp; + } + + if ((*pBlock)->boundColumnInfo.cols) { + tmp = malloc((*pBlock)->boundColumnInfo.numOfCols * sizeof(SBoundColumn)); + memcpy(tmp, (*pBlock)->boundColumnInfo.cols, (*pBlock)->boundColumnInfo.numOfCols * sizeof(SBoundColumn)); + (*pBlock)->boundColumnInfo.cols = tmp; + } +#endif + + tNameAssign(&(*pBlock)->tableName, name); + + SSubmitBlk* blk = (SSubmitBlk*)(*pBlock)->pData; + memset(blk, 0, sizeof(*blk)); + + code = tsSetBlockInfo(blk, pTableMeta, 0); + if (code != TSDB_CODE_SUCCESS) { + STMT_RET(code); + } + + return TSDB_CODE_SUCCESS; +} + static int doBindBatchParam(STableDataBlocks* pBlock, SParamInfo* param, TAOS_MULTI_BIND* bind, int32_t rowNum) { if (bind->buffer_type != param->type || !isValidDataType(param->type)) { @@ -1173,6 +1268,8 @@ static void insertBatchClean(STscStmt* pStmt) { static int insertBatchStmtExecute(STscStmt* pStmt) { int32_t code = 0; + + int64_t st1 = taosGetTimestampUs(); if(pStmt->mtb.nameSet == false) { tscError("0x%"PRIx64" no table name set", pStmt->pSql->self); @@ -1188,23 +1285,35 @@ static int insertBatchStmtExecute(STscStmt* pStmt) { fillTablesColumnsNull(pStmt->pSql); + int64_t st2 = taosGetTimestampUs(); + if ((code = tscMergeTableDataBlocks(&pStmt->pSql->cmd.insertParam, false)) != TSDB_CODE_SUCCESS) { return code; } + int64_t st3 = taosGetTimestampUs(); + code = tscHandleMultivnodeInsert(pStmt->pSql); if (code != TSDB_CODE_SUCCESS) { return code; } + int64_t st4 = taosGetTimestampUs(); + // wait for the callback function to post the semaphore tsem_wait(&pStmt->pSql->rspSem); + int64_t st5 = taosGetTimestampUs(); + code = pStmt->pSql->res.code; insertBatchClean(pStmt); + int64_t st6 = taosGetTimestampUs(); + + tscDebug("use time:%"PRId64 ",%"PRId64 ",%"PRId64 ",%"PRId64 ",%"PRId64, st2-st1, st3-st2, st4-st3, st5-st4, st6-st5); + return code; } @@ -1228,11 +1337,11 @@ int stmtParseInsertTbTags(SSqlObj* pSql, STscStmt* pStmt) { pStmt->mtb.tbname = sToken; pStmt->mtb.nameSet = false; if (pStmt->mtb.pTableHash == NULL) { - pStmt->mtb.pTableHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + pStmt->mtb.pTableHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); } if (pStmt->mtb.pTableBlockHashList == NULL) { - pStmt->mtb.pTableBlockHashList = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); + pStmt->mtb.pTableBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); } pStmt->mtb.tagSet = true; @@ -1561,6 +1670,9 @@ int taos_stmt_set_tbname_tags(TAOS_STMT* stmt, const char* name, TAOS_BIND* tags SSubmitBlk* pBlk = (SSubmitBlk*) (*t1)->pData; pCmd->batchSize = pBlk->numOfRows; + if (pBlk->numOfRows == 0) { + (*t1)->prevTS = INT64_MIN; + } taosHashPut(pCmd->insertParam.pTableBlockHashList, (void *)&pStmt->mtb.currentUid, sizeof(pStmt->mtb.currentUid), (void*)t1, POINTER_BYTES); @@ -1580,6 +1692,9 @@ int taos_stmt_set_tbname_tags(TAOS_STMT* stmt, const char* name, TAOS_BIND* tags tname.n = strlen(name); SName fullname = {0}; tscSetTableFullName(&fullname, &tname, pSql); + + memcpy(&pTableMetaInfo->name, &fullname, sizeof(fullname)); + code = tscGetTableMeta(pSql, pTableMetaInfo); if (code != TSDB_CODE_SUCCESS) { STMT_RET(code); @@ -1592,25 +1707,21 @@ int taos_stmt_set_tbname_tags(TAOS_STMT* stmt, const char* name, TAOS_BIND* tags STMT_RET(TSDB_CODE_TSC_APP_ERROR); } - memcpy(&pTableMetaInfo->name, &fullname, sizeof(fullname)); - STableDataBlocks* pBlock = NULL; - code = tscGetDataBlockFromList(pCmd->insertParam.pTableBlockHashList, pTableMeta->id.uid, TSDB_PAYLOAD_SIZE, sizeof(SSubmitBlk), - pTableMeta->tableInfo.rowSize, &pTableMetaInfo->name, pTableMeta, &pBlock, NULL); - if (code != TSDB_CODE_SUCCESS) { - STMT_RET(code); - } - SSubmitBlk* blk = (SSubmitBlk*)pBlock->pData; - blk->numOfRows = 0; + insertStmtGenBlock(pStmt, &pBlock, pTableMeta, &pTableMetaInfo->name); + pCmd->batchSize = 0; + pStmt->mtb.currentUid = pTableMeta->id.uid; pStmt->mtb.tbNum++; - + + taosHashPut(pCmd->insertParam.pTableBlockHashList, (void *)&pStmt->mtb.currentUid, sizeof(pStmt->mtb.currentUid), (void*)&pBlock, POINTER_BYTES); taosHashPut(pStmt->mtb.pTableBlockHashList, (void *)&pStmt->mtb.currentUid, sizeof(pStmt->mtb.currentUid), (void*)&pBlock, POINTER_BYTES); taosHashPut(pStmt->mtb.pTableHash, name, strlen(name), (char*) &pTableMeta->id.uid, sizeof(pTableMeta->id.uid)); tscDebug("0x%"PRIx64" table:%s is prepared, uid:%" PRIx64, pSql->self, name, pStmt->mtb.currentUid); + STMT_RET(TSDB_CODE_SUCCESS); } @@ -1670,6 +1781,10 @@ int taos_stmt_set_tbname_tags(TAOS_STMT* stmt, const char* name, TAOS_BIND* tags taosHashPut(pStmt->mtb.pTableBlockHashList, (void *)&pStmt->mtb.currentUid, sizeof(pStmt->mtb.currentUid), (void*)&pBlock, POINTER_BYTES); taosHashPut(pStmt->mtb.pTableHash, name, strlen(name), (char*) &pTableMeta->id.uid, sizeof(pTableMeta->id.uid)); + if (pStmt->mtb.lastBlock == NULL) { + insertStmtGenLastBlock(&pStmt->mtb.lastBlock, pBlock); + } + tscDebug("0x%"PRIx64" table:%s is prepared, uid:%" PRIx64, pSql->self, name, pStmt->mtb.currentUid); } @@ -1711,6 +1826,7 @@ int taos_stmt_close(TAOS_STMT* stmt) { if (pStmt->pSql && pStmt->pSql->res.code != 0) { rmMeta = true; } + tscDestroyDataBlock(pStmt->mtb.lastBlock, rmMeta); pStmt->mtb.pTableBlockHashList = tscDestroyBlockHashTable(pStmt->mtb.pTableBlockHashList, rmMeta); taosHashCleanup(pStmt->pSql->cmd.insertParam.pTableBlockHashList); pStmt->pSql->cmd.insertParam.pTableBlockHashList = NULL; @@ -1745,6 +1861,8 @@ int taos_stmt_bind_param(TAOS_STMT* stmt, TAOS_BIND* bind) { pStmt->last = STMT_BIND; + tscDebug("tableId:%" PRIu64 ", try to bind one row", pStmt->mtb.currentUid); + STMT_RET(insertStmtBindParam(pStmt, bind)); } else { STMT_RET(normalStmtBindParam(pStmt, bind)); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 9b05922f49..020b71d9b8 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1478,12 +1478,6 @@ void tscDestroyDataBlock(STableDataBlocks* pDataBlock, bool removeMeta) { } tfree(pDataBlock->pData); - tfree(pDataBlock->params); - - // free the refcount for metermeta - if (pDataBlock->pTableMeta != NULL) { - tfree(pDataBlock->pTableMeta); - } if (removeMeta) { char name[TSDB_TABLE_FNAME_LEN] = {0}; @@ -1492,7 +1486,17 @@ void tscDestroyDataBlock(STableDataBlocks* pDataBlock, bool removeMeta) { taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); } - tscDestroyBoundColumnInfo(&pDataBlock->boundColumnInfo); + if (!pDataBlock->cloned) { + tfree(pDataBlock->params); + + // free the refcount for metermeta + if (pDataBlock->pTableMeta != NULL) { + tfree(pDataBlock->pTableMeta); + } + + tscDestroyBoundColumnInfo(&pDataBlock->boundColumnInfo); + } + tfree(pDataBlock); } @@ -1630,12 +1634,14 @@ int32_t tscCreateDataBlock(size_t defaultSize, int32_t rowSize, int32_t startOff dataBuf->nAllocSize = dataBuf->headerSize * 2; } - dataBuf->pData = calloc(1, dataBuf->nAllocSize); + //dataBuf->pData = calloc(1, dataBuf->nAllocSize); + dataBuf->pData = malloc(dataBuf->nAllocSize); if (dataBuf->pData == NULL) { tscError("failed to allocated memory, reason:%s", strerror(errno)); tfree(dataBuf); return TSDB_CODE_TSC_OUT_OF_MEMORY; } + memset(dataBuf->pData, 0, sizeof(SSubmitBlk)); //Here we keep the tableMeta to avoid it to be remove by other threads. dataBuf->pTableMeta = tscTableMetaDup(pTableMeta); @@ -1761,16 +1767,14 @@ static int32_t getRowExpandSize(STableMeta* pTableMeta) { static void extractTableNameList(SInsertStatementParam *pInsertParam, bool freeBlockMap) { pInsertParam->numOfTables = (int32_t) taosHashGetSize(pInsertParam->pTableBlockHashList); if (pInsertParam->pTableNameList == NULL) { - pInsertParam->pTableNameList = calloc(pInsertParam->numOfTables, POINTER_BYTES); - } else { - memset(pInsertParam->pTableNameList, 0, pInsertParam->numOfTables * POINTER_BYTES); + pInsertParam->pTableNameList = malloc(pInsertParam->numOfTables * POINTER_BYTES); } STableDataBlocks **p1 = taosHashIterate(pInsertParam->pTableBlockHashList, NULL); int32_t i = 0; while(p1) { STableDataBlocks* pBlocks = *p1; - tfree(pInsertParam->pTableNameList[i]); + //tfree(pInsertParam->pTableNameList[i]); pInsertParam->pTableNameList[i++] = tNameDup(&pBlocks->tableName); p1 = taosHashIterate(pInsertParam->pTableBlockHashList, p1); @@ -1809,14 +1813,12 @@ int32_t tscMergeTableDataBlocks(SInsertStatementParam *pInsertParam, bool freeBl int64_t destSize = dataBuf->size + pOneTableBlock->size + pBlocks->numOfRows * expandSize + sizeof(STColumn) * tscGetNumOfColumns(pOneTableBlock->pTableMeta); if (dataBuf->nAllocSize < destSize) { - while (dataBuf->nAllocSize < destSize) { - dataBuf->nAllocSize = (uint32_t)(dataBuf->nAllocSize * 1.5); - } + dataBuf->nAllocSize = (uint32_t)(destSize * 1.5); char* tmp = realloc(dataBuf->pData, dataBuf->nAllocSize); if (tmp != NULL) { dataBuf->pData = tmp; - memset(dataBuf->pData + dataBuf->size, 0, dataBuf->nAllocSize - dataBuf->size); + //memset(dataBuf->pData + dataBuf->size, 0, dataBuf->nAllocSize - dataBuf->size); } else { // failed to allocate memory, free already allocated memory and return error code tscError("0x%"PRIx64" failed to allocate memory for merging submit block, size:%d", pInsertParam->objectId, dataBuf->nAllocSize); @@ -4144,7 +4146,7 @@ STableMeta* tscTableMetaDup(STableMeta* pTableMeta) { assert(pTableMeta != NULL); size_t size = tscGetTableMetaSize(pTableMeta); - STableMeta* p = calloc(1, size); + STableMeta* p = malloc(size); memcpy(p, pTableMeta, size); return p; } diff --git a/src/common/src/tname.c b/src/common/src/tname.c index 72e2d42ff9..26502c5d9c 100644 --- a/src/common/src/tname.c +++ b/src/common/src/tname.c @@ -306,7 +306,7 @@ bool tIsValidName(const SName* name) { SName* tNameDup(const SName* name) { assert(name != NULL); - SName* p = calloc(1, sizeof(SName)); + SName* p = malloc(sizeof(SName)); memcpy(p, name, sizeof(SName)); return p; } From 35c0cf9df024c1121e11fd973030d313f30c99cd Mon Sep 17 00:00:00 2001 From: wpan Date: Mon, 19 Jul 2021 15:37:06 +0800 Subject: [PATCH 3/4] fix windows compile error --- src/client/src/tscPrepare.c | 45 +------------------------------------ 1 file changed, 1 insertion(+), 44 deletions(-) diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 2900fb668d..5ea00479f5 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -1010,32 +1010,12 @@ static int32_t insertStmtGenLastBlock(STableDataBlocks** lastBlock, STableDataBl memcpy(*lastBlock, pBlock, sizeof(STableDataBlocks)); (*lastBlock)->cloned = true; -#if 0 - void* tmp = malloc((*pBlock)->numOfAllocedParams * sizeof(SParamInfo)); - memcpy(tmp, (*pBlock)->params, (*pBlock)->numOfAllocedParams * sizeof(SParamInfo)); - (*pBlock)->params = tmp; -#endif - (*lastBlock)->pData = NULL; (*lastBlock)->ordered = true; (*lastBlock)->prevTS = INT64_MIN; (*lastBlock)->size = sizeof(SSubmitBlk); (*lastBlock)->tsSource = -1; -#if 0 - if ((*pBlock)->boundColumnInfo.boundedColumns) { - tmp = malloc((*pBlock)->boundColumnInfo.numOfCols * sizeof(int32_t)); - memcpy(tmp, (*pBlock)->boundColumnInfo.boundedColumns, (*pBlock)->boundColumnInfo.numOfCols * sizeof(int32_t)); - (*pBlock)->boundColumnInfo.boundedColumns = tmp; - } - - if ((*pBlock)->boundColumnInfo.cols) { - tmp = malloc((*pBlock)->boundColumnInfo.numOfCols * sizeof(SBoundColumn)); - memcpy(tmp, (*pBlock)->boundColumnInfo.cols, (*pBlock)->boundColumnInfo.numOfCols * sizeof(SBoundColumn)); - (*pBlock)->boundColumnInfo.cols = tmp; - } -#endif - return TSDB_CODE_SUCCESS; } @@ -1054,39 +1034,16 @@ static int32_t insertStmtGenBlock(STscStmt* pStmt, STableDataBlocks** pBlock, ST void *t = malloc(tsize); *pBlock = t; - //*pBlock = (STableDataBlocks*)malloc(sizeof(STableDataBlocks)); memcpy(*pBlock, pStmt->mtb.lastBlock, sizeof(STableDataBlocks)); - -#if 0 - void* tmp = malloc((*pBlock)->numOfAllocedParams * sizeof(SParamInfo)); - memcpy(tmp, (*pBlock)->params, (*pBlock)->numOfAllocedParams * sizeof(SParamInfo)); - (*pBlock)->params = tmp; -#endif t = (char *)t + sizeof(STableDataBlocks); (*pBlock)->pTableMeta = t; memcpy((*pBlock)->pTableMeta, pTableMeta, msize); (*pBlock)->pData = malloc((*pBlock)->nAllocSize); - //(*pBlock)->pTableMeta = tscTableMetaDup(pTableMeta); - //(*pBlock)->pTableMeta = pTableMeta; (*pBlock)->vgId = (*pBlock)->pTableMeta->vgId; -#if 0 - if ((*pBlock)->boundColumnInfo.boundedColumns) { - tmp = malloc((*pBlock)->boundColumnInfo.numOfCols * sizeof(int32_t)); - memcpy(tmp, (*pBlock)->boundColumnInfo.boundedColumns, (*pBlock)->boundColumnInfo.numOfCols * sizeof(int32_t)); - (*pBlock)->boundColumnInfo.boundedColumns = tmp; - } - - if ((*pBlock)->boundColumnInfo.cols) { - tmp = malloc((*pBlock)->boundColumnInfo.numOfCols * sizeof(SBoundColumn)); - memcpy(tmp, (*pBlock)->boundColumnInfo.cols, (*pBlock)->boundColumnInfo.numOfCols * sizeof(SBoundColumn)); - (*pBlock)->boundColumnInfo.cols = tmp; - } -#endif - tNameAssign(&(*pBlock)->tableName, name); SSubmitBlk* blk = (SSubmitBlk*)(*pBlock)->pData; @@ -1881,7 +1838,7 @@ int taos_stmt_set_tbname_tags(TAOS_STMT* stmt, const char* name, TAOS_BIND* tags SStrToken tname = {0}; tname.type = TK_STRING; tname.z = (char *)name; - tname.n = strlen(name); + tname.n = (uint32_t)strlen(name); SName fullname = {0}; tscSetTableFullName(&fullname, &tname, pSql); From 5e07893a44de388cf492f7420fc33494579d73ec Mon Sep 17 00:00:00 2001 From: XYWang Date: Tue, 20 Jul 2021 19:33:04 +0800 Subject: [PATCH 4/4] [TD-5234]: added name for threads --- src/balance/src/bnThread.c | 2 ++ src/dnode/src/dnodeMPeer.c | 2 ++ src/dnode/src/dnodeMRead.c | 2 ++ src/dnode/src/dnodeMWrite.c | 4 +++- src/dnode/src/dnodeTelemetry.c | 2 ++ src/dnode/src/dnodeVMgmt.c | 2 ++ src/dnode/src/dnodeVRead.c | 5 +++++ src/dnode/src/dnodeVWrite.c | 2 ++ src/dnode/src/dnodeVnodes.c | 2 ++ src/kit/shell/src/shellCheck.c | 2 ++ src/kit/shell/src/shellDarwin.c | 2 ++ src/kit/shell/src/shellImport.c | 2 ++ src/kit/shell/src/shellLinux.c | 2 ++ src/kit/shell/src/shellMain.c | 2 ++ src/kit/taosdemo/taosdemo.c | 19 +++++++++++++++++-- src/kit/taosdump/taosdump.c | 4 ++++ src/mnode/src/mnodeSdb.c | 1 + src/os/inc/osDef.h | 19 +++++++++++++++++++ src/os/inc/osInc.h | 1 + src/os/src/darwin/dwSemaphore.c | 2 ++ src/os/src/darwin/dwTimer.c | 1 + src/os/src/detail/osTimer.c | 2 ++ src/plugins/http/src/httpQueue.c | 2 ++ src/plugins/http/src/httpServer.c | 2 ++ src/plugins/monitor/src/monMain.c | 1 + src/plugins/mqtt/src/mqttSystem.c | 4 +++- src/rpc/src/rpcTcp.c | 6 ++++++ src/rpc/src/rpcUdp.c | 2 ++ src/rpc/test/rclient.c | 2 ++ src/rpc/test/rsclient.c | 4 +++- src/sync/src/syncRestore.c | 1 + src/sync/src/syncRetrieve.c | 1 + src/sync/src/syncTcp.c | 3 +++ src/sync/test/syncClient.c | 2 ++ src/sync/test/syncServer.c | 2 ++ src/tsdb/src/tsdbCommitQueue.c | 4 +++- src/util/src/tcache.c | 2 ++ src/util/src/tlog.c | 4 ++++ src/util/src/tnettest.c | 8 ++++++-- src/util/src/tnote.c | 2 ++ src/util/src/tsched.c | 4 +++- src/util/tests/trefTest.c | 8 ++++++++ src/vnode/src/vnodeBackup.c | 2 ++ src/vnode/src/vnodeWorker.c | 2 ++ src/wal/src/walMgmt.c | 1 + 45 files changed, 142 insertions(+), 9 deletions(-) diff --git a/src/balance/src/bnThread.c b/src/balance/src/bnThread.c index 44cb24effa..c5dca2da85 100644 --- a/src/balance/src/bnThread.c +++ b/src/balance/src/bnThread.c @@ -23,6 +23,8 @@ static SBnThread tsBnThread; static void *bnThreadFunc(void *arg) { + setThreadName("bnThreadd"); + while (1) { pthread_mutex_lock(&tsBnThread.mutex); if (tsBnThread.stop) { diff --git a/src/dnode/src/dnodeMPeer.c b/src/dnode/src/dnodeMPeer.c index e4942c49aa..8aa28d1618 100644 --- a/src/dnode/src/dnodeMPeer.c +++ b/src/dnode/src/dnodeMPeer.c @@ -150,6 +150,8 @@ static void *dnodeProcessMPeerQueue(void *param) { SMnodeMsg *pPeerMsg; int32_t type; void * unUsed; + + setThreadName("dnodeMPeerQ"); while (1) { if (taosReadQitemFromQset(tsMPeerQset, &type, (void **)&pPeerMsg, &unUsed) == 0) { diff --git a/src/dnode/src/dnodeMRead.c b/src/dnode/src/dnodeMRead.c index 90332e6783..184a6b743a 100644 --- a/src/dnode/src/dnodeMRead.c +++ b/src/dnode/src/dnodeMRead.c @@ -155,6 +155,8 @@ static void *dnodeProcessMReadQueue(void *param) { int32_t type; void * unUsed; + setThreadName("dnodeMReadQ"); + while (1) { if (taosReadQitemFromQset(tsMReadQset, &type, (void **)&pRead, &unUsed) == 0) { dDebug("qset:%p, mnode read got no message from qset, exiting", tsMReadQset); diff --git a/src/dnode/src/dnodeMWrite.c b/src/dnode/src/dnodeMWrite.c index a409d537fa..904ddc21d0 100644 --- a/src/dnode/src/dnodeMWrite.c +++ b/src/dnode/src/dnodeMWrite.c @@ -168,7 +168,9 @@ static void *dnodeProcessMWriteQueue(void *param) { SMnodeMsg *pWrite; int32_t type; void * unUsed; - + + setThreadName("dnodeMWriteQ"); + while (1) { if (taosReadQitemFromQset(tsMWriteQset, &type, (void **)&pWrite, &unUsed) == 0) { dDebug("qset:%p, mnode write got no message from qset, exiting", tsMWriteQset); diff --git a/src/dnode/src/dnodeTelemetry.c b/src/dnode/src/dnodeTelemetry.c index 4caece1661..59b66879d4 100644 --- a/src/dnode/src/dnodeTelemetry.c +++ b/src/dnode/src/dnodeTelemetry.c @@ -245,6 +245,8 @@ static void* telemetryThread(void* param) { clock_gettime(CLOCK_REALTIME, &end); end.tv_sec += 300; // wait 5 minutes before send first report + setThreadName("telemetryThrd"); + while (!tsExit) { int r = 0; struct timespec ts = end; diff --git a/src/dnode/src/dnodeVMgmt.c b/src/dnode/src/dnodeVMgmt.c index daf62aac94..c1bfb1460b 100644 --- a/src/dnode/src/dnodeVMgmt.c +++ b/src/dnode/src/dnodeVMgmt.c @@ -103,6 +103,8 @@ static void *dnodeProcessMgmtQueue(void *wparam) { int32_t qtype; void * handle; + setThreadName("dnodeMgmtQ"); + while (1) { if (taosReadQitemFromQset(pPool->qset, &qtype, (void **)&pMgmt, &handle) == 0) { dDebug("qdnode mgmt got no message from qset:%p, , exit", pPool->qset); diff --git a/src/dnode/src/dnodeVRead.c b/src/dnode/src/dnodeVRead.c index 41016d7b99..e8003a8fe7 100644 --- a/src/dnode/src/dnodeVRead.c +++ b/src/dnode/src/dnodeVRead.c @@ -118,6 +118,11 @@ static void *dnodeProcessReadQueue(void *wparam) { SVReadMsg * pRead; int32_t qtype; void * pVnode; + char name[16]; + + memset(name, 0, 16); + snprintf(name, 16, "%s-dnReadQ", pPool->name); + setThreadName(name); while (1) { if (taosReadQitemFromQset(pPool->qset, &qtype, (void **)&pRead, &pVnode) == 0) { diff --git a/src/dnode/src/dnodeVWrite.c b/src/dnode/src/dnodeVWrite.c index ff2d12f001..ed2a6e2109 100644 --- a/src/dnode/src/dnodeVWrite.c +++ b/src/dnode/src/dnodeVWrite.c @@ -191,6 +191,8 @@ static void *dnodeProcessVWriteQueue(void *wparam) { taosBlockSIGPIPE(); dDebug("dnode vwrite worker:%d is running", pWorker->workerId); + setThreadName("dnodeWriteQ"); + while (1) { numOfMsgs = taosReadAllQitemsFromQset(pWorker->qset, pWorker->qall, &pVnode); if (numOfMsgs == 0) { diff --git a/src/dnode/src/dnodeVnodes.c b/src/dnode/src/dnodeVnodes.c index f01a510370..8ea8e280de 100644 --- a/src/dnode/src/dnodeVnodes.c +++ b/src/dnode/src/dnodeVnodes.c @@ -91,6 +91,8 @@ static void *dnodeOpenVnode(void *param) { dDebug("thread:%d, start to open %d vnodes", pThread->threadIndex, pThread->vnodeNum); + setThreadName("dnodeOpenVnode"); + for (int32_t v = 0; v < pThread->vnodeNum; ++v) { int32_t vgId = pThread->vnodeList[v]; snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to restore, %d of %d have been opened", vgId, tsOpenVnodes, tsTotalVnodes); diff --git a/src/kit/shell/src/shellCheck.c b/src/kit/shell/src/shellCheck.c index 4ff5dc36fc..d78f1a6b99 100644 --- a/src/kit/shell/src/shellCheck.c +++ b/src/kit/shell/src/shellCheck.c @@ -104,6 +104,8 @@ static void shellFreeTbnames() { static void *shellCheckThreadFp(void *arg) { ShellThreadObj *pThread = (ShellThreadObj *)arg; + setThreadName("shellCheckThrd"); + int32_t interval = tbNum / pThread->totalThreads + 1; int32_t start = pThread->threadIndex * interval; int32_t end = (pThread->threadIndex + 1) * interval; diff --git a/src/kit/shell/src/shellDarwin.c b/src/kit/shell/src/shellDarwin.c index 31ad7046e9..86c0fea573 100644 --- a/src/kit/shell/src/shellDarwin.c +++ b/src/kit/shell/src/shellDarwin.c @@ -336,6 +336,8 @@ void *shellLoopQuery(void *arg) { TAOS *con = (TAOS *)arg; + setThreadName("shellLoopQuery"); + pthread_cleanup_push(cleanup_handler, NULL); char *command = malloc(MAX_COMMAND_SIZE); diff --git a/src/kit/shell/src/shellImport.c b/src/kit/shell/src/shellImport.c index 5de50a3aaf..222d69e854 100644 --- a/src/kit/shell/src/shellImport.c +++ b/src/kit/shell/src/shellImport.c @@ -223,6 +223,8 @@ static void shellSourceFile(TAOS *con, char *fptr) { void* shellImportThreadFp(void *arg) { ShellThreadObj *pThread = (ShellThreadObj*)arg; + setThreadName("shellImportThrd"); + for (int f = 0; f < shellSQLFileNum; ++f) { if (f % pThread->totalThreads == pThread->threadIndex) { char *SQLFileName = shellSQLFiles[f]; diff --git a/src/kit/shell/src/shellLinux.c b/src/kit/shell/src/shellLinux.c index 4eead252fd..2a32a8d82e 100644 --- a/src/kit/shell/src/shellLinux.c +++ b/src/kit/shell/src/shellLinux.c @@ -336,6 +336,8 @@ void *shellLoopQuery(void *arg) { TAOS *con = (TAOS *)arg; + setThreadName("shellLoopQuery"); + pthread_cleanup_push(cleanup_handler, NULL); char *command = malloc(MAX_COMMAND_SIZE); diff --git a/src/kit/shell/src/shellMain.c b/src/kit/shell/src/shellMain.c index 4c7e550760..0c70386061 100644 --- a/src/kit/shell/src/shellMain.c +++ b/src/kit/shell/src/shellMain.c @@ -26,6 +26,8 @@ void shellQueryInterruptHandler(int32_t signum, void *sigInfo, void *context) { } void *cancelHandler(void *arg) { + setThreadName("cancelHandler"); + while(1) { if (tsem_wait(&cancelSem) != 0) { taosMsleep(10); diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 2beb0c8e7e..a15caa008e 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -3025,10 +3025,11 @@ static void* createTable(void *sarg) threadInfo *pThreadInfo = (threadInfo *)sarg; SSuperTable* superTblInfo = pThreadInfo->superTblInfo; + setThreadName("createTable"); + uint64_t lastPrintTime = taosGetTimestampMs(); - int buff_len; - buff_len = BUFFER_SIZE; + int buff_len = BUFFER_SIZE; pThreadInfo->buffer = calloc(buff_len, 1); if (pThreadInfo->buffer == NULL) { @@ -6428,6 +6429,8 @@ static void* syncWrite(void *sarg) { threadInfo *pThreadInfo = (threadInfo *)sarg; SSuperTable* superTblInfo = pThreadInfo->superTblInfo; + setThreadName("syncWrite"); + uint32_t interlaceRows; if (superTblInfo) { @@ -6513,6 +6516,8 @@ static void *asyncWrite(void *sarg) { threadInfo *pThreadInfo = (threadInfo *)sarg; SSuperTable* superTblInfo = pThreadInfo->superTblInfo; + setThreadName("asyncWrite"); + pThreadInfo->st = 0; pThreadInfo->et = 0; pThreadInfo->lastTs = pThreadInfo->start_time; @@ -6911,6 +6916,7 @@ static void *readTable(void *sarg) { #if 1 threadInfo *pThreadInfo = (threadInfo *)sarg; TAOS *taos = pThreadInfo->taos; + setThreadName("readTable"); char command[BUFFER_SIZE] = "\0"; uint64_t sTime = pThreadInfo->start_time; char *tb_prefix = pThreadInfo->tb_prefix; @@ -6983,6 +6989,7 @@ static void *readMetric(void *sarg) { #if 1 threadInfo *pThreadInfo = (threadInfo *)sarg; TAOS *taos = pThreadInfo->taos; + setThreadName("readMetric"); char command[BUFFER_SIZE] = "\0"; FILE *fp = fopen(pThreadInfo->filePath, "a"); if (NULL == fp) { @@ -7159,6 +7166,8 @@ static int insertTestProcess() { static void *specifiedTableQuery(void *sarg) { threadInfo *pThreadInfo = (threadInfo *)sarg; + setThreadName("specTableQuery"); + if (pThreadInfo->taos == NULL) { TAOS * taos = NULL; taos = taos_connect(g_queryInfo.host, @@ -7258,6 +7267,8 @@ static void *superTableQuery(void *sarg) { char sqlstr[MAX_QUERY_SQL_LENGTH]; threadInfo *pThreadInfo = (threadInfo *)sarg; + setThreadName("superTableQuery"); + if (pThreadInfo->taos == NULL) { TAOS * taos = NULL; taos = taos_connect(g_queryInfo.host, @@ -7560,6 +7571,8 @@ static void *superSubscribe(void *sarg) { TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT] = {0}; uint64_t tsubSeq; + setThreadName("superSub"); + if (pThreadInfo->ntables > MAX_QUERY_SQL_COUNT) { errorPrint("The table number(%"PRId64") of the thread is more than max query sql count: %d\n", pThreadInfo->ntables, MAX_QUERY_SQL_COUNT); @@ -7706,6 +7719,8 @@ static void *specifiedSubscribe(void *sarg) { threadInfo *pThreadInfo = (threadInfo *)sarg; // TAOS_SUB* tsub = NULL; + setThreadName("specSub"); + if (pThreadInfo->taos == NULL) { pThreadInfo->taos = taos_connect(g_queryInfo.host, g_queryInfo.user, diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index 98521d8420..e5501b4366 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -1474,6 +1474,8 @@ static void* taosDumpOutWorkThreadFp(void *arg) STableRecord tableRecord; int fd; + setThreadName("dumpOutWorkThrd"); + char tmpBuf[4096] = {0}; sprintf(tmpBuf, ".tables.tmp.%d", pThread->threadIndex); fd = open(tmpBuf, O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); @@ -2571,6 +2573,8 @@ static int taosDumpInOneFile(TAOS* taos, FILE* fp, char* fcharset, static void* taosDumpInWorkThreadFp(void *arg) { SThreadParaObj *pThread = (SThreadParaObj*)arg; + setThreadName("dumpInWorkThrd"); + for (int32_t f = 0; f < g_tsSqlFileNum; ++f) { if (f % pThread->totalThreads == pThread->threadIndex) { char *SQLFileName = g_tsDumpInSqlFiles[f]; diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 897c3a2f0f..7644f4d733 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -1113,6 +1113,7 @@ static void *sdbWorkerFp(void *pWorker) { void * unUsed; taosBlockSIGPIPE(); + setThreadName("sdbWorker"); while (1) { int32_t numOfMsgs = taosReadAllQitemsFromQset(tsSdbWQset, tsSdbWQall, &unUsed); diff --git a/src/os/inc/osDef.h b/src/os/inc/osDef.h index 9176da5b8e..54a4f98254 100644 --- a/src/os/inc/osDef.h +++ b/src/os/inc/osDef.h @@ -210,6 +210,25 @@ extern "C" { #define PRIzu "zu" #endif + +#if defined(_TD_LINUX_64) || defined(_TD_LINUX_32) || defined(_TD_MIPS_64) || defined(_TD_ARM_32) || defined(_TD_ARM_64) || defined(_TD_DARWIN_64) + #if defined(_TD_DARWIN_64) + // MacOS + #if !defined(_GNU_SOURCE) + #define setThreadName(name) do { pthread_setname_np((name)); } while (0) + #else + // pthread_setname_np not defined + #define setThreadName(name) + #endif + #else + // Linux, length of name must <= 16 (the last '\0' included) + #define setThreadName(name) do { prctl(PR_SET_NAME, (name)); } while (0) + #endif +#else + // Windows + #define setThreadName(name) +#endif + #ifdef __cplusplus } #endif diff --git a/src/os/inc/osInc.h b/src/os/inc/osInc.h index 340ff34635..9b78110833 100644 --- a/src/os/inc/osInc.h +++ b/src/os/inc/osInc.h @@ -85,6 +85,7 @@ extern "C" { #include #include #include + #include #if !(defined(_ALPINE)) #include diff --git a/src/os/src/darwin/dwSemaphore.c b/src/os/src/darwin/dwSemaphore.c index 898410647a..25cb28cff1 100644 --- a/src/os/src/darwin/dwSemaphore.c +++ b/src/os/src/darwin/dwSemaphore.c @@ -41,6 +41,8 @@ static semaphore_t sem_exit; static void* sem_thread_routine(void *arg) { (void)arg; + setThreadName("sem_thrd"); + sem_port = mach_task_self(); kern_return_t ret = semaphore_create(sem_port, &sem_exit, SYNC_POLICY_FIFO, 0); if (ret != KERN_SUCCESS) { diff --git a/src/os/src/darwin/dwTimer.c b/src/os/src/darwin/dwTimer.c index ee1becc91a..d395a7f53f 100644 --- a/src/os/src/darwin/dwTimer.c +++ b/src/os/src/darwin/dwTimer.c @@ -32,6 +32,7 @@ static volatile int timer_stop = 0; static void* timer_routine(void *arg) { (void)arg; + setThreadName("timer"); int r = 0; struct timespec to = {0}; diff --git a/src/os/src/detail/osTimer.c b/src/os/src/detail/osTimer.c index b054f08c78..c381b3e825 100644 --- a/src/os/src/detail/osTimer.c +++ b/src/os/src/detail/osTimer.c @@ -38,6 +38,8 @@ static void *taosProcessAlarmSignal(void *tharg) { struct sigevent sevent = {{0}}; + setThreadName("alarmSignal"); + #ifdef _ALPINE sevent.sigev_notify = SIGEV_THREAD; sevent.sigev_value.sival_int = syscall(__NR_gettid); diff --git a/src/plugins/http/src/httpQueue.c b/src/plugins/http/src/httpQueue.c index 7f7ce40460..677ab0c91d 100644 --- a/src/plugins/http/src/httpQueue.c +++ b/src/plugins/http/src/httpQueue.c @@ -70,6 +70,8 @@ static void *httpProcessResultQueue(void *param) { int32_t type; void * unUsed; + setThreadName("httpResultQ"); + while (1) { if (taosReadQitemFromQset(tsHttpQset, &type, (void **)&pMsg, &unUsed) == 0) { httpDebug("qset:%p, http queue got no message from qset, exiting", tsHttpQset); diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index 9d98d3f113..f02859f165 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -117,6 +117,7 @@ static void httpProcessHttpData(void *param) { int32_t fdNum; taosSetMaskSIGPIPE(); + setThreadName("httpData"); while (1) { struct epoll_event events[HTTP_MAX_EVENTS]; @@ -208,6 +209,7 @@ static void *httpAcceptHttpConnection(void *arg) { int32_t totalFds = 0; taosSetMaskSIGPIPE(); + setThreadName("httpAcceptConn"); pServer->fd = taosOpenTcpServerSocket(pServer->serverIp, pServer->serverPort); diff --git a/src/plugins/monitor/src/monMain.c b/src/plugins/monitor/src/monMain.c index 2c4a0c1a4c..960a097f5d 100644 --- a/src/plugins/monitor/src/monMain.c +++ b/src/plugins/monitor/src/monMain.c @@ -114,6 +114,7 @@ int32_t monStartSystem() { static void *monThreadFunc(void *param) { monDebug("starting to initialize monitor module ..."); + setThreadName("monThrd"); while (1) { static int32_t accessTimes = 0; diff --git a/src/plugins/mqtt/src/mqttSystem.c b/src/plugins/mqtt/src/mqttSystem.c index eacc3b1f74..e0f2f393bb 100644 --- a/src/plugins/mqtt/src/mqttSystem.c +++ b/src/plugins/mqtt/src/mqttSystem.c @@ -100,6 +100,8 @@ void mqttPublishCallback(void** unused, struct mqtt_response_publish* published) } void* mqttClientRefresher(void* client) { + setThreadName("mqttCliRefresh"); + while (tsMqttIsRuning) { mqtt_sync((struct mqtt_client*)client); taosMsleep(100); @@ -141,4 +143,4 @@ void mqttReconnectClient(struct mqtt_client* client, void** unused) { mqtt_reinit(client, sockfd, tsMqttStatus.sendbuf, tsMqttStatus.sendbufsz, tsMqttStatus.recvbuf, tsMqttStatus.recvbufsz); mqtt_connect(client, tsMqttClientId, NULL, NULL, 0, tsMqttUser, tsMqttPass, MQTT_CONNECT_CLEAN_SESSION, 400); mqtt_subscribe(client, tsMqttTopic, 0); -} \ No newline at end of file +} diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c index 029629eff0..e9feeef9d3 100644 --- a/src/rpc/src/rpcTcp.c +++ b/src/rpc/src/rpcTcp.c @@ -242,6 +242,7 @@ static void *taosAcceptTcpConnection(void *arg) { pServerObj = (SServerObj *)arg; tDebug("%s TCP server is ready, ip:0x%x:%hu", pServerObj->label, pServerObj->ip, pServerObj->port); + setThreadName("acceptTcpConn"); while (1) { socklen_t addrlen = sizeof(caddr); @@ -528,6 +529,11 @@ static void *taosProcessTcpData(void *param) { SFdObj *pFdObj; struct epoll_event events[maxEvents]; SRecvInfo recvInfo; + char name[16]; + + memset(name, 0, sizeof(name)); + snprintf(name, 16, "%s-tcpData", pThreadObj->label); + setThreadName(name); while (1) { int fdNum = epoll_wait(pThreadObj->pollFd, events, maxEvents, TAOS_EPOLL_WAIT_TIME); diff --git a/src/rpc/src/rpcUdp.c b/src/rpc/src/rpcUdp.c index 7a46dbe5c3..086a390cb8 100644 --- a/src/rpc/src/rpcUdp.c +++ b/src/rpc/src/rpcUdp.c @@ -195,6 +195,8 @@ static void *taosRecvUdpData(void *param) { tDebug("%s UDP thread is created, index:%d", pConn->label, pConn->index); char *msg = pConn->buffer; + setThreadName("recvUdpData"); + while (1) { dataLen = recvfrom(pConn->fd, pConn->buffer, RPC_MAX_UDP_SIZE, 0, (struct sockaddr *)&sourceAdd, &addLen); if (dataLen <= 0) { diff --git a/src/rpc/test/rclient.c b/src/rpc/test/rclient.c index faa6d40da3..de30114bd1 100644 --- a/src/rpc/test/rclient.c +++ b/src/rpc/test/rclient.c @@ -47,6 +47,8 @@ static int tcount = 0; static void *sendRequest(void *param) { SInfo *pInfo = (SInfo *)param; SRpcMsg rpcMsg = {0}; + + setThreadName("sendCliReq"); tDebug("thread:%d, start to send request", pInfo->index); diff --git a/src/rpc/test/rsclient.c b/src/rpc/test/rsclient.c index a152d8e4a5..3e94a56efb 100644 --- a/src/rpc/test/rsclient.c +++ b/src/rpc/test/rsclient.c @@ -39,8 +39,10 @@ static int terror = 0; static void *sendRequest(void *param) { SInfo *pInfo = (SInfo *)param; - SRpcMsg rpcMsg, rspMsg; + SRpcMsg rpcMsg, rspMsg; + setThreadName("sendSrvReq"); + tDebug("thread:%d, start to send request", pInfo->index); while ( pInfo->numOfReqs == 0 || pInfo->num < pInfo->numOfReqs) { diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index c0d66316cd..bf9d5201a0 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -263,6 +263,7 @@ static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) { } void *syncRestoreData(void *param) { + setThreadName("syncRestoreData"); int64_t rid = (int64_t)param; SSyncPeer *pPeer = syncAcquirePeer(rid); if (pPeer == NULL) { diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index c86ab85499..89fdda0686 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -415,6 +415,7 @@ static int32_t syncRetrieveDataStepByStep(SSyncPeer *pPeer) { } void *syncRetrieveData(void *param) { + setThreadName("syncRetrievData"); int64_t rid = (int64_t)param; SSyncPeer *pPeer = syncAcquirePeer(rid); if (pPeer == NULL) { diff --git a/src/sync/src/syncTcp.c b/src/sync/src/syncTcp.c index 3ad9e9bba0..698245f9e4 100644 --- a/src/sync/src/syncTcp.c +++ b/src/sync/src/syncTcp.c @@ -195,6 +195,8 @@ static void *syncProcessTcpData(void *param) { SConnObj * pConn = NULL; struct epoll_event events[maxEvents]; + setThreadName("syncTcpData"); + void *buffer = malloc(pInfo->bufferSize); taosBlockSIGPIPE(); @@ -257,6 +259,7 @@ static void *syncAcceptPeerTcpConnection(void *argv) { SPoolInfo *pInfo = &pPool->info; taosBlockSIGPIPE(); + setThreadName("acceptTcpConn"); while (1) { struct sockaddr_in clientAddr; diff --git a/src/sync/test/syncClient.c b/src/sync/test/syncClient.c index 23ea54ee0c..303d2376ef 100644 --- a/src/sync/test/syncClient.c +++ b/src/sync/test/syncClient.c @@ -48,6 +48,8 @@ void *sendRequest(void *param) { SInfo * pInfo = (SInfo *)param; SRpcMsg rpcMsg = {0}; + setThreadName("sendCliReq"); + uDebug("thread:%d, start to send request", pInfo->index); while (pInfo->numOfReqs == 0 || pInfo->num < pInfo->numOfReqs) { diff --git a/src/sync/test/syncServer.c b/src/sync/test/syncServer.c index eeaa6a08c2..a3d0696648 100644 --- a/src/sync/test/syncServer.c +++ b/src/sync/test/syncServer.c @@ -178,6 +178,8 @@ void *processWriteQueue(void *param) { int type; void *item; + setThreadName("writeQ"); + while (1) { int ret = taosReadQitem(qhandle, &type, &item); if (ret <= 0) { diff --git a/src/tsdb/src/tsdbCommitQueue.c b/src/tsdb/src/tsdbCommitQueue.c index e25014bc1e..e45ac05e97 100644 --- a/src/tsdb/src/tsdbCommitQueue.c +++ b/src/tsdb/src/tsdbCommitQueue.c @@ -158,6 +158,8 @@ static void *tsdbLoopCommit(void *arg) { STsdbRepo * pRepo = NULL; TSDB_REQ_T req; + setThreadName("tsdbCommit"); + while (true) { pthread_mutex_lock(&(pQueue->lock)); @@ -208,4 +210,4 @@ void tsdbDecCommitRef(int vgId) { int refCount = atomic_sub_fetch_32(&tsCommitQueue.refCount, 1); pthread_cond_broadcast(&(tsCommitQueue.queueNotEmpty)); tsdbDebug("vgId:%d, dec commit queue ref to %d", vgId, refCount); -} \ No newline at end of file +} diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index 569f9b01bd..c6ee79e101 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -656,6 +656,8 @@ void* taosCacheTimedRefresh(void *handle) { return NULL; } + setThreadName("cacheTimedRefre"); + const int32_t SLEEP_DURATION = 500; //500 ms int64_t totalTick = pCacheObj->refreshTime / SLEEP_DURATION; diff --git a/src/util/src/tlog.c b/src/util/src/tlog.c index 45ff14ffa4..88f57e8ac2 100644 --- a/src/util/src/tlog.c +++ b/src/util/src/tlog.c @@ -178,6 +178,8 @@ static void *taosThreadToOpenNewFile(void *param) { char keepName[LOG_FILE_NAME_LEN + 20]; sprintf(keepName, "%s.%d", tsLogObj.logName, tsLogObj.flag); + setThreadName("openNewFile"); + tsLogObj.flag ^= 1; tsLogObj.lines = 0; char name[LOG_FILE_NAME_LEN + 20]; @@ -687,6 +689,8 @@ static void taosWriteLog(SLogBuff *tLogBuff) { static void *taosAsyncOutputLog(void *param) { SLogBuff *tLogBuff = (SLogBuff *)param; + + setThreadName("asyncOutputLog"); while (1) { //tsem_wait(&(tLogBuff->buffNotEmpty)); diff --git a/src/util/src/tnettest.c b/src/util/src/tnettest.c index 318a2d4860..0bab7b7e66 100644 --- a/src/util/src/tnettest.c +++ b/src/util/src/tnettest.c @@ -50,7 +50,9 @@ static void *taosNetBindUdpPort(void *sarg) { struct sockaddr_in server_addr; struct sockaddr_in clientAddr; - if ((serverSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) { + setThreadName("netBindUdpPort"); + + if ((serverSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) { uError("failed to create UDP socket since %s", strerror(errno)); return NULL; } @@ -106,13 +108,15 @@ static void *taosNetBindTcpPort(void *sarg) { struct sockaddr_in server_addr; struct sockaddr_in clientAddr; - STestInfo *pinfo = sarg; + STestInfo *pinfo = sarg; int32_t port = pinfo->port; SOCKET serverSocket; int32_t addr_len = sizeof(clientAddr); SOCKET client; char buffer[BUFFER_SIZE]; + setThreadName("netBindTcpPort"); + if ((serverSocket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0) { uError("failed to create TCP socket since %s", strerror(errno)); return NULL; diff --git a/src/util/src/tnote.c b/src/util/src/tnote.c index c6d49dfb3d..b691abc5b9 100644 --- a/src/util/src/tnote.c +++ b/src/util/src/tnote.c @@ -84,6 +84,8 @@ static void *taosThreadToOpenNewNote(void *param) { char name[NOTE_FILE_NAME_LEN * 2]; SNoteObj *pNote = (SNoteObj *)param; + setThreadName("openNewNote"); + pNote->flag ^= 1; pNote->lines = 0; sprintf(name, "%s.%d", pNote->name, pNote->flag); diff --git a/src/util/src/tsched.c b/src/util/src/tsched.c index 16142470c9..3d3dfd9899 100644 --- a/src/util/src/tsched.c +++ b/src/util/src/tsched.c @@ -122,6 +122,8 @@ void *taosProcessSchedQueue(void *scheduler) { SSchedQueue *pSched = (SSchedQueue *)scheduler; int ret = 0; + setThreadName("schedQ"); + while (1) { if ((ret = tsem_wait(&pSched->fullSem)) != 0) { uFatal("wait %s fullSem failed(%s)", pSched->label, strerror(errno)); @@ -234,4 +236,4 @@ void taosDumpSchedulerStatus(void *qhandle, void *tmrId) { } taosTmrReset(taosDumpSchedulerStatus, DUMP_SCHEDULER_TIME_WINDOW, pSched, pSched->pTmrCtrl, &pSched->pTimer); -} \ No newline at end of file +} diff --git a/src/util/tests/trefTest.c b/src/util/tests/trefTest.c index 454860410b..d89ccb6879 100644 --- a/src/util/tests/trefTest.c +++ b/src/util/tests/trefTest.c @@ -36,6 +36,8 @@ void *addRef(void *param) { int id; int64_t rid; + setThreadName("addRef"); + for (int i=0; i < pSpace->steps; ++i) { printf("a"); id = random() % pSpace->refNum; @@ -54,6 +56,8 @@ void *removeRef(void *param) { int id; int64_t rid; + setThreadName("removeRef"); + for (int i=0; i < pSpace->steps; ++i) { printf("d"); id = random() % pSpace->refNum; @@ -73,6 +77,8 @@ void *acquireRelease(void *param) { int id; int64_t rid; + setThreadName("acquireRelease"); + for (int i=0; i < pSpace->steps; ++i) { printf("a"); @@ -94,6 +100,8 @@ void myfree(void *p) { void *openRefSpace(void *param) { SRefSpace *pSpace = (SRefSpace *)param; + setThreadName("openRefSpace"); + printf("c"); pSpace->rsetId = taosOpenRef(50, myfree); diff --git a/src/vnode/src/vnodeBackup.c b/src/vnode/src/vnodeBackup.c index a0a975be2b..801af42e0e 100644 --- a/src/vnode/src/vnodeBackup.c +++ b/src/vnode/src/vnodeBackup.c @@ -61,6 +61,8 @@ static void vnodeProcessBackupMsg(SVBackupMsg *pMsg) { } static void *vnodeBackupFunc(void *param) { + setThreadName("vnodeBackup"); + while (1) { SVBackupMsg *pMsg = NULL; if (taosReadQitemFromQset(tsVBackupQset, NULL, (void **)&pMsg, NULL) == 0) { diff --git a/src/vnode/src/vnodeWorker.c b/src/vnode/src/vnodeWorker.c index 6fb79d10fe..e94c99cbea 100644 --- a/src/vnode/src/vnodeWorker.c +++ b/src/vnode/src/vnodeWorker.c @@ -188,6 +188,8 @@ static void vnodeProcessMWorkerMsg(SVMWorkerMsg *pMsg) { } static void *vnodeMWorkerFunc(void *param) { + setThreadName("vnodeMWorker"); + while (1) { SVMWorkerMsg *pMsg = NULL; if (taosReadQitemFromQset(tsVMWorkerQset, NULL, (void **)&pMsg, NULL) == 0) { diff --git a/src/wal/src/walMgmt.c b/src/wal/src/walMgmt.c index 9bd5cdf175..45f65b2c2f 100644 --- a/src/wal/src/walMgmt.c +++ b/src/wal/src/walMgmt.c @@ -192,6 +192,7 @@ static void walFsyncAll() { static void *walThreadFunc(void *param) { int stop = 0; + setThreadName("walThrd"); while (1) { walUpdateSeq(); walFsyncAll();