From 82317294ba82d4931e8f53037bad780153e6abc4 Mon Sep 17 00:00:00 2001 From: lihui Date: Thu, 9 Jan 2020 10:35:47 +0800 Subject: [PATCH 01/13] [#1102] --- src/client/inc/tsclient.h | 1 + src/client/src/tscParseInsert.c | 10 ++++++---- src/client/src/tscUtil.c | 1 + 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 6df78f6b7b..9116a7c60b 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -239,6 +239,7 @@ typedef struct STableDataBlocks { int32_t rowSize; // row size for current table uint32_t nAllocSize; + uint32_t headerSize; // header for metadata (submit metadata) uint32_t size; /* diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index f0824959b0..d9c86a955e 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -579,7 +579,8 @@ int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int3 size_t remain = pDataBlock->nAllocSize - pDataBlock->size; const int factor = 5; uint32_t nAllocSizeOld = pDataBlock->nAllocSize; - + assert(pDataBlock->headerSize >= 0); + // expand the allocated size if (remain < rowSize * factor) { while (remain < rowSize * factor) { @@ -595,12 +596,12 @@ int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int3 //assert(false); // do nothing pDataBlock->nAllocSize = nAllocSizeOld; - *numOfRows = (int32_t)(pDataBlock->nAllocSize) / rowSize; + *numOfRows = (int32_t)(pDataBlock->nAllocSize - pDataBlock->headerSize) / rowSize; return TSDB_CODE_CLI_OUT_OF_MEMORY; } } - *numOfRows = (int32_t)(pDataBlock->nAllocSize) / rowSize; + *numOfRows = (int32_t)(pDataBlock->nAllocSize - pDataBlock->headerSize) / rowSize; return TSDB_CODE_SUCCESS; } @@ -908,6 +909,7 @@ static int32_t tscParseSqlForCreateTableOnDemand(char **sqlstr, SSqlObj *pSql) { createTable = true; code = tscGetMeterMetaEx(pSql, pMeterMetaInfo->name, true); + if (TSDB_CODE_ACTION_IN_PROGRESS == code) return code; } else { if (cstart != NULL) { sql = cstart; @@ -1015,7 +1017,7 @@ int doParserInsertSql(SSqlObj *pSql, char *str) { tscTrace("async insert and waiting to get meter meta, then continue parse sql from offset: %" PRId64, pos); return code; } - tscTrace("async insert parse error, code:%d, %s", code, tsError[code]); + tscError("async insert parse error, code:%d, %s", code, tsError[code]); pSql->asyncTblPos = NULL; goto _error_clean; // TODO: should _clean or _error_clean to async flow ???? } else { diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index bc6c73aaae..5116b01418 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -598,6 +598,7 @@ int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOff } dataBuf->nAllocSize = (uint32_t)initialSize; + dataBuf->headerSize = startOffset; // the header size will always be the startOffset value, reserved for the subumit block header dataBuf->pData = calloc(1, dataBuf->nAllocSize); dataBuf->ordered = true; dataBuf->prevTS = INT64_MIN; From 4217bacf545d3cb58c48d19f611183f02cdfed00 Mon Sep 17 00:00:00 2001 From: lihui Date: Thu, 9 Jan 2020 10:41:21 +0800 Subject: [PATCH 02/13] [#1096] --- src/util/src/ttime.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/util/src/ttime.c b/src/util/src/ttime.c index 64a4d0fb9e..52faa0cac4 100644 --- a/src/util/src/ttime.c +++ b/src/util/src/ttime.c @@ -292,8 +292,8 @@ int32_t parseLocaltime(char* timestr, int64_t* time, int32_t timePrec) { } /* mktime will be affected by TZ, set by using taos_options */ - int64_t seconds = mktime(&tm); - //int64_t seconds = (int64_t)user_mktime(&tm); + //int64_t seconds = mktime(&tm); + int64_t seconds = (int64_t)user_mktime(&tm); int64_t fraction = 0; From 9e7f6703761e8664494ff7b4c1036cf71edc67b1 Mon Sep 17 00:00:00 2001 From: haojun Liao Date: Thu, 9 Jan 2020 14:11:26 +0800 Subject: [PATCH 03/13] fix bugs avoiding two super table join crash fix bugs avoiding two super table join crashed. --- src/system/detail/src/mgmtSupertableQuery.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/system/detail/src/mgmtSupertableQuery.c b/src/system/detail/src/mgmtSupertableQuery.c index f83ffd4247..1b7ae66e6d 100644 --- a/src/system/detail/src/mgmtSupertableQuery.c +++ b/src/system/detail/src/mgmtSupertableQuery.c @@ -329,7 +329,7 @@ int32_t mgmtDoJoin(SMetricMetaMsg* pMetricMetaMsg, tQueryResultset* pRes) { bool allEmpty = false; for (int32_t i = 0; i < pMetricMetaMsg->numOfMeters; ++i) { - if (pRes->num == 0) { // all results are empty if one of them is empty + if (pRes[i].num == 0) { // all results are empty if one of them is empty allEmpty = true; break; } From 308c4174f61bb09db48e66cb4ce0d9c6102a9f8d Mon Sep 17 00:00:00 2001 From: lihui Date: Thu, 9 Jan 2020 17:16:41 +0800 Subject: [PATCH 04/13] [dump sql] --- src/client/src/tscAsync.c | 2 +- src/client/src/tscSql.c | 2 +- src/client/src/tscUtil.c | 6 ++++-- src/inc/tlog.h | 5 ++++- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 9c543952b4..4b2093b722 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -95,7 +95,7 @@ void taos_query_a(TAOS *taos, const char *sqlstr, void (*fp)(void *, TAOS_RES *, pRes->numOfRows = 1; strtolower(pSql->sqlstr, sqlstr); - tscTrace("%p Async SQL: %s, pObj:%p", pSql, pSql->sqlstr, pObj); + tscDump("%p pObj:%p, Async SQL: %s", pSql, pObj, pSql->sqlstr); int32_t code = tsParseSql(pSql, pObj->acctId, pObj->db, true); if (code == TSDB_CODE_ACTION_IN_PROGRESS) return; diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index c9d9050a29..13071d284e 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -207,7 +207,7 @@ int taos_query_imp(STscObj *pObj, SSqlObj *pSql) { pSql->pTableHashList = NULL; } - tscTrace("%p SQL: %s pObj:%p", pSql, pSql->sqlstr, pObj); + tscDump("%p pObj:%p, SQL: %s", pSql, pObj, pSql->sqlstr); pRes->code = (uint8_t)tsParseSql(pSql, pObj->acctId, pObj->db, false); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 5116b01418..02d0d9c6b0 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -692,8 +692,10 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SDataBlockList* pTableDataBlockLi SShellSubmitBlock* pBlocks = (SShellSubmitBlock*)pOneTableBlock->pData; sortRemoveDuplicates(pOneTableBlock); - tscTrace("%p meterId:%s, sid:%d, rows:%d, sversion:%d", pSql, pOneTableBlock->meterId, pBlocks->sid, - pBlocks->numOfRows, pBlocks->sversion); + char* e = (char*)pBlocks->payLoad + pOneTableBlock->rowSize*(pBlocks->numOfRows-1); + + tscTrace("%p meterId:%s, sid:%d rows:%d sversion:%d skey:%" PRId64 ", ekey:%" PRId64, pSql, pOneTableBlock->meterId, pBlocks->sid, + pBlocks->numOfRows, pBlocks->sversion, GET_INT64_VAL(pBlocks->payLoad), GET_INT64_VAL(e)); pBlocks->sid = htonl(pBlocks->sid); pBlocks->uid = htobe64(pBlocks->uid); diff --git a/src/inc/tlog.h b/src/inc/tlog.h index 0d348c27ce..7556cc50a1 100644 --- a/src/inc/tlog.h +++ b/src/inc/tlog.h @@ -113,7 +113,10 @@ extern uint32_t cdebugFlag; } #define tscPrint(...) \ { tprintf("TSC ", 255, __VA_ARGS__); } - +#define tscDump(...) \ + if (cdebugFlag & DEBUG_TRACE) { \ + taosPrintLongString("TSC ", cdebugFlag, __VA_ARGS__); \ + } #define jniError(...) \ if (jnidebugFlag & DEBUG_ERROR) { \ tprintf("ERROR JNI ", jnidebugFlag, __VA_ARGS__); \ From fc03f90661a589c9c59171e7491a13dd0a645b27 Mon Sep 17 00:00:00 2001 From: haojun Liao Date: Thu, 9 Jan 2020 21:34:26 +0800 Subject: [PATCH 05/13] fix memory leaks in client fix memory leaks in client --- src/client/src/tscSecondaryMerge.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/client/src/tscSecondaryMerge.c b/src/client/src/tscSecondaryMerge.c index 737c1342d8..ae41f739ee 100644 --- a/src/client/src/tscSecondaryMerge.c +++ b/src/client/src/tscSecondaryMerge.c @@ -447,6 +447,10 @@ void tscDestroyLocalReducer(SSqlObj *pSql) { for(int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { SQLFunctionCtx *pCtx = &pLocalReducer->pCtx[i]; tVariantDestroy(&pCtx->tag); + + if (pCtx->tagInfo.pTagCtxList != NULL) { + tfree(pCtx->tagInfo.pTagCtxList); + } } tfree(pLocalReducer->pCtx); From ecd07ad9bdb410fb2e5d420b0fa9ac85bd62e3ed Mon Sep 17 00:00:00 2001 From: lihui Date: Fri, 10 Jan 2020 15:26:14 +0800 Subject: [PATCH 06/13] [modify for solving warn] --- src/client/src/tscAst.c | 2 +- src/client/src/tscJoinProcess.c | 4 +++- src/client/src/tscProfile.c | 5 +++-- src/client/src/tscSQLParser.c | 2 +- src/client/src/tscSQLParserImpl.c | 4 ++-- src/client/src/tscSecondaryMerge.c | 4 +++- src/client/src/tscServer.c | 9 +++++++-- src/client/src/tscStream.c | 6 +++--- src/os/linux/src/tsystem.c | 3 ++- src/rpc/src/tudp.c | 2 +- src/system/detail/src/vnodeFile.c | 2 +- src/util/src/tbase64.c | 2 +- src/util/src/textbuffer.c | 10 ++++++---- src/util/src/ttypes.c | 2 +- 14 files changed, 35 insertions(+), 22 deletions(-) diff --git a/src/client/src/tscAst.c b/src/client/src/tscAst.c index d071358dbf..03d474a4f2 100644 --- a/src/client/src/tscAst.c +++ b/src/client/src/tscAst.c @@ -833,7 +833,7 @@ void tSQLBinaryExprCalcTraverse(tSQLBinaryExpr *pExprs, int32_t numOfRows, char tSQLSyntaxNode *pRight = pExprs->pRight; /* the left output has result from the left child syntax tree */ - char *pLeftOutput = malloc(sizeof(int64_t) * numOfRows); + char *pLeftOutput = (char*)malloc(sizeof(int64_t) * numOfRows); if (pLeft->nodeType == TSQL_NODE_EXPR) { tSQLBinaryExprCalcTraverse(pLeft->pExpr, numOfRows, pLeftOutput, param, order, getSourceDataBlock); } diff --git a/src/client/src/tscJoinProcess.c b/src/client/src/tscJoinProcess.c index a94b308e87..d49d6d7512 100644 --- a/src/client/src/tscJoinProcess.c +++ b/src/client/src/tscJoinProcess.c @@ -793,7 +793,9 @@ STSBuf* tsBufCreate(bool autoDelete) { return NULL; } - allocResForTSBuf(pTSBuf); + if (NULL == allocResForTSBuf(pTSBuf)) { + return NULL; + } // update the header info STSBufFileHeader header = {.magic = TS_COMP_FILE_MAGIC, .numOfVnode = pTSBuf->numOfVnodes, .tsOrder = TSQL_SO_ASC}; diff --git a/src/client/src/tscProfile.c b/src/client/src/tscProfile.c index f5925b61cd..a7a774b3a8 100644 --- a/src/client/src/tscProfile.c +++ b/src/client/src/tscProfile.c @@ -202,10 +202,10 @@ void tscKillStream(STscObj *pObj, uint32_t killId) { tscTrace("%p stream:%p is killed, streamId:%d", pStream->pSql, pStream, killId); } - taos_close_stream(pStream); if (pStream->callback) { pStream->callback(pStream->param); } + taos_close_stream(pStream); } char *tscBuildQueryStreamDesc(char *pMsg, STscObj *pObj) { @@ -285,8 +285,9 @@ void tscKillConnection(STscObj *pObj) { SSqlStream *pStream = pObj->streamList; while (pStream) { + SSqlStream *tmp = pStream->next; taos_close_stream(pStream); - pStream = pStream->next; + pStream = tmp; } pthread_mutex_unlock(&pObj->mutex); diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 422ae707ad..df6afde1a4 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -2918,7 +2918,7 @@ static SColumnFilterInfo* addColumnFilterInfo(SColumnBase* pColumn) { } int32_t size = pColumn->numOfFilters + 1; - char* tmp = realloc(pColumn->filterInfo, sizeof(SColumnFilterInfo) * (size)); + char* tmp = (char*)realloc((void*)(pColumn->filterInfo), sizeof(SColumnFilterInfo) * (size)); if (tmp != NULL) { pColumn->filterInfo = (SColumnFilterInfo*)tmp; } diff --git a/src/client/src/tscSQLParserImpl.c b/src/client/src/tscSQLParserImpl.c index cc4375fb03..f7423f3a73 100644 --- a/src/client/src/tscSQLParserImpl.c +++ b/src/client/src/tscSQLParserImpl.c @@ -706,14 +706,14 @@ void setDCLSQLElems(SSqlInfo *pInfo, int32_t type, int32_t nParam, ...) { pInfo->sqlType = type; if (nParam == 0) return; - if (pInfo->pDCLInfo == NULL) pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL)); + if (pInfo->pDCLInfo == NULL) pInfo->pDCLInfo = (tDCLSQL *)calloc(1, sizeof(tDCLSQL)); va_list va; va_start(va, nParam); while (nParam-- > 0) { SSQLToken *pToken = va_arg(va, SSQLToken *); - tTokenListAppend(pInfo->pDCLInfo, pToken); + (void)tTokenListAppend(pInfo->pDCLInfo, pToken); } va_end(va); } diff --git a/src/client/src/tscSecondaryMerge.c b/src/client/src/tscSecondaryMerge.c index 737c1342d8..dc2bdae390 100644 --- a/src/client/src/tscSecondaryMerge.c +++ b/src/client/src/tscSecondaryMerge.c @@ -598,7 +598,9 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr rlen += pExpr->resBytes; } - int32_t capacity = nBufferSizes / rlen; + int32_t capacity = 0; + if (0 != rlen) capacity = nBufferSizes / rlen; + pModel = tColModelCreate(pSchema, pCmd->fieldsInfo.numOfOutputCols, capacity); for (int32_t i = 0; i < pMeterMetaInfo->pMetricMeta->numOfVnodes; ++i) { diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index bdac9185eb..20893a388b 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1380,7 +1380,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { SSqlObj *pNew = tscCreateSqlObjForSubquery(trsupport->pParentSqlObj, trsupport, pSql); if (pNew == NULL) { tscError("%p sub:%p failed to create new subquery due to out of memory, abort retry, vid:%d, orderOfSub:%d", - trsupport->pParentSqlObj, pSql, pSvd->vnode, trsupport->subqueryIndex); + trsupport->pParentSqlObj, pSql, pSvd != NULL ? pSvd->vnode : 10000, trsupport->subqueryIndex); pState->code = -TSDB_CODE_CLI_OUT_OF_MEMORY; trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; @@ -1404,9 +1404,14 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { tscRetrieveFromVnodeCallBack(param, tres, pState->code); } else { // success, proceed to retrieve data from dnode - tscTrace("%p sub:%p query complete,ip:%u,vid:%d,orderOfSub:%d,retrieve data", trsupport->pParentSqlObj, pSql, + if (vnodeInfo != NULL) { + tscTrace("%p sub:%p query complete,ip:%u,vid:%d,orderOfSub:%d,retrieve data", trsupport->pParentSqlObj, pSql, vnodeInfo->vpeerDesc[vnodeInfo->index].ip, vnodeInfo->vpeerDesc[vnodeInfo->index].vnode, trsupport->subqueryIndex); + } else { + tscTrace("%p sub:%p query complete, orderOfSub:%d,retrieve data", trsupport->pParentSqlObj, pSql, + trsupport->subqueryIndex); + } taos_fetch_rows_a(tres, tscRetrieveFromVnodeCallBack, param); } diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index cd2736e910..fd102510c3 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -268,11 +268,11 @@ static void tscSetRetryTimer(SSqlStream *pStream, SSqlObj *pSql, int64_t timer) tscTrace("%p stream:%p, etime:%" PRId64 " is too old, exceeds the max retention time window:%" PRId64 ", stop the stream", pStream->pSql, pStream, pStream->stime, pStream->etime); // TODO : How to terminate stream here - taos_close_stream(pStream); if (pStream->callback) { // Callback function from upper level pStream->callback(pStream->param); } + taos_close_stream(pStream); return; } @@ -302,11 +302,11 @@ static void tscSetNextLaunchTimer(SSqlStream *pStream, SSqlObj *pSql) { tscTrace("%p stream:%p, stime:%" PRId64 " is larger than end time: %" PRId64 ", stop the stream", pStream->pSql, pStream, pStream->stime, pStream->etime); // TODO : How to terminate stream here - taos_close_stream(pStream); if (pStream->callback) { // Callback function from upper level pStream->callback(pStream->param); } + taos_close_stream(pStream); return; } } else { @@ -315,11 +315,11 @@ static void tscSetNextLaunchTimer(SSqlStream *pStream, SSqlObj *pSql) { tscTrace("%p stream:%p, stime:%ld is larger than end time: %ld, stop the stream", pStream->pSql, pStream, pStream->stime, pStream->etime); // TODO : How to terminate stream here - taos_close_stream(pStream); if (pStream->callback) { // Callback function from upper level pStream->callback(pStream->param); } + taos_close_stream(pStream); return; } diff --git a/src/os/linux/src/tsystem.c b/src/os/linux/src/tsystem.c index c3b8b41c9d..8cd0e69436 100644 --- a/src/os/linux/src/tsystem.c +++ b/src/os/linux/src/tsystem.c @@ -517,7 +517,8 @@ bool taosGetProcIO(float *readKB, float *writeKB) { static int64_t lastReadbyte = -1; static int64_t lastWritebyte = -1; - int64_t curReadbyte, curWritebyte; + int64_t curReadbyte = 0; + int64_t curWritebyte = 0; if (!taosReadProcIO(&curReadbyte, &curWritebyte)) { return false; diff --git a/src/rpc/src/tudp.c b/src/rpc/src/tudp.c index fb0b37d93b..4324f5a68a 100644 --- a/src/rpc/src/tudp.c +++ b/src/rpc/src/tudp.c @@ -703,7 +703,7 @@ int taosSendPacketViaTcp(uint32_t ip, uint16_t port, char *data, int dataLen, vo pHead->msgLen = (int32_t)htonl(msgLen); code = taosSendUdpData(ip, port, buffer, msgLen, chandle); - pHead = (STaosHeader *)data; + //pHead = (STaosHeader *)data; tinet_ntoa(ipstr, ip); int fd = taosOpenTcpClientSocket(ipstr, pConn->port, tsLocalIp); diff --git a/src/system/detail/src/vnodeFile.c b/src/system/detail/src/vnodeFile.c index b69320a03c..8074db0c80 100644 --- a/src/system/detail/src/vnodeFile.c +++ b/src/system/detail/src/vnodeFile.c @@ -1383,7 +1383,7 @@ int vnodeSearchPointInFile(SMeterObj *pObj, SQuery *pQuery) { firstSlot = 0; lastSlot = pQuery->numOfBlocks - 1; - numOfBlocks = pQuery->numOfBlocks; + //numOfBlocks = pQuery->numOfBlocks; if (QUERY_IS_ASC_QUERY(pQuery) && pBlock[lastSlot].keyLast < pQuery->skey) continue; if (!QUERY_IS_ASC_QUERY(pQuery) && pBlock[firstSlot].keyFirst > pQuery->skey) continue; diff --git a/src/util/src/tbase64.c b/src/util/src/tbase64.c index 02ec756e04..937adfde5c 100644 --- a/src/util/src/tbase64.c +++ b/src/util/src/tbase64.c @@ -98,7 +98,7 @@ unsigned char *base64_decode(const char *value, int inlen, int *outlen) { base64_decode_error: free(result); - *result = 0; + result = 0; *outlen = 0; return result; diff --git a/src/util/src/textbuffer.c b/src/util/src/textbuffer.c index 3e71d90147..5c4ab1e74d 100644 --- a/src/util/src/textbuffer.c +++ b/src/util/src/textbuffer.c @@ -516,20 +516,20 @@ tMemBucket* tMemBucketCreate(int32_t totalSlots, int32_t nBufferSize, int16_t nE if (pDesc->pSchema->numOfCols != 1 || pDesc->pSchema->colOffset[0] != 0) { pError("MemBucket:%p,only consecutive data is allowed,invalid numOfCols:%d or offset:%d", - *pBucket, pDesc->pSchema->numOfCols, pDesc->pSchema->colOffset[0]); + pBucket, pDesc->pSchema->numOfCols, pDesc->pSchema->colOffset[0]); tfree(pBucket); return NULL; } if (pDesc->pSchema->pFields[0].type != dataType) { - pError("MemBucket:%p,data type is not consistent,%d in schema, %d in param", *pBucket, + pError("MemBucket:%p,data type is not consistent,%d in schema, %d in param", pBucket, pDesc->pSchema->pFields[0].type, dataType); tfree(pBucket); return NULL; } if (pBucket->numOfTotalPages < pBucket->nTotalSlots) { - pWarn("MemBucket:%p,total buffer pages %d are not enough for all slots", *pBucket, pBucket->numOfTotalPages); + pWarn("MemBucket:%p,total buffer pages %d are not enough for all slots", pBucket, pBucket->numOfTotalPages); } pBucket->pSegs = (tMemBucketSegment *)malloc(pBucket->numOfSegs * sizeof(tMemBucketSegment)); @@ -540,7 +540,7 @@ tMemBucket* tMemBucketCreate(int32_t totalSlots, int32_t nBufferSize, int16_t nE pBucket->pSegs[i].pBoundingEntries = NULL; } - pTrace("MemBucket:%p,created,buffer size:%d,elem size:%d", *pBucket, pBucket->numOfTotalPages * DEFAULT_PAGE_SIZE, + pTrace("MemBucket:%p,created,buffer size:%d,elem size:%d", pBucket, pBucket->numOfTotalPages * DEFAULT_PAGE_SIZE, pBucket->nElemSize); return pBucket; @@ -1258,6 +1258,7 @@ static tFilePage *loadIntoBucketFromDisk(tMemBucket *pMemBucket, int32_t segIdx, for (uint32_t j = 0; j < pFlushInfo->numOfPages; ++j) { ret = fread(pPage, pMemBuffer->nPageSize, 1, pMemBuffer->dataFile); + UNUSED(ret); assert(pPage->numOfElems > 0); tColModelAppend(pDesc->pSchema, buffer, pPage->data, 0, pPage->numOfElems, pPage->numOfElems); @@ -1917,6 +1918,7 @@ double getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction) for (uint32_t jx = 0; jx < pFlushInfo->numOfPages; ++jx) { ret = fread(pPage, pMemBuffer->nPageSize, 1, pMemBuffer->dataFile); + UNUSED(ret); tMemBucketPut(pMemBucket, pPage->data, pPage->numOfElems); } diff --git a/src/util/src/ttypes.c b/src/util/src/ttypes.c index b048748d95..60712a5d06 100644 --- a/src/util/src/ttypes.c +++ b/src/util/src/ttypes.c @@ -726,7 +726,7 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, char type) { *((int64_t *)payload) = TSDB_DATA_DOUBLE_NULL; return 0; } else { - double value; + double value = 0; int32_t ret; ret = convertToDouble(pVariant->pz, pVariant->nLen, &value); if ((errno == ERANGE && value == -1) || (ret != 0)) { From a3ac982150e3298a4cb2d076de30ef92ab61d6bd Mon Sep 17 00:00:00 2001 From: lihui Date: Fri, 10 Jan 2020 15:33:48 +0800 Subject: [PATCH 07/13] [modify for solving warn] --- src/client/src/tscServer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 20893a388b..3780eafb20 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1380,7 +1380,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { SSqlObj *pNew = tscCreateSqlObjForSubquery(trsupport->pParentSqlObj, trsupport, pSql); if (pNew == NULL) { tscError("%p sub:%p failed to create new subquery due to out of memory, abort retry, vid:%d, orderOfSub:%d", - trsupport->pParentSqlObj, pSql, pSvd != NULL ? pSvd->vnode : 10000, trsupport->subqueryIndex); + trsupport->pParentSqlObj, pSql, pSvd != NULL ? pSvd->vnode : -1, trsupport->subqueryIndex); pState->code = -TSDB_CODE_CLI_OUT_OF_MEMORY; trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; From 36d8000c8ed5a8a231d76ae2d82e4fb3d3def947 Mon Sep 17 00:00:00 2001 From: lihui Date: Fri, 10 Jan 2020 16:40:55 +0800 Subject: [PATCH 08/13] [Go dll for windows] --- .../webdocs/markdowndocs/connector-ch.md | 57 +++++++++++++------ 1 file changed, 41 insertions(+), 16 deletions(-) diff --git a/documentation/webdocs/markdowndocs/connector-ch.md b/documentation/webdocs/markdowndocs/connector-ch.md index 3ae47910d7..dd81ce8aa5 100644 --- a/documentation/webdocs/markdowndocs/connector-ch.md +++ b/documentation/webdocs/markdowndocs/connector-ch.md @@ -210,26 +210,26 @@ TDengine 的 JDBC 驱动实现尽可能的与关系型数据库驱动保持一 ## TAOS-JDBCDriver 版本以及支持的 TDengine 版本和 JDK 版本 -| taos-jdbcdriver 版本 | TDengine 版本 | JDK 版本 | -| --- | --- | --- | +| taos-jdbcdriver 版本 | TDengine 版本 | JDK 版本 | +| --- | --- | --- | | 1.0.3 | 1.6.1.x 及以上 | 1.8.x | -| 1.0.2 | 1.6.1.x 及以上 | 1.8.x | -| 1.0.1 | 1.6.1.x 及以上 | 1.8.x | +| 1.0.2 | 1.6.1.x 及以上 | 1.8.x | +| 1.0.1 | 1.6.1.x 及以上 | 1.8.x | ## TDengine DataType 和 Java DataType TDengine 目前支持时间戳、数字、字符、布尔类型，与 Java 对应类型转换如下： -| TDengine DataType | Java DataType | -| --- | --- | -| TIMESTAMP | java.sql.Timestamp | -| INT | java.lang.Integer | -| BIGINT | java.lang.Long | -| FLOAT | java.lang.Float | -| DOUBLE | java.lang.Double | +| TDengine DataType | Java DataType | +| --- | --- | +| TIMESTAMP | java.sql.Timestamp | +| INT | java.lang.Integer | +| BIGINT | java.lang.Long | +| FLOAT | java.lang.Float | +| DOUBLE | java.lang.Double | | SMALLINT, TINYINT |java.lang.Short | -| BOOL | java.lang.Boolean | -| BINARY, NCHAR | java.lang.String | +| BOOL | java.lang.Boolean | +| BINARY, NCHAR | java.lang.String | ## 如何获取 TAOS-JDBCDriver @@ -807,6 +807,8 @@ HTTP请求URL采用`sqlutc`时，返回结果集的时间戳将采用UTC时间 ## Go Connector +### linux环境 + #### 安装TDengine Go的连接器使用到了 libtaos.so 和taos.h，因此，在使用Go连接器之前，需要在程序运行的机器上安装TDengine以获得相关的驱动文件。 @@ -867,7 +869,15 @@ taosSql驱动包内采用cgo模式，调用了TDengine的C/C++同步接口，与 3. 创建表、写入和查询数据 -在创建好了数据库后，就可以开始创建表和写入查询数据了。这些操作的基本思路都是首先组装SQL语句，然后调用db.Exec执行，并检查错误信息和执行相应的处理。可以参考上面的样例代码 +在创建好了数据库后，就可以开始创建表和写入查询数据了。这些操作的基本思路都是首先组装SQL语句，然后调用db.Exec执行，并检查错误信息和执行相应的处理。可以参考上面的样例代码。 + +### windows环境 + +在windows上使用Go，请参考  + +[TDengine GO windows驱动的编译和使用]: https://www.taosdata.com/blog/2020/01/06/tdengine-go-windows%E9%A9%B1%E5%8A%A8%E7%9A%84%E7%BC%96%E8%AF%91/ + + ## Node.js Connector @@ -1054,6 +1064,8 @@ https://gitee.com/maikebing/Maikebing.EntityFrameworkCore.Taos ├───├── jdbc ├───└── python ├── driver +├───├── libtaos.dll +├───├── libtaos.dll.a ├───├── taos.dll ├───├── taos.exp ├───└── taos.lib @@ -1078,8 +1090,8 @@ https://gitee.com/maikebing/Maikebing.EntityFrameworkCore.Taos + Client可执行文件: C:/TDengine/taos.exe + 配置文件: C:/TDengine/cfg/taos.cfg -+ C驱动程序目录: C:/TDengine/driver -+ C驱动程序头文件: C:/TDengine/include ++ 驱动程序目录: C:/TDengine/driver ++ 驱动程序头文件: C:/TDengine/include + JDBC驱动程序目录: C:/TDengine/connector/jdbc + GO驱动程序目录：C:/TDengine/connector/go + Python驱动程序目录：C:/TDengine/connector/python @@ -1106,6 +1118,16 @@ taos -h TDengine在Window系统上提供的API与Linux系统是相同的， 应用程序使用时，需要包含TDengine头文件taos.h，连接时需要链接TDengine库taos.lib，运行时将taos.dll放到可执行文件目录下。 +#### Go接口注意事项 + +TDengine在Window系统上提供的API与Linux系统是相同的， 应用程序使用时，除了需要Go的驱动包（C:\TDengine\connector\go）外，还需要包含TDengine头文件taos.h，连接时需要链接TDengine库libtaos.dll、libtaos.dll.a（C:\TDengine\driver），运行时将libtaos.dll、libtaos.dll.a放到可执行文件目录下。 + +使用参考请见： + +[TDengine GO windows驱动的编译和使用]: https://www.taosdata.com/blog/2020/01/06/tdengine-go-windows%E9%A9%B1%E5%8A%A8%E7%9A%84%E7%BC%96%E8%AF%91/ + + + #### JDBC接口注意事项 在Windows系统上，应用程序可以使用JDBC接口来操纵数据库，使用JDBC接口的注意事项如下： @@ -1137,3 +1159,6 @@ TDengine在Window系统上提供的API与Linux系统是相同的， 应用程序 [13]: https://www.taosdata.com/cn/documentation/administrator/#%E5%AE%A2%E6%88%B7%E7%AB%AF%E9%85%8D%E7%BD%AE [14]: https://www.taosdata.com/cn/documentation/connector/#Windows%E5%AE%A2%E6%88%B7%E7%AB%AF%E5%8F%8A%E7%A8%8B%E5%BA%8F%E6%8E%A5%E5%8F%A3 [15]: https://www.taosdata.com/cn/getting-started/#%E5%BF%AB%E9%80%9F%E4%B8%8A%E6%89%8B + +[16]: https://www.taosdata.com/blog/2020/01/06/tdengine-go-windows%E9%A9%B1%E5%8A%A8%E7%9A%84%E7%BC%96%E8%AF%91/ + From 69e1a78a467555083a5fa12964ee6233da6bbaba Mon Sep 17 00:00:00 2001 From: lihui Date: Fri, 10 Jan 2020 16:44:31 +0800 Subject: [PATCH 09/13] [Go dll for windows] --- documentation/webdocs/markdowndocs/connector-ch.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/documentation/webdocs/markdowndocs/connector-ch.md b/documentation/webdocs/markdowndocs/connector-ch.md index dd81ce8aa5..8aaacd418b 100644 --- a/documentation/webdocs/markdowndocs/connector-ch.md +++ b/documentation/webdocs/markdowndocs/connector-ch.md @@ -1159,6 +1159,3 @@ TDengine在Window系统上提供的API与Linux系统是相同的， 应用程序 [13]: https://www.taosdata.com/cn/documentation/administrator/#%E5%AE%A2%E6%88%B7%E7%AB%AF%E9%85%8D%E7%BD%AE [14]: https://www.taosdata.com/cn/documentation/connector/#Windows%E5%AE%A2%E6%88%B7%E7%AB%AF%E5%8F%8A%E7%A8%8B%E5%BA%8F%E6%8E%A5%E5%8F%A3 [15]: https://www.taosdata.com/cn/getting-started/#%E5%BF%AB%E9%80%9F%E4%B8%8A%E6%89%8B - -[16]: https://www.taosdata.com/blog/2020/01/06/tdengine-go-windows%E9%A9%B1%E5%8A%A8%E7%9A%84%E7%BC%96%E8%AF%91/ - From 071c342f0befcb20c514f2282e6aedd5d017ca2c Mon Sep 17 00:00:00 2001 From: fang Date: Fri, 10 Jan 2020 17:01:59 +0800 Subject: [PATCH 10/13] user guide for go connector, edit the html link --- documentation/webdocs/markdowndocs/connector-ch.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/documentation/webdocs/markdowndocs/connector-ch.md b/documentation/webdocs/markdowndocs/connector-ch.md index 8aaacd418b..4ee49a7029 100644 --- a/documentation/webdocs/markdowndocs/connector-ch.md +++ b/documentation/webdocs/markdowndocs/connector-ch.md @@ -874,8 +874,7 @@ taosSql驱动包内采用cgo模式，调用了TDengine的C/C++同步接口，与 ### windows环境 在windows上使用Go，请参考  - -[TDengine GO windows驱动的编译和使用]: https://www.taosdata.com/blog/2020/01/06/tdengine-go-windows%E9%A9%B1%E5%8A%A8%E7%9A%84%E7%BC%96%E8%AF%91/ +[TDengine GO windows驱动的编译和使用](https://www.taosdata.com/blog/2020/01/06/tdengine-go-windows%E9%A9%B1%E5%8A%A8%E7%9A%84%E7%BC%96%E8%AF%91/) From efd8a1708861019635f64a2d43f8cbf3b7d095c3 Mon Sep 17 00:00:00 2001 From: fang Date: Fri, 10 Jan 2020 17:44:56 +0800 Subject: [PATCH 11/13] user guide for go connector, edit the html link --- documentation/webdocs/markdowndocs/connector-ch.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/webdocs/markdowndocs/connector-ch.md b/documentation/webdocs/markdowndocs/connector-ch.md index 4ee49a7029..8c26da0d45 100644 --- a/documentation/webdocs/markdowndocs/connector-ch.md +++ b/documentation/webdocs/markdowndocs/connector-ch.md @@ -1123,7 +1123,7 @@ TDengine在Window系统上提供的API与Linux系统是相同的， 应用程序 使用参考请见： -[TDengine GO windows驱动的编译和使用]: https://www.taosdata.com/blog/2020/01/06/tdengine-go-windows%E9%A9%B1%E5%8A%A8%E7%9A%84%E7%BC%96%E8%AF%91/ +[TDengine GO windows驱动的编译和使用](https://www.taosdata.com/blog/2020/01/06/tdengine-go-windows%E9%A9%B1%E5%8A%A8%E7%9A%84%E7%BC%96%E8%AF%91/) From 3a9c82899c756ba5de6bbd644d2702faaf7c8ea0 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 13 Jan 2020 11:42:16 +0800 Subject: [PATCH 12/13] Fix #1124 TBASE-1483 --- src/client/src/tscServer.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index bdac9185eb..f741fc8620 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1101,8 +1101,10 @@ static void tscHandleSubRetrievalError(SRetrieveSupport *trsupport, SSqlObj *pSq } } + int32_t numOfTotal = pState->numOfTotal; + int32_t finished = atomic_add_fetch_32(&pState->numOfCompleted, 1); - if (finished < pState->numOfTotal) { + if (finished < numOfTotal) { tscTrace("%p sub:%p orderOfSub:%d freed, finished subqueries:%d", pPObj, pSql, trsupport->subqueryIndex, finished); return tscFreeSubSqlObj(trsupport, pSql); } @@ -1231,8 +1233,13 @@ void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfRows) { return tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_CLI_NO_DISKSPACE); } + // keep this value local variable, since the pState variable may be released by other threads, if atomic_add opertion + // increases the finished value up to pState->numOfTotal value, which means all subqueries are completed. + // In this case, the comparsion between finished value and released pState->numOfTotal is not safe. + int32_t numOfTotal = pState->numOfTotal; + int32_t finished = atomic_add_fetch_32(&pState->numOfCompleted, 1); - if (finished < pState->numOfTotal) { + if (finished < numOfTotal) { tscTrace("%p sub:%p orderOfSub:%d freed, finished subqueries:%d", pPObj, pSql, trsupport->subqueryIndex, finished); return tscFreeSubSqlObj(trsupport, pSql); } @@ -1241,7 +1248,7 @@ void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfRows) { pDesc->pSchema->maxCapacity = trsupport->pExtMemBuffer[idx]->numOfElemsPerPage; tscTrace("%p retrieve from %d vnodes completed.final NumOfRows:%d,start to build loser tree", pPObj, - pState->numOfTotal, pState->numOfCompleted); + pState->numOfTotal, pState->numOfRetrievedRows); tscClearInterpInfo(&pPObj->cmd); tscCreateLocalReducer(trsupport->pExtMemBuffer, pState->numOfTotal, pDesc, trsupport->pFinalColModel, From 1c943a71fda7ed39abeefa82f78a8cedf827e297 Mon Sep 17 00:00:00 2001 From: xieyinglin Date: Mon, 13 Jan 2020 21:52:48 +0800 Subject: [PATCH 13/13] edit plugin id --- src/connector/grafana/tdengine/dist/plugin.json | 6 +++--- src/connector/grafana/tdengine/package.json | 5 ++--- src/connector/grafana/tdengine/src/plugin.json | 6 +++--- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/connector/grafana/tdengine/dist/plugin.json b/src/connector/grafana/tdengine/dist/plugin.json index 3734fbfc2d..e9954ce6ce 100644 --- a/src/connector/grafana/tdengine/dist/plugin.json +++ b/src/connector/grafana/tdengine/dist/plugin.json @@ -1,6 +1,6 @@ { "name": "TDengine", - "id": "tdengine", + "id": "taosdata-tdengine-datasource", "type": "datasource", "partials": { @@ -24,8 +24,8 @@ {"name": "GitHub", "url": "https://github.com/taosdata/TDengine/tree/develop/src/connector/grafana/tdengine"}, {"name": "AGPL 3.0", "url": "https://github.com/taosdata/TDengine/tree/develop/src/connector/grafana/tdengine/LICENSE"} ], - "version": "1.6.0", - "updated": "2019-11-12" + "version": "1.0.0", + "updated": "2020-01-13" }, "dependencies": { diff --git a/src/connector/grafana/tdengine/package.json b/src/connector/grafana/tdengine/package.json index 83d29b78ce..8e542bef26 100644 --- a/src/connector/grafana/tdengine/package.json +++ b/src/connector/grafana/tdengine/package.json @@ -1,9 +1,8 @@ { "name": "TDengine", - "private": true, + "private": false, "version": "1.0.0", "description": "grafana datasource plugin for tdengine", - "main": "index.js", "scripts": { "build": "./node_modules/grunt-cli/bin/grunt", "test": "./node_modules/grunt-cli/bin/grunt mochaTest" @@ -12,7 +11,7 @@ "type": "git", "url": "git+https://github.com/taosdata/TDengine.git" }, - "author": "", + "author": "https://www.taosdata.com", "license": "AGPL 3.0", "bugs": { "url": "https://github.com/taosdata/TDengine/issues" diff --git a/src/connector/grafana/tdengine/src/plugin.json b/src/connector/grafana/tdengine/src/plugin.json index 3734fbfc2d..e9954ce6ce 100644 --- a/src/connector/grafana/tdengine/src/plugin.json +++ b/src/connector/grafana/tdengine/src/plugin.json @@ -1,6 +1,6 @@ { "name": "TDengine", - "id": "tdengine", + "id": "taosdata-tdengine-datasource", "type": "datasource", "partials": { @@ -24,8 +24,8 @@ {"name": "GitHub", "url": "https://github.com/taosdata/TDengine/tree/develop/src/connector/grafana/tdengine"}, {"name": "AGPL 3.0", "url": "https://github.com/taosdata/TDengine/tree/develop/src/connector/grafana/tdengine/LICENSE"} ], - "version": "1.6.0", - "updated": "2019-11-12" + "version": "1.0.0", + "updated": "2020-01-13" }, "dependencies": {