From d34cabf2c1a782fe7001e3f82699aadfd6f3911c Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Fri, 29 May 2020 17:15:09 +0800 Subject: [PATCH 01/22] TD-449: fix several bugs --- src/cq/src/cqMain.c | 6 ++++-- src/dnode/inc/dnodeModule.h | 3 ++- src/dnode/src/dnodeMain.c | 1 + src/dnode/src/dnodeMgmt.c | 18 +++++++++++++++++- src/inc/tcq.h | 1 + src/inc/tsdb.h | 1 + src/inc/vnode.h | 1 + src/tsdb/src/tsdbMain.c | 12 ++++++++++++ src/tsdb/src/tsdbMeta.c | 8 -------- src/vnode/src/vnodeMain.c | 12 +++++++++++- 10 files changed, 50 insertions(+), 13 deletions(-) diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c index 9406a2fdce..799a0b0514 100644 --- a/src/cq/src/cqMain.c +++ b/src/cq/src/cqMain.c @@ -38,6 +38,7 @@ typedef struct { int vgId; char user[TSDB_USER_LEN]; char pass[TSDB_PASSWORD_LEN]; + char db[TSDB_DB_NAME_LEN]; FCqWrite cqWrite; void *ahandle; int num; // number of continuous streams @@ -73,6 +74,7 @@ void *cqOpen(void *ahandle, const SCqCfg *pCfg) { strcpy(pContext->user, pCfg->user); strcpy(pContext->pass, pCfg->pass); + strcpy(pContext->db, pCfg->db); pContext->vgId = pCfg->vgId; pContext->cqWrite = pCfg->cqWrite; pContext->ahandle = ahandle; @@ -207,9 +209,8 @@ void cqDrop(void *handle) { } static void cqCreateStream(SCqContext *pContext, SCqObj *pObj) { - if (pContext->dbConn == NULL) { - pContext->dbConn = taos_connect("localhost", pContext->user, pContext->pass, NULL, 0); + pContext->dbConn = taos_connect("localhost", pContext->user, pContext->pass, pContext->db, 0); if (pContext->dbConn == NULL) { cError("vgId:%d, failed to connect to TDengine(%s)", pContext->vgId, tstrerror(terrno)); } @@ -217,6 +218,7 @@ static void cqCreateStream(SCqContext *pContext, SCqObj *pObj) { } int64_t lastKey = 0; + pObj->pContext = pContext; pObj->pStream = taos_open_stream(pContext->dbConn, pObj->sqlStr, cqProcessStreamRes, lastKey, pObj, NULL); if (pObj->pStream) { pContext->num++; diff --git a/src/dnode/inc/dnodeModule.h b/src/dnode/inc/dnodeModule.h index 8618de3244..fb529ee67c 100644 --- a/src/dnode/inc/dnodeModule.h +++ b/src/dnode/inc/dnodeModule.h @@ -22,7 +22,8 @@ extern "C" { int32_t dnodeInitModules(); void dnodeStartModules(); -void dnodeCleanupModules(); +void dnodeStartStream(); +void dnodeCleanUpModules(); void dnodeProcessModuleStatus(uint32_t moduleStatus); #ifdef __cplusplus diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index 7683843371..73bc2923b2 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -124,6 +124,7 @@ int32_t dnodeInitSystem() { dnodeStartModules(); dnodeSetRunStatus(TSDB_DNODE_RUN_STATUS_RUNING); + dnodeStartStream(); dPrint("TDengine is initialized successfully"); diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c index 7c457defca..a29f99dda6 100644 --- a/src/dnode/src/dnodeMgmt.c +++ b/src/dnode/src/dnodeMgmt.c @@ -260,11 +260,27 @@ static int32_t dnodeOpenVnodes() { } free(vnodeList); - dPrint("there are total vnodes:%d, openned:%d failed:%d", numOfVnodes, numOfVnodes-failed, failed); return TSDB_CODE_SUCCESS; } +void dnodeStartStream() { + int32_t vnodeList[TSDB_MAX_VNODES]; + int32_t numOfVnodes = 0; + int32_t status = dnodeGetVnodeList(vnodeList, &numOfVnodes); + + if (status != TSDB_CODE_SUCCESS) { + dPrint("Get dnode list failed"); + return; + } + + for (int32_t i = 0; i < numOfVnodes; ++i) { + vnodeStartStream(vnodeList[i]); + } + + dPrint("streams started"); +} + static void dnodeCloseVnodes() { int32_t *vnodeList = (int32_t *)malloc(sizeof(int32_t) * TSDB_MAX_VNODES); int32_t numOfVnodes; diff --git a/src/inc/tcq.h b/src/inc/tcq.h index e025afaa0a..ba198ab66e 100644 --- a/src/inc/tcq.h +++ b/src/inc/tcq.h @@ -27,6 +27,7 @@ typedef struct { int vgId; char user[TSDB_USER_LEN]; char pass[TSDB_PASSWORD_LEN]; + char db[TSDB_DB_NAME_LEN]; FCqWrite cqWrite; } SCqCfg; diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 2dc9b977b4..d3ec96acf6 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -118,6 +118,7 @@ int tsdbDropTable(TsdbRepoT *pRepo, STableId tableId); int tsdbAlterTable(TsdbRepoT *repo, STableCfg *pCfg); int tsdbUpdateTagValue(TsdbRepoT *repo, SUpdateTableTagValMsg *pMsg); TSKEY tsdbGetTableLastKey(TsdbRepoT *repo, uint64_t uid); +void tsdbStartStream(TsdbRepoT *repo); uint32_t tsdbGetFileInfo(TsdbRepoT *repo, char *name, uint32_t *index, uint32_t eindex, int32_t *size); diff --git a/src/inc/vnode.h b/src/inc/vnode.h index f4fb8060fe..e507c55942 100644 --- a/src/inc/vnode.h +++ b/src/inc/vnode.h @@ -38,6 +38,7 @@ typedef struct { int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg); int32_t vnodeDrop(int32_t vgId); int32_t vnodeOpen(int32_t vgId, char *rootDir); +int32_t vnodeStartStream(int32_t vgId); int32_t vnodeAlter(void *pVnode, SMDCreateVnodeMsg *pVnodeCfg); int32_t vnodeClose(int32_t vgId); diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 0efb477e73..632c1dd34e 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -474,6 +474,18 @@ TSKEY tsdbGetTableLastKey(TsdbRepoT *repo, uint64_t uid) { return TSDB_GET_TABLE_LAST_KEY(pTable); } +void tsdbStartStream(TsdbRepoT *repo) { + STsdbRepo *pRepo = (STsdbRepo *)repo; + STsdbMeta *pMeta = pRepo->tsdbMeta; + + for (int i = 0; i < pRepo->config.maxTables; i++) { + STable *pTable = pMeta->tables[i]; + if (pTable && pTable->type == TSDB_STREAM_TABLE) { + pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, pTable->tableId.tid, pTable->sql, tsdbGetTableSchema(pMeta, pTable)); + } + } +} + STableInfo *tsdbGetTableInfo(TsdbRepoT *pRepo, STableId tableId) { // TODO return NULL; diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 9f1e507d54..bf5fac4e45 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -150,7 +150,6 @@ int tsdbRestoreTable(void *pHandle, void *cont, int contLen) { void tsdbOrgMeta(void *pHandle) { STsdbMeta *pMeta = (STsdbMeta *)pHandle; - STsdbRepo *pRepo = (STsdbRepo *)pMeta->pRepo; for (int i = 1; i < pMeta->maxTables; i++) { STable *pTable = pMeta->tables[i]; @@ -158,13 +157,6 @@ void tsdbOrgMeta(void *pHandle) { tsdbAddTableIntoIndex(pMeta, pTable); } } - - for (int i = 0; i < pMeta->maxTables; i++) { - STable *pTable = pMeta->tables[i]; - if (pTable && pTable->type == TSDB_STREAM_TABLE) { - pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, i, pTable->sql, tsdbGetTableSchema(pMeta, pTable)); - } - } } /** diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 3d8cf67d02..037dadb8fd 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -208,8 +208,9 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { } SCqCfg cqCfg = {0}; - sprintf(cqCfg.user, "root"); + sprintf(cqCfg.user, "_root"); strcpy(cqCfg.pass, tsInternalPass); + strcpy(cqCfg.db, "s1_db0"); // TODO: replace hard coded db name cqCfg.vgId = vnode; cqCfg.cqWrite = vnodeWriteToQueue; pVnode->cq = cqOpen(pVnode, &cqCfg); @@ -277,6 +278,15 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { return TSDB_CODE_SUCCESS; } +int32_t vnodeStartStream(int32_t vnode) { + SVnodeObj* pVnode = vnodeAccquireVnode(vnode); + if (pVnode != NULL) { + tsdbStartStream(pVnode->tsdb); + vnodeRelease(pVnode); + } + return TSDB_CODE_SUCCESS; +} + int32_t vnodeClose(int32_t vgId) { SVnodeObj **ppVnode = (SVnodeObj **)taosHashGet(tsDnodeVnodesHash, (const char *)&vgId, sizeof(int32_t)); if (ppVnode == NULL || *ppVnode == NULL) return 0; From 69142ea5aaa29af8fca4dea9115d88665abcd307 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Mon, 1 Jun 2020 17:17:22 +0800 Subject: [PATCH 02/22] TD-449: data is written to TSDB --- src/client/src/tscStream.c | 8 ++++-- src/common/inc/tdataformat.h | 1 + src/cq/src/cqMain.c | 50 ++++++++++++++++++++++++++---------- src/cq/test/cqtest.c | 2 +- src/inc/tcq.h | 2 +- src/inc/tsdb.h | 2 +- src/tsdb/src/tsdbMain.c | 2 +- src/tsdb/src/tsdbMeta.c | 2 +- src/vnode/src/vnodeMain.c | 2 +- 9 files changed, 50 insertions(+), 21 deletions(-) diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index 9e1628bb9b..3c7cd3e1eb 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -19,6 +19,7 @@ #include "tscLog.h" #include "tscUtil.h" #include "tsched.h" +#include "tcache.h" #include "tsclient.h" #include "ttime.h" #include "ttimer.h" @@ -147,7 +148,8 @@ static void tscProcessStreamQueryCallback(void *param, TAOS_RES *tres, int numOf retryDelay); STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pStream->pSql->cmd, 0, 0); - tscClearTableMetaInfo(pTableMetaInfo, true); + taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), true); + tfree(pTableMetaInfo->vgroupList); tscSetRetryTimer(pStream, pStream->pSql, retryDelay); return; @@ -259,7 +261,9 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf pStream->numOfRes); // release the metric/meter meta information reference, so data in cache can be updated - tscClearTableMetaInfo(pTableMetaInfo, false); + + taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), false); + tfree(pTableMetaInfo->vgroupList); tscSetNextLaunchTimer(pStream, pSql); } } diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index 3a84ce8f6f..e706f9a9d5 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -70,6 +70,7 @@ typedef struct { int numOfCols; // Number of columns appended int tlen; // maximum length of a SDataRow without the header part int flen; // First part length in a SDataRow after the header part + int32_t version; STColumn columns[]; } STSchema; diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c index 799a0b0514..d96e0fe6a7 100644 --- a/src/cq/src/cqMain.c +++ b/src/cq/src/cqMain.c @@ -49,7 +49,8 @@ typedef struct { } SCqContext; typedef struct SCqObj { - int tid; // table ID + uint64_t uid; + int32_t tid; // table ID int rowSize; // bytes of a row char * sqlStr; // SQL string STSchema * pSchema; // pointer to schema array @@ -155,17 +156,19 @@ void cqStop(void *handle) { pthread_mutex_unlock(&pContext->mutex); } -void *cqCreate(void *handle, int tid, char *sqlStr, STSchema *pSchema) { +void *cqCreate(void *handle, uint64_t uid, int tid, char *sqlStr, STSchema *pSchema) { SCqContext *pContext = handle; SCqObj *pObj = calloc(sizeof(SCqObj), 1); if (pObj == NULL) return NULL; + pObj->uid = uid; pObj->tid = tid; pObj->sqlStr = malloc(strlen(sqlStr)+1); strcpy(pObj->sqlStr, sqlStr); pObj->pSchema = tdDupSchema(pSchema); + pObj->rowSize = pSchema->tlen; cTrace("vgId:%d, id:%d CQ:%s is created", pContext->vgId, pObj->tid, pObj->sqlStr); @@ -213,8 +216,8 @@ static void cqCreateStream(SCqContext *pContext, SCqObj *pObj) { pContext->dbConn = taos_connect("localhost", pContext->user, pContext->pass, pContext->db, 0); if (pContext->dbConn == NULL) { cError("vgId:%d, failed to connect to TDengine(%s)", pContext->vgId, tstrerror(terrno)); + return; } - return; } int64_t lastKey = 0; @@ -231,6 +234,7 @@ static void cqCreateStream(SCqContext *pContext, SCqObj *pObj) { static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) { SCqObj *pObj = (SCqObj *)param; SCqContext *pContext = pObj->pContext; + STSchema *pSchema = pObj->pSchema; if (pObj->pStream == NULL) return; cTrace("vgId:%d, id:%d CQ:%s stream result is ready", pContext->vgId, pObj->tid, pObj->sqlStr); @@ -240,18 +244,38 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) { char *buffer = calloc(size, 1); SWalHead *pHead = (SWalHead *)buffer; + SSubmitMsg *pMsg = (SSubmitMsg *) (buffer + sizeof(SWalHead)); + SSubmitBlk *pBlk = (SSubmitBlk *) (buffer + sizeof(SWalHead) + sizeof(SSubmitMsg)); + + int32_t flen = 0; + for (int32_t i = 0; i < pSchema->numOfCols; i++) { + flen += TYPE_BYTES[pSchema->columns[i].type]; + } + + SDataRow trow = (SDataRow)pBlk->data; + dataRowSetLen(trow, TD_DATA_ROW_HEAD_SIZE + flen); + + int toffset = 0; + for (int32_t i = 0; i < pSchema->numOfCols; i++) { + tdAppendColVal(trow, row[i], pSchema->columns[i].type, pSchema->columns[i].bytes, toffset); + toffset += TYPE_BYTES[pSchema->columns[i].type]; + } + pBlk->len = htonl(dataRowLen(trow)); + + pBlk->uid = htobe64(pObj->uid); + pBlk->tid = htonl(pObj->tid); + pBlk->numOfRows = htons(1); + pBlk->sversion = htonl(pSchema->version); + pBlk->padding = 0; + + pMsg->header.vgId = htonl(pContext->vgId); + pMsg->header.contLen = htonl(size - sizeof(SWalHead)); + pMsg->length = pMsg->header.contLen; + pMsg->numOfBlocks = htonl(1); + pHead->msgType = TSDB_MSG_TYPE_SUBMIT; pHead->len = size - sizeof(SWalHead); - - SSubmitMsg *pSubmit = (SSubmitMsg *) (buffer + sizeof(SWalHead)); - // to do: fill in the SSubmitMsg structure - pSubmit->numOfBlocks = 1; - - - SSubmitBlk *pBlk = (SSubmitBlk *) (buffer + sizeof(SWalHead) + sizeof(SSubmitMsg)); - // to do: fill in the SSubmitBlk strucuture - pBlk->tid = pObj->tid; - + pHead->version = 0; // write into vnode write queue pContext->cqWrite(pContext->ahandle, pHead, TAOS_QTYPE_CQ); diff --git a/src/cq/test/cqtest.c b/src/cq/test/cqtest.c index fbe3c95b86..1416a591be 100644 --- a/src/cq/test/cqtest.c +++ b/src/cq/test/cqtest.c @@ -70,7 +70,7 @@ int main(int argc, char *argv[]) { tdDestroyTSchemaBuilder(&schemaBuilder); for (int sid =1; sid<10; ++sid) { - cqCreate(pCq, sid, "select avg(speed) from demo.t1 sliding(1s) interval(5s)", pSchema); + cqCreate(pCq, sid, sid, "select avg(speed) from demo.t1 sliding(1s) interval(5s)", pSchema); } tdFreeSchema(pSchema); diff --git a/src/inc/tcq.h b/src/inc/tcq.h index ba198ab66e..32b75674c3 100644 --- a/src/inc/tcq.h +++ b/src/inc/tcq.h @@ -42,7 +42,7 @@ void cqStart(void *handle); void cqStop(void *handle); // cqCreate is called by TSDB to start an instance of CQ -void *cqCreate(void *handle, int sid, char *sqlStr, STSchema *pSchema); +void *cqCreate(void *handle, uint64_t uid, int sid, char *sqlStr, STSchema *pSchema); // cqDrop is called by TSDB to stop an instance of CQ, handle is the return value of cqCreate void cqDrop(void *handle); diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index d3ec96acf6..a678f213bb 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -43,7 +43,7 @@ typedef struct { void *cqH; int (*notifyStatus)(void *, int status); int (*eventCallBack)(void *); - void *(*cqCreateFunc)(void *handle, int sid, char *sqlStr, STSchema *pSchema); + void *(*cqCreateFunc)(void *handle, uint64_t uid, int sid, char *sqlStr, STSchema *pSchema); void (*cqDropFunc)(void *handle); void *(*configFunc)(int32_t vgId, int32_t sid); } STsdbAppH; diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 632c1dd34e..f8fea779f3 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -481,7 +481,7 @@ void tsdbStartStream(TsdbRepoT *repo) { for (int i = 0; i < pRepo->config.maxTables; i++) { STable *pTable = pMeta->tables[i]; if (pTable && pTable->type == TSDB_STREAM_TABLE) { - pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, pTable->tableId.tid, pTable->sql, tsdbGetTableSchema(pMeta, pTable)); + pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, pTable->tableId.uid, pTable->tableId.tid, pTable->sql, tsdbGetTableSchema(pMeta, pTable)); } } } diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index bf5fac4e45..ac0f6b26ac 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -675,7 +675,7 @@ static int tsdbAddTableToMeta(STsdbMeta *pMeta, STable *pTable, bool addIdx) { tsdbAddTableIntoIndex(pMeta, pTable); } if (pTable->type == TSDB_STREAM_TABLE && addIdx) { - pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, pTable->tableId.tid, pTable->sql, tsdbGetTableSchema(pMeta, pTable)); + pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, pTable->tableId.uid, pTable->tableId.tid, pTable->sql, tsdbGetTableSchema(pMeta, pTable)); } pMeta->nTables++; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 037dadb8fd..26ba392a76 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -210,7 +210,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { SCqCfg cqCfg = {0}; sprintf(cqCfg.user, "_root"); strcpy(cqCfg.pass, tsInternalPass); - strcpy(cqCfg.db, "s1_db0"); // TODO: replace hard coded db name + strcpy(cqCfg.db, "db"); // TODO: replace hard coded db name cqCfg.vgId = vnode; cqCfg.cqWrite = vnodeWriteToQueue; pVnode->cq = cqOpen(pVnode, &cqCfg); From 40c1d665b5481850c4a0e0604675f5eedf648e03 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Tue, 2 Jun 2020 18:12:33 +0800 Subject: [PATCH 03/22] fix some issues --- src/client/src/tscStream.c | 14 ++++++++++++++ src/common/inc/tdataformat.h | 1 - src/cq/src/cqMain.c | 13 +++++++------ src/dnode/inc/dnodeModule.h | 2 +- src/inc/tcq.h | 2 +- src/vnode/src/vnodeMain.c | 2 +- 6 files changed, 24 insertions(+), 10 deletions(-) diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index 3c7cd3e1eb..83a097f2f5 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -477,6 +477,14 @@ static void setErrorInfo(SSqlObj* pSql, int32_t code, char* info) { } } +static void asyncCallback(void *param, TAOS_RES *tres, int code) { + assert(param != NULL); + SSqlObj *pSql = ((SSqlObj *)param); + + pSql->res.code = code; + sem_post(&pSql->rspSem); +} + TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row), int64_t stime, void *param, void (*callback)(void *)) { STscObj *pObj = (STscObj *)taos; @@ -521,7 +529,13 @@ TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *p return NULL; } + pSql->param = pSql; + pSql->fp = asyncCallback; pRes->code = tscToSQLCmd(pSql, &SQLInfo); + if (pRes->code == TSDB_CODE_ACTION_IN_PROGRESS) { + sem_wait(&pSql->rspSem); + } + SQLInfoDestroy(&SQLInfo); if (pRes->code != TSDB_CODE_SUCCESS) { diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index e706f9a9d5..3a84ce8f6f 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -70,7 +70,6 @@ typedef struct { int numOfCols; // Number of columns appended int tlen; // maximum length of a SDataRow without the header part int flen; // First part length in a SDataRow after the header part - int32_t version; STColumn columns[]; } STSchema; diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c index d96e0fe6a7..401d61b7a2 100644 --- a/src/cq/src/cqMain.c +++ b/src/cq/src/cqMain.c @@ -239,19 +239,19 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) { cTrace("vgId:%d, id:%d CQ:%s stream result is ready", pContext->vgId, pObj->tid, pObj->sqlStr); + int32_t flen = 0; + for (int32_t i = 0; i < pSchema->numOfCols; i++) { + flen += TYPE_BYTES[pSchema->columns[i].type]; + } + // construct data - int size = sizeof(SWalHead) + sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + pObj->rowSize; + int size = sizeof(SWalHead) + sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + TD_DATA_ROW_HEAD_SIZE + flen; char *buffer = calloc(size, 1); SWalHead *pHead = (SWalHead *)buffer; SSubmitMsg *pMsg = (SSubmitMsg *) (buffer + sizeof(SWalHead)); SSubmitBlk *pBlk = (SSubmitBlk *) (buffer + sizeof(SWalHead) + sizeof(SSubmitMsg)); - int32_t flen = 0; - for (int32_t i = 0; i < pSchema->numOfCols; i++) { - flen += TYPE_BYTES[pSchema->columns[i].type]; - } - SDataRow trow = (SDataRow)pBlk->data; dataRowSetLen(trow, TD_DATA_ROW_HEAD_SIZE + flen); @@ -279,5 +279,6 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) { // write into vnode write queue pContext->cqWrite(pContext->ahandle, pHead, TAOS_QTYPE_CQ); + free(buffer); } diff --git a/src/dnode/inc/dnodeModule.h b/src/dnode/inc/dnodeModule.h index fb529ee67c..6a6da0a2a5 100644 --- a/src/dnode/inc/dnodeModule.h +++ b/src/dnode/inc/dnodeModule.h @@ -23,7 +23,7 @@ extern "C" { int32_t dnodeInitModules(); void dnodeStartModules(); void dnodeStartStream(); -void dnodeCleanUpModules(); +void dnodeCleanupModules(); void dnodeProcessModuleStatus(uint32_t moduleStatus); #ifdef __cplusplus diff --git a/src/inc/tcq.h b/src/inc/tcq.h index 32b75674c3..9d987da468 100644 --- a/src/inc/tcq.h +++ b/src/inc/tcq.h @@ -27,7 +27,7 @@ typedef struct { int vgId; char user[TSDB_USER_LEN]; char pass[TSDB_PASSWORD_LEN]; - char db[TSDB_DB_NAME_LEN]; + char db[TSDB_DB_NAME_LEN + 1]; FCqWrite cqWrite; } SCqCfg; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 26ba392a76..e4f1f7d2f5 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -210,7 +210,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { SCqCfg cqCfg = {0}; sprintf(cqCfg.user, "_root"); strcpy(cqCfg.pass, tsInternalPass); - strcpy(cqCfg.db, "db"); // TODO: replace hard coded db name + strcpy(cqCfg.db, pVnode->db); cqCfg.vgId = vnode; cqCfg.cqWrite = vnodeWriteToQueue; pVnode->cq = cqOpen(pVnode, &cqCfg); From 81ea4a8a8617d03d301935529ee2e7c7d9ff9bbb Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Wed, 3 Jun 2020 16:16:41 +0800 Subject: [PATCH 04/22] first test case passed --- src/client/inc/tsclient.h | 1 + src/client/src/tscAsync.c | 37 +++++----- src/client/src/tscStream.c | 39 ++-------- src/cq/src/cqMain.c | 9 ++- tests/pytest/stream/stream1.py | 131 +++++++++++++++++++++++++++++++++ 5 files changed, 166 insertions(+), 51 deletions(-) create mode 100644 tests/pytest/stream/stream1.py diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index d7f1881209..e92bb5e64e 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -404,6 +404,7 @@ TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void *param, void **taos); void waitForQueryRsp(void *param, TAOS_RES *tres, int code) ; +int doAsyncParseSql(SSqlObj* pSql, const char* sqlstr, size_t sqlLen); void doAsyncQuery(STscObj *pObj, SSqlObj *pSql, void (*fp)(), void *param, const char *sqlstr, size_t sqlLen); void tscProcessMultiVnodesInsertFromFile(SSqlObj *pSql); diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index ebcdddffde..bd2dc64ef0 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -40,30 +40,23 @@ static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRo static void tscAsyncFetchRowsProxy(void *param, TAOS_RES *tres, int numOfRows); static void tscAsyncFetchSingleRowProxy(void *param, TAOS_RES *tres, int numOfRows); -void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const char* sqlstr, size_t sqlLen) { - SSqlCmd *pCmd = &pSql->cmd; - SSqlRes *pRes = &pSql->res; - - pSql->signature = pSql; - pSql->param = param; - pSql->pTscObj = pObj; - pSql->maxRetry = TSDB_MAX_REPLICA_NUM; - pSql->fp = fp; - - sem_init(&pSql->rspSem, 0, 0); - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE)) { +int doAsyncParseSql(SSqlObj* pSql, const char* sqlstr, size_t sqlLen) { + SSqlCmd* pCmd = &pSql->cmd; + SSqlRes* pRes = &pSql->res; + int32_t code = tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE); + if (code != TSDB_CODE_SUCCESS) { tscError("failed to malloc payload"); - tscQueueAsyncError(fp, param, TSDB_CODE_CLI_OUT_OF_MEMORY); - return; + tscQueueAsyncError(pSql->fp, pSql->param, TSDB_CODE_CLI_OUT_OF_MEMORY); + return code; } // todo check for OOM problem pSql->sqlstr = calloc(1, sqlLen + 1); if (pSql->sqlstr == NULL) { tscError("%p failed to malloc sql string buffer", pSql); - tscQueueAsyncError(fp, param, TSDB_CODE_CLI_OUT_OF_MEMORY); + tscQueueAsyncError(pSql->fp, pSql->param, TSDB_CODE_CLI_OUT_OF_MEMORY); free(pCmd->payload); - return; + return TSDB_CODE_CLI_OUT_OF_MEMORY; } pRes->qhandle = 0; @@ -72,7 +65,17 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const strtolower(pSql->sqlstr, sqlstr); tscDump("%p SQL: %s", pSql, pSql->sqlstr); - int32_t code = tsParseSql(pSql, true); + return tsParseSql(pSql, true); +} + +void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const char* sqlstr, size_t sqlLen) { + pSql->signature = pSql; + pSql->param = param; + pSql->pTscObj = pObj; + pSql->maxRetry = TSDB_MAX_REPLICA_NUM; + pSql->fp = fp; + + int32_t code = doAsyncParseSql(pSql, sqlstr, sqlLen); if (code == TSDB_CODE_ACTION_IN_PROGRESS) return; if (code != TSDB_CODE_SUCCESS) { diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index 83a097f2f5..028ae0592a 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -497,46 +497,18 @@ TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *p } pSql->signature = pSql; + pSql->param = pSql; pSql->pTscObj = pObj; + pSql->fp = asyncCallback; + SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; - int ret = tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE); - if (TSDB_CODE_SUCCESS != ret) { - setErrorInfo(pSql, ret, NULL); - free(pSql); - return NULL; - } - - pSql->sqlstr = strdup(sqlstr); - if (pSql->sqlstr == NULL) { - setErrorInfo(pSql, TSDB_CODE_CLI_OUT_OF_MEMORY, NULL); - - tfree(pSql); - return NULL; - } tsem_init(&pSql->rspSem, 0, 0); - - SSqlInfo SQLInfo = {0}; - tSQLParse(&SQLInfo, pSql->sqlstr); - - tscResetSqlCmdObj(&pSql->cmd); - ret = tscAllocPayload(&pSql->cmd, TSDB_DEFAULT_PAYLOAD_SIZE); - if (TSDB_CODE_SUCCESS != ret) { - setErrorInfo(pSql, ret, NULL); - tscError("%p open stream failed, sql:%s, code:%d", pSql, sqlstr, TSDB_CODE_CLI_OUT_OF_MEMORY); - tscFreeSqlObj(pSql); - return NULL; - } - - pSql->param = pSql; - pSql->fp = asyncCallback; - pRes->code = tscToSQLCmd(pSql, &SQLInfo); - if (pRes->code == TSDB_CODE_ACTION_IN_PROGRESS) { + int32_t code = doAsyncParseSql(pSql, sqlstr, strlen(sqlstr)); + if (code == TSDB_CODE_ACTION_IN_PROGRESS) { sem_wait(&pSql->rspSem); } - - SQLInfoDestroy(&SQLInfo); if (pRes->code != TSDB_CODE_SUCCESS) { setErrorInfo(pSql, pRes->code, pCmd->payload); @@ -575,6 +547,7 @@ TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *p pStream->stime = tscGetStreamStartTimestamp(pSql, pStream, stime); int64_t starttime = tscGetLaunchTimestamp(pStream); + pCmd->command = TSDB_SQL_SELECT; taosTmrReset(tscProcessStreamTimer, starttime, pStream, tscTmr, &pStream->pTimer); tscTrace("%p stream:%p is opened, query on:%s, interval:%" PRId64 ", sliding:%" PRId64 ", first launched in:%" PRId64 ", sql:%s", pSql, diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c index 401d61b7a2..5c936388dd 100644 --- a/src/cq/src/cqMain.c +++ b/src/cq/src/cqMain.c @@ -75,7 +75,14 @@ void *cqOpen(void *ahandle, const SCqCfg *pCfg) { strcpy(pContext->user, pCfg->user); strcpy(pContext->pass, pCfg->pass); - strcpy(pContext->db, pCfg->db); + const char* db = pCfg->db; + for (const char* p = db; *p != 0; p++) { + if (*p == '.') { + db = p + 1; + break; + } + } + strcpy(pContext->db, db); pContext->vgId = pCfg->vgId; pContext->cqWrite = pCfg->cqWrite; pContext->ahandle = ahandle; diff --git a/tests/pytest/stream/stream1.py b/tests/pytest/stream/stream1.py new file mode 100644 index 0000000000..7a9d88da3b --- /dev/null +++ b/tests/pytest/stream/stream1.py @@ -0,0 +1,131 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import time +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tbNum = 10 + rowNum = 20 + + tdSql.prepare() + + tdLog.info("===== step1 =====") + tdSql.execute("create table stb0(ts timestamp, col1 int, col2 float) tags(tgcol int)") + for i in range(tbNum): + tdSql.execute("create table tb%d using stb0 tags(%d)" % (i, i)) + for j in range(rowNum): + tdSql.execute("insert into tb%d values (now - %dm, %d, %d)" % (i, 1440 - j, j, j)) + time.sleep(0.1) + + tdLog.info("===== step2 =====") + tdSql.query("select count(*), count(col1), count(col2) from tb0 interval(1d)") + tdSql.checkData(0, 1, rowNum) + tdSql.checkData(0, 2, rowNum) + tdSql.checkData(0, 3, rowNum) + tdSql.query("show tables") + tdSql.checkRows(tbNum) + tdSql.execute("create table s0 as select count(*), count(col1), count(col2) from tb0 interval(1d)") + tdSql.query("show tables") + tdSql.checkRows(tbNum + 1) + + tdLog.info("===== step3 =====") + tdLog.info("sleeping 120 seconds") + time.sleep(120) + tdSql.query("select * from s0") + tdSql.checkData(0, 1, rowNum) + tdSql.checkData(0, 2, rowNum) + tdSql.checkData(0, 3, rowNum) + + tdLog.info("===== step4 =====") + tdSql.execute("drop table s0") + tdSql.query("show tables") + tdSql.checkRows(tbNum) + + tdLog.info("===== step5 =====") + tdSql.error("select * from s0") + + tdLog.info("===== step6 =====") + time.sleep(0.1) + tdSql.execute("create table s0 as select count(*), count(col1), count(col2) from tb0 interval(1d)") + tdSql.query("show tables") + tdSql.checkRows(tbNum + 1) + + tdLog.info("===== step7 =====") + tdLog.info("sleeping 120 seconds") + time.sleep(120) + + tdSql.query("select * from s0") + tdSql.checkData(0, 1, rowNum) + tdSql.checkData(0, 2, rowNum) + tdSql.checkData(0, 3, rowNum) + + tdLog.info("===== step8 =====") + tdSql.query("select count(*), count(col1), count(col2) from stb0 interval(1d)") + tdSql.checkData(0, 1, rowNum * tbNum) + tdSql.checkData(0, 2, rowNum * tbNum) + tdSql.checkData(0, 3, rowNum * tbNum) + tdSql.query("show tables") + tdSql.checkRows(tbNum + 1) + + tdSql.execute("create table s1 as select count(*), count(col1), count(col2) from stb0 interval(1d)") + tdSql.query("show tables") + tdSql.checkRows(tbNum + 2) + + tdLog.info("===== step9 =====") + tdLog.info("sleeping 120 seconds") + time.sleep(120) + + tdSql.query("select * from s1") + tdSql.checkData(0, 1, rowNum * tbNum) + tdSql.checkData(0, 2, rowNum * tbNum) + tdSql.checkData(0, 3, rowNum * tbNum) + + tdLog.info("===== step10 =====") + tdSql.execute("drop table s1") + tdSql.query("show tables") + tdSql.checkRows(tbNum + 1) + + tdLog.info("===== step11 =====") + tdSql.error("select * from s1") + + tdLog.info("===== step12 =====") + tdSql.execute("create table s1 as select count(*), count(col1), count(col2) from stb0 interval(1d)") + tdSql.query("show tables") + tdSql.checkRows(tbNum + 2) + + tdLog.info("===== step13 =====") + tdLog.info("sleeping 120 seconds") + time.sleep(120) + tdSql.query("select * from s1") + tdSql.checkData(0, 1, rowNum * tbNum) + tdSql.checkData(0, 2, rowNum * tbNum) + tdSql.checkData(0, 3, rowNum * tbNum) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 51de20d2e0c0dea8bec77be77258c2db2283c328 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Fri, 5 Jun 2020 19:00:04 +0800 Subject: [PATCH 05/22] td-449: more cases passedcurrently, only below four cases fail:* agg_stream.sim ('us' ?)* column_stream.sim ('us' ?)* table_replica1_vnoden.sim (crash)* new_stream.sim (failed at line 101) --- src/client/inc/tsclient.h | 2 +- src/client/src/tscAsync.c | 34 ++--- src/client/src/tscStream.c | 40 +++--- src/common/inc/tdataformat.h | 1 + src/cq/src/cqMain.c | 19 ++- tests/pytest/stream/stream2.py | 122 ++++++++++++++++++ tests/script/general/stream/metrics_1.sim | 4 +- tests/script/general/stream/metrics_del.sim | 10 +- tests/script/general/stream/metrics_n.sim | 4 +- .../stream/metrics_replica1_vnoden.sim | 4 +- .../script/general/stream/restart_stream.sim | 8 +- tests/script/general/stream/stream_1.sim | 16 +-- tests/script/general/stream/stream_2.sim | 20 +-- tests/script/general/stream/stream_3.sim | 8 +- .../script/general/stream/stream_restart.sim | 4 +- tests/script/general/stream/table_1.sim | 4 +- tests/script/general/stream/table_del.sim | 10 +- tests/script/general/stream/table_n.sim | 4 +- 18 files changed, 210 insertions(+), 104 deletions(-) create mode 100644 tests/pytest/stream/stream2.py diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index e92bb5e64e..5356a71579 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -404,7 +404,7 @@ TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void *param, void **taos); void waitForQueryRsp(void *param, TAOS_RES *tres, int code) ; -int doAsyncParseSql(SSqlObj* pSql, const char* sqlstr, size_t sqlLen); +int doAsyncParseSql(SSqlObj* pSql); void doAsyncQuery(STscObj *pObj, SSqlObj *pSql, void (*fp)(), void *param, const char *sqlstr, size_t sqlLen); void tscProcessMultiVnodesInsertFromFile(SSqlObj *pSql); diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index bd2dc64ef0..3cf9e0daa5 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -40,7 +40,7 @@ static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRo static void tscAsyncFetchRowsProxy(void *param, TAOS_RES *tres, int numOfRows); static void tscAsyncFetchSingleRowProxy(void *param, TAOS_RES *tres, int numOfRows); -int doAsyncParseSql(SSqlObj* pSql, const char* sqlstr, size_t sqlLen) { +int doAsyncParseSql(SSqlObj* pSql) { SSqlCmd* pCmd = &pSql->cmd; SSqlRes* pRes = &pSql->res; int32_t code = tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE); @@ -50,21 +50,10 @@ int doAsyncParseSql(SSqlObj* pSql, const char* sqlstr, size_t sqlLen) { return code; } - // todo check for OOM problem - pSql->sqlstr = calloc(1, sqlLen + 1); - if (pSql->sqlstr == NULL) { - tscError("%p failed to malloc sql string buffer", pSql); - tscQueueAsyncError(pSql->fp, pSql->param, TSDB_CODE_CLI_OUT_OF_MEMORY); - free(pCmd->payload); - return TSDB_CODE_CLI_OUT_OF_MEMORY; - } - pRes->qhandle = 0; pRes->numOfRows = 1; - strtolower(pSql->sqlstr, sqlstr); tscDump("%p SQL: %s", pSql, pSql->sqlstr); - return tsParseSql(pSql, true); } @@ -74,8 +63,15 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const pSql->pTscObj = pObj; pSql->maxRetry = TSDB_MAX_REPLICA_NUM; pSql->fp = fp; + pSql->sqlstr = calloc(1, sqlLen + 1); + if (pSql->sqlstr == NULL) { + tscError("%p failed to malloc sql string buffer", pSql); + tscQueueAsyncError(pSql->fp, pSql->param, TSDB_CODE_CLI_OUT_OF_MEMORY); + return; + } + strtolower(pSql->sqlstr, sqlstr); - int32_t code = doAsyncParseSql(pSql, sqlstr, sqlLen); + int32_t code = doAsyncParseSql(pSql); if (code == TSDB_CODE_ACTION_IN_PROGRESS) return; if (code != TSDB_CODE_SUCCESS) { @@ -521,15 +517,9 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { if (pSql->pStream) { tscTrace("%p stream:%p meta is updated, start new query, command:%d", pSql, pSql->pStream, pSql->cmd.command); - /* - * NOTE: - * transfer the sql function for super table query before get meter/metric meta, - * since in callback functions, only tscProcessSql(pStream->pSql) is executed! - */ - SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - - tscTansformSQLFuncForSTableQuery(pQueryInfo); - tscIncStreamExecutionCount(pSql->pStream); + tsParseSql(pSql, false); + sem_post(&pSql->rspSem); + return; } else { tscTrace("%p get tableMeta successfully", pSql); } diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index 028ae0592a..c4c9860f3b 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -99,7 +99,7 @@ static void tscProcessStreamLaunchQuery(SSchedMsg *pMsg) { } tscTrace("%p stream:%p start stream query on:%s", pSql, pStream, pTableMetaInfo->name); - tscProcessSql(pStream->pSql); + tscDoQuery(pStream->pSql); tscIncStreamExecutionCount(pStream); } @@ -477,14 +477,6 @@ static void setErrorInfo(SSqlObj* pSql, int32_t code, char* info) { } } -static void asyncCallback(void *param, TAOS_RES *tres, int code) { - assert(param != NULL); - SSqlObj *pSql = ((SSqlObj *)param); - - pSql->res.code = code; - sem_post(&pSql->rspSem); -} - TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row), int64_t stime, void *param, void (*callback)(void *)) { STscObj *pObj = (STscObj *)taos; @@ -492,20 +484,34 @@ TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *p SSqlObj *pSql = (SSqlObj *)calloc(1, sizeof(SSqlObj)); if (pSql == NULL) { - setErrorInfo(pSql, TSDB_CODE_CLI_OUT_OF_MEMORY, NULL); return NULL; } pSql->signature = pSql; pSql->param = pSql; pSql->pTscObj = pObj; - pSql->fp = asyncCallback; SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; + SSqlStream *pStream = (SSqlStream *)calloc(1, sizeof(SSqlStream)); + if (pStream == NULL) { + tscError("%p open stream failed, sql:%s, reason:%s, code:%d", pSql, sqlstr, pCmd->payload, pRes->code); + tscFreeSqlObj(pSql); + return NULL; + } + pSql->pStream = pStream; + + pSql->sqlstr = calloc(1, strlen(sqlstr) + 1); + if (pSql->sqlstr == NULL) { + tscError("%p failed to malloc sql string buffer", pSql); + tscFreeSqlObj(pSql); + return NULL;; + } + strtolower(pSql->sqlstr, sqlstr); + tsem_init(&pSql->rspSem, 0, 0); - int32_t code = doAsyncParseSql(pSql, sqlstr, strlen(sqlstr)); + int32_t code = doAsyncParseSql(pSql); if (code == TSDB_CODE_ACTION_IN_PROGRESS) { sem_wait(&pSql->rspSem); } @@ -518,15 +524,6 @@ TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *p return NULL; } - SSqlStream *pStream = (SSqlStream *)calloc(1, sizeof(SSqlStream)); - if (pStream == NULL) { - setErrorInfo(pSql, TSDB_CODE_CLI_OUT_OF_MEMORY, NULL); - - tscError("%p open stream failed, sql:%s, reason:%s, code:%d", pSql, sqlstr, pCmd->payload, pRes->code); - tscFreeSqlObj(pSql); - return NULL; - } - SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); @@ -540,7 +537,6 @@ TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *p pStream->ctime = taosGetTimestamp(pStream->precision); pStream->etime = pQueryInfo->window.ekey; - pSql->pStream = pStream; tscAddIntoStreamList(pStream); tscSetSlidingWindowInfo(pSql, pStream); diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index 3a84ce8f6f..ec4e544e18 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -145,6 +145,7 @@ void tdFreeDataRow(SDataRow row); void tdInitDataRow(SDataRow row, STSchema *pSchema); SDataRow tdDataRowDup(SDataRow row); +// offset here not include dataRow header length static FORCE_INLINE int tdAppendColVal(SDataRow row, void *value, int8_t type, int32_t bytes, int32_t offset) { ASSERT(value != NULL); int32_t toffset = offset + TD_DATA_ROW_HEAD_SIZE; diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c index 5c936388dd..23455b6f50 100644 --- a/src/cq/src/cqMain.c +++ b/src/cq/src/cqMain.c @@ -246,13 +246,7 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) { cTrace("vgId:%d, id:%d CQ:%s stream result is ready", pContext->vgId, pObj->tid, pObj->sqlStr); - int32_t flen = 0; - for (int32_t i = 0; i < pSchema->numOfCols; i++) { - flen += TYPE_BYTES[pSchema->columns[i].type]; - } - - // construct data - int size = sizeof(SWalHead) + sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + TD_DATA_ROW_HEAD_SIZE + flen; + int size = sizeof(SWalHead) + sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + TD_DATA_ROW_HEAD_SIZE + pObj->rowSize; char *buffer = calloc(size, 1); SWalHead *pHead = (SWalHead *)buffer; @@ -260,12 +254,15 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) { SSubmitBlk *pBlk = (SSubmitBlk *) (buffer + sizeof(SWalHead) + sizeof(SSubmitMsg)); SDataRow trow = (SDataRow)pBlk->data; - dataRowSetLen(trow, TD_DATA_ROW_HEAD_SIZE + flen); + tdInitDataRow(trow, pSchema); - int toffset = 0; for (int32_t i = 0; i < pSchema->numOfCols; i++) { - tdAppendColVal(trow, row[i], pSchema->columns[i].type, pSchema->columns[i].bytes, toffset); - toffset += TYPE_BYTES[pSchema->columns[i].type]; + STColumn *c = pSchema->columns + i; + char* val = (char*)row[i]; + if (IS_VAR_DATA_TYPE(c->type)) { + val -= sizeof(VarDataLenT); + } + tdAppendColVal(trow, val, c->type, c->bytes, c->offset); } pBlk->len = htonl(dataRowLen(trow)); diff --git a/tests/pytest/stream/stream2.py b/tests/pytest/stream/stream2.py new file mode 100644 index 0000000000..96eff3131d --- /dev/null +++ b/tests/pytest/stream/stream2.py @@ -0,0 +1,122 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import time +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + + def run(self): + tbNum = 10 + rowNum = 20 + totalNum = tbNum * rowNum + + tdSql.prepare() + + tdLog.info("===== step1 =====") + tdSql.execute("create table stb0(ts timestamp, col1 int, col2 float) tags(tgcol int)") + for i in range(tbNum): + tdSql.execute("create table tb%d using stb0 tags(%d)" % (i, i)) + for j in range(rowNum): + tdSql.execute("insert into tb%d values (now - %dm, %d, %d)" % (i, 1440 - j, j, j)) + time.sleep(0.1) + + tdLog.info("===== step2 =====") + tdSql.query("select count(col1) from tb0 interval(1d)") + tdSql.checkData(0, 1, rowNum) + tdSql.query("show tables") + tdSql.checkRows(tbNum) + tdSql.execute("create table s0 as select count(col1) from tb0 interval(1d)") + tdSql.query("show tables") + tdSql.checkRows(tbNum + 1) + + tdLog.info("===== step3 =====") + time.sleep(120) + tdSql.query("select * from s0") + tdSql.checkData(0, 1, rowNum) + + tdLog.info("===== step4 =====") + tdSql.execute("drop table s0") + tdSql.query("show tables") + tdSql.checkRows(tbNum) + + tdLog.info("===== step5 =====") + tdSql.error("select * from s0") + + tdLog.info("===== step6 =====") + tdSql.execute("create table s0 as select count(*), count(col1), count(col2) from tb0 interval(1d)") + tdSql.query("show tables") + tdSql.checkRows(tbNum + 1) + + tdLog.info("===== step7 =====") + time.sleep(120) + tdSql.query("select * from s0") + tdSql.checkData(0, 1, rowNum) + tdSql.checkData(0, 2, rowNum) + tdSql.checkData(0, 3, rowNum) + + tdLog.info("===== step8 =====") + tdSql.query("select count(*), count(col1), count(col2) from stb0 interval(1d)") + tdSql.checkData(0, 1, totalNum) + tdSql.checkData(0, 2, totalNum) + tdSql.checkData(0, 3, totalNum) + tdSql.query("show tables") + tdSql.checkRows(tbNum + 1) + tdSql.execute("create table s1 as select count(*), count(col1), count(col2) from stb0 interval(1d)") + tdSql.query("show tables") + tdSql.checkRows(tbNum + 2) + + tdLog.info("===== step9 =====") + time.sleep(120) + tdSql.query("select * from s1") + tdSql.checkData(0, 1, totalNum) + tdSql.checkData(0, 2, totalNum) + tdSql.checkData(0, 3, totalNum) + + tdLog.info("===== step10 =====") + tdSql.execute("drop table s1") + tdSql.query("show tables") + tdSql.checkRows(tbNum + 1) + + tdLog.info("===== step11 =====") + tdSql.error("select * from s1") + + tdLog.info("===== step12 =====") + tdSql.execute("create table s1 as select count(col1) from stb0 interval(1d)") + tdSql.query("show tables") + tdSql.checkRows(tbNum + 2) + + tdLog.info("===== step13 =====") + time.sleep(120) + tdSql.query("select * from s1") + tdSql.checkData(0, 1, totalNum) + #tdSql.checkData(0, 2, None) + #tdSql.checkData(0, 3, None) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/script/general/stream/metrics_1.sim b/tests/script/general/stream/metrics_1.sim index c60bde6b2f..94498cb925 100644 --- a/tests/script/general/stream/metrics_1.sim +++ b/tests/script/general/stream/metrics_1.sim @@ -205,8 +205,8 @@ if $data01 != 20 then endi print =============== step21 -print sleep 22 seconds -sleep 22000 +print sleep 120 seconds +sleep 120000 print =============== step22 $st = $stPrefix . c1 diff --git a/tests/script/general/stream/metrics_del.sim b/tests/script/general/stream/metrics_del.sim index c856871e93..e21fa5999a 100644 --- a/tests/script/general/stream/metrics_del.sim +++ b/tests/script/general/stream/metrics_del.sim @@ -76,20 +76,20 @@ endw sql drop table $mt print =============== step4 -print sleep 22 seconds -sleep 22000 +print sleep 120 seconds +sleep 120000 print =============== step5 $st = $stPrefix . c3 sql select * from $st print ===> select * from $st print ===> $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 -if $data01 != NULL then +if $data01 != null then return -1 endi -if $data02 != NULL then +if $data02 != null then return -1 endi -if $data03 != NULL then +if $data03 != null then return -1 endi diff --git a/tests/script/general/stream/metrics_n.sim b/tests/script/general/stream/metrics_n.sim index 47089403dd..7fc08064b2 100644 --- a/tests/script/general/stream/metrics_n.sim +++ b/tests/script/general/stream/metrics_n.sim @@ -187,8 +187,8 @@ $st = $stPrefix . as #sql create table $st as select avg(tbcol) as a1, sum(tbcol) as a2, min(tbcol) as a3, max(tbcol) as a4, first(tbcol) as a5, last(tbcol) as a6, count(tbcol) as a7, avg(tbcol) as a8, sum(tbcol) as a9, min(tbcol) as a3, max(tbcol) as a4, first(tbcol) as a5, last(tbcol) as a6, count(tbcol) as a7 from $mt where ts < now + 4m interval(1d) print =============== step9 -print sleep 22 seconds -sleep 22000 +print sleep 120 seconds +sleep 120000 print =============== step10 $st = $stPrefix . c3 diff --git a/tests/script/general/stream/metrics_replica1_vnoden.sim b/tests/script/general/stream/metrics_replica1_vnoden.sim index 13cd18adf3..ee071fd681 100644 --- a/tests/script/general/stream/metrics_replica1_vnoden.sim +++ b/tests/script/general/stream/metrics_replica1_vnoden.sim @@ -163,8 +163,8 @@ $st = $stPrefix . as sql create table $st as select count(tbcol) as c from $mt interval(1d) print =============== step13 -print sleep 22 seconds -sleep 32000 +print sleep 120 seconds +sleep 120000 print =============== step14 $st = $stPrefix . c1 diff --git a/tests/script/general/stream/restart_stream.sim b/tests/script/general/stream/restart_stream.sim index aa16934f33..138e2a6e2e 100644 --- a/tests/script/general/stream/restart_stream.sim +++ b/tests/script/general/stream/restart_stream.sim @@ -73,8 +73,8 @@ print =============== step3 sql create table $stt as select count(*) from $tb interval(1d) sql create table $stm as select count(*) from $mt interval(1d) -print sleep 22 seconds -sleep 22000 +print sleep 120 seconds +sleep 120000 sql select * from $stt print select count(*) from $stt ===> $data00 $data01 @@ -152,8 +152,8 @@ print =============== step8 sql create table $stt as select count(*) from $tb interval(1d) sql create table $stm as select count(*) from $mt interval(1d) -print sleep 22 seconds -sleep 22000 +print sleep 120 seconds +sleep 120000 sql select * from $stt sleep 1000 diff --git a/tests/script/general/stream/stream_1.sim b/tests/script/general/stream/stream_1.sim index 2309d341c1..958c877ee5 100644 --- a/tests/script/general/stream/stream_1.sim +++ b/tests/script/general/stream/stream_1.sim @@ -78,8 +78,8 @@ if $rows != 11 then endi print =============== step3 -print sleep 22 seconds -sleep 22000 +print sleep 120 seconds +sleep 120000 sql select * from $st print select * from $st => $data01 if $data01 != 20 then @@ -112,8 +112,8 @@ if $rows != 11 then endi print =============== step7 -print sleep 22 seconds -sleep 22000 +print sleep 120 seconds +sleep 120000 sql select * from $st print select * from $st => $data01 if $data01 != 20 then @@ -155,8 +155,8 @@ if $rows != 12 then endi print =============== step9 -print sleep 22 seconds -sleep 22000 +print sleep 120 seconds +sleep 120000 sql select * from $st print select * from $st => $data01 if $data01 != 200 then @@ -190,8 +190,8 @@ if $rows != 12 then endi print =============== step13 -print sleep 22 seconds -sleep 22000 +print sleep 120 seconds +sleep 120000 sql select * from $st print select * from $st => $data01 if $data01 != 200 then diff --git a/tests/script/general/stream/stream_2.sim b/tests/script/general/stream/stream_2.sim index 13aac98337..057529b427 100644 --- a/tests/script/general/stream/stream_2.sim +++ b/tests/script/general/stream/stream_2.sim @@ -72,8 +72,8 @@ if $rows != 11 then endi print =============== step3 -print sleep 22 seconds -sleep 22000 +print sleep 120 seconds +sleep 120000 sql select * from $st print select * from $st => $data01 if $data01 != 20 then @@ -100,8 +100,8 @@ if $rows != 11 then endi print =============== step7 -print sleep 22 seconds -sleep 22000 +print sleep 120 seconds +sleep 120000 sql select * from $st print select * from $st => $data01 if $data01 != 20 then @@ -143,8 +143,8 @@ if $rows != 12 then endi print =============== step9 -print sleep 22 seconds -sleep 22000 +print sleep 120 seconds +sleep 120000 sql select * from $st print select * from $st => $data01 $data02, $data03 if $data01 != 200 then @@ -178,17 +178,17 @@ if $rows != 12 then endi print =============== step13 -print sleep 22 seconds -sleep 22000 +print sleep 120 seconds +sleep 120000 sql select * from $st print select * from $st => $data01 $data02, $data03 if $data01 != 200 then return -1 endi -if $data02 != NULL then +if $data02 != null then return -1 endi -if $data03 != NULL then +if $data03 != null then return -1 endi diff --git a/tests/script/general/stream/stream_3.sim b/tests/script/general/stream/stream_3.sim index 914ecd8484..88105a77d6 100644 --- a/tests/script/general/stream/stream_3.sim +++ b/tests/script/general/stream/stream_3.sim @@ -79,8 +79,8 @@ $st = $stPrefix . c3 sql create table $st as select count(tbcol2) from $tb interval(1d) print =============== step5 -print sleep 22 seconds -sleep 22000 +print sleep 120 seconds +sleep 120000 print =============== step6 $st = $stPrefix . c1 @@ -173,8 +173,8 @@ $st = $stPrefix . c3 sql create table $st as select count(*), count(tbcol), count(tbcol2) from $tb interval(1d) print =============== step10 -print sleep 22 seconds -sleep 22000 +print sleep 120 seconds +sleep 120000 print =============== step11 #$st = $stPrefix . c3 diff --git a/tests/script/general/stream/stream_restart.sim b/tests/script/general/stream/stream_restart.sim index d5fbef4908..480b23055e 100644 --- a/tests/script/general/stream/stream_restart.sim +++ b/tests/script/general/stream/stream_restart.sim @@ -79,8 +79,8 @@ sleep 1000 system sh/exec.sh -n dnode1 -s start print =============== step4 -print sleep 23 seconds -sleep 23000 +print sleep 120 seconds +sleep 120000 print =============== step5 $i = 1 diff --git a/tests/script/general/stream/table_1.sim b/tests/script/general/stream/table_1.sim index efc7c370ca..f028b1626d 100644 --- a/tests/script/general/stream/table_1.sim +++ b/tests/script/general/stream/table_1.sim @@ -214,8 +214,8 @@ sql select count(tbcol) from $tb where ts < now + 4m interval(1d) group by tgcol step20: print =============== step21 -print sleep 22 seconds -sleep 22000 +print sleep 120 seconds +sleep 120000 print =============== step22 $st = $stPrefix . c1 diff --git a/tests/script/general/stream/table_del.sim b/tests/script/general/stream/table_del.sim index 66fd58308f..ce4065a1a8 100644 --- a/tests/script/general/stream/table_del.sim +++ b/tests/script/general/stream/table_del.sim @@ -71,20 +71,20 @@ print =============== step3 sql drop table $tb print =============== step4 -print sleep 22 seconds -sleep 22000 +print sleep 120 seconds +sleep 120000 print =============== step5 $st = $stPrefix . c3 sql select * from $st print ===> select * from $st print ===> $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 -if $data01 != NULL then +if $data01 != null then return -1 endi -if $data02 != NULL then +if $data02 != null then return -1 endi -if $data03 != NULL then +if $data03 != null then return -1 endi diff --git a/tests/script/general/stream/table_n.sim b/tests/script/general/stream/table_n.sim index e6037c5292..d1b4a87a9e 100644 --- a/tests/script/general/stream/table_n.sim +++ b/tests/script/general/stream/table_n.sim @@ -191,8 +191,8 @@ $st = $stPrefix . as #sql create table $st as select avg(tbcol) as a1, sum(tbcol) as a2, min(tbcol) as a3, max(tbcol) as a4, first(tbcol) as a5, last(tbcol) as a6, stddev(tbcol) as a7, percentile(tbcol, 1) as a8, count(tbcol) as a9, leastsquares(tbcol, 1, 1) as a10 from $tb where ts < now + 4m interval(1d) print =============== step10 -print sleep 22 seconds -sleep 22000 +print sleep 120 seconds +sleep 120000 print =============== step11 $st = $stPrefix . c3 From 06ac5328ec801a07c18e38416e823814ef40934e Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Sat, 6 Jun 2020 16:55:31 +0800 Subject: [PATCH 06/22] td-449: fix invalid write --- src/cq/src/cqMain.c | 7 ++++--- tests/script/general/stream/table_replica1_vnoden.sim | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c index 23455b6f50..edb588b554 100644 --- a/src/cq/src/cqMain.c +++ b/src/cq/src/cqMain.c @@ -175,7 +175,7 @@ void *cqCreate(void *handle, uint64_t uid, int tid, char *sqlStr, STSchema *pSch strcpy(pObj->sqlStr, sqlStr); pObj->pSchema = tdDupSchema(pSchema); - pObj->rowSize = pSchema->tlen; + pObj->rowSize = schemaTLen(pSchema); cTrace("vgId:%d, id:%d CQ:%s is created", pContext->vgId, pObj->tid, pObj->sqlStr); @@ -272,13 +272,14 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) { pBlk->sversion = htonl(pSchema->version); pBlk->padding = 0; + pHead->len = sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + dataRowLen(trow); + pMsg->header.vgId = htonl(pContext->vgId); - pMsg->header.contLen = htonl(size - sizeof(SWalHead)); + pMsg->header.contLen = htonl(pHead->len); pMsg->length = pMsg->header.contLen; pMsg->numOfBlocks = htonl(1); pHead->msgType = TSDB_MSG_TYPE_SUBMIT; - pHead->len = size - sizeof(SWalHead); pHead->version = 0; // write into vnode write queue diff --git a/tests/script/general/stream/table_replica1_vnoden.sim b/tests/script/general/stream/table_replica1_vnoden.sim index 44d4008dbd..e1d5a9babf 100644 --- a/tests/script/general/stream/table_replica1_vnoden.sim +++ b/tests/script/general/stream/table_replica1_vnoden.sim @@ -196,8 +196,8 @@ $st = $stPrefix . as sql create table $st as select count(tbcol) as c from $tb interval(1d) print =============== step16 -print sleep 22 seconds -sleep 22000 +print sleep 120 seconds +sleep 120000 print =============== step17 $st = $stPrefix . c1 From 7ba8c232126b0887b6a3f0454aee2f01f824c4ef Mon Sep 17 00:00:00 2001 From: freemine Date: Sun, 7 Jun 2020 10:23:18 +0800 Subject: [PATCH 07/22] replace gethostbyname with getaddrinfo --- src/util/src/tsocket.c | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/src/util/src/tsocket.c b/src/util/src/tsocket.c index 86ae77b54c..00c8bba94e 100644 --- a/src/util/src/tsocket.c +++ b/src/util/src/tsocket.c @@ -24,25 +24,37 @@ int taosGetFqdn(char *fqdn) { hostname[1023] = '\0'; gethostname(hostname, 1023); - struct hostent* h; - h = gethostbyname(hostname); - if (h != NULL) { - strcpy(fqdn, h->h_name); + struct addrinfo hints = {0}; + struct addrinfo *result = NULL; + + hints.ai_flags = AI_CANONNAME; + + getaddrinfo(hostname, NULL, &hints, &result); + if (result) { + strcpy(fqdn, result->ai_canonname); + freeaddrinfo(result); } else { - uError("failed to get host name(%s)", strerror(errno)); code = -1; } - // to do: free the resources - // free(h); - return code; } uint32_t taosGetIpFromFqdn(const char *fqdn) { - struct hostent * record = gethostbyname(fqdn); - if(record == NULL) return -1; - return ((struct in_addr *)record->h_addr)->s_addr; + struct addrinfo hints = {0}; + struct addrinfo *result = NULL; + + getaddrinfo(fqdn, NULL, &hints, &result); + if (result) { + struct sockaddr *sa = result->ai_addr; + struct sockaddr_in *si = (struct sockaddr_in*)sa; + struct in_addr ia = si->sin_addr; + uint32_t ip = ia.s_addr; + freeaddrinfo(result); + return ip; + } else { + return -1; + } } // Function converting an IP address string to an unsigned int. From 51d9d54ddcea0c364e97cdba357f345bb461792f Mon Sep 17 00:00:00 2001 From: freemine Date: Sun, 7 Jun 2020 11:00:22 +0800 Subject: [PATCH 08/22] allocate enough space to holding null-terminator, especially in case of wchar_t --- src/client/src/tscUtil.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 2bde10f076..caf424080d 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1153,7 +1153,7 @@ SColumnFilterInfo* tscFilterInfoClone(const SColumnFilterInfo* src, int32_t numO for (int32_t j = 0; j < numOfFilters; ++j) { if (pFilter[j].filterstr) { - size_t len = (size_t) pFilter[j].len + 1; + size_t len = (size_t) pFilter[j].len + 1 * TSDB_NCHAR_SIZE; pFilter[j].pz = (int64_t) calloc(1, len); memcpy((char*)pFilter[j].pz, (char*)src[j].pz, (size_t)len); From 6f29506e178b5bc269eed82e358bc70a99cb764e Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 7 Jun 2020 03:21:30 +0000 Subject: [PATCH 09/22] make rpcTcp multi-thread safe --- src/rpc/src/rpcMain.c | 24 ++++--- src/rpc/src/rpcTcp.c | 147 +++++++++++++++++++++++++++-------------- src/util/src/tsocket.c | 24 ++++++- 3 files changed, 130 insertions(+), 65 deletions(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index fe07e616a1..9c90a90fc0 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -805,16 +805,16 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv) { if (pConn == NULL) { tTrace("%s %p, failed to get connection obj(%s)", pRpc->label, (void *)pHead->ahandle, tstrerror(terrno)); return NULL; - } else { - if (rpcIsReq(pHead->msgType)) { - pConn->ahandle = (void *)pHead->ahandle; - sprintf(pConn->info, "%s %p %p", pRpc->label, pConn, pConn->ahandle); - } - } + } rpcLockConn(pConn); - sid = pConn->sid; + if (rpcIsReq(pHead->msgType)) { + pConn->ahandle = (void *)pHead->ahandle; + sprintf(pConn->info, "%s %p %p", pRpc->label, pConn, pConn->ahandle); + } + + sid = pConn->sid; pConn->chandle = pRecv->chandle; pConn->peerIp = pRecv->ip; pConn->peerPort = pRecv->port; @@ -847,10 +847,11 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv) { } static void rpcProcessBrokenLink(SRpcConn *pConn) { + if (pConn == NULL) return; SRpcInfo *pRpc = pConn->pRpc; - tTrace("%s, link is broken", pConn->info); - // pConn->chandle = NULL; + + rpcLockConn(pConn); if (pConn->outType) { SRpcReqContext *pContext = pConn->pContext; @@ -871,7 +872,8 @@ static void rpcProcessBrokenLink(SRpcConn *pConn) { (*(pRpc->cfp))(&rpcMsg); */ } - + + rpcUnlockConn(pConn); rpcCloseConn(pConn); } @@ -885,7 +887,7 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) { // underlying UDP layer does not know it is server or client pRecv->connType = pRecv->connType | pRpc->connType; - if (pRecv->ip == 0 && pConn) { + if (pRecv->ip == 0) { rpcProcessBrokenLink(pConn); return NULL; } diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c index 511a57f3fe..151f5db65f 100644 --- a/src/rpc/src/rpcTcp.c +++ b/src/rpc/src/rpcTcp.c @@ -16,6 +16,7 @@ #include "os.h" #include "tsocket.h" #include "tutil.h" +#include "taoserror.h" #include "rpcLog.h" #include "rpcHead.h" #include "rpcTcp.h" @@ -26,8 +27,9 @@ typedef struct SFdObj { void *signature; - int fd; // TCP socket FD - void *thandle; // handle from upper layer, like TAOS + int fd; // TCP socket FD + int closedByApp; // 1: already closed by App + void *thandle; // handle from upper layer, like TAOS uint32_t ip; uint16_t port; struct SThreadObj *pThreadObj; @@ -71,6 +73,12 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread SThreadObj *pThreadObj; pServerObj = (SServerObj *)calloc(sizeof(SServerObj), 1); + if (pServerObj == NULL) { + tError("TCP:%s no enough memory", label); + terrno = TAOS_SYSTEM_ERROR(errno); + return NULL; + } + pServerObj->ip = ip; pServerObj->port = port; tstrncpy(pServerObj->label, label, sizeof(pServerObj->label)); @@ -79,6 +87,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread pServerObj->pThreadObj = (SThreadObj *)calloc(sizeof(SThreadObj), numOfThreads); if (pServerObj->pThreadObj == NULL) { tError("TCP:%s no enough memory", label); + terrno = TAOS_SYSTEM_ERROR(errno); free(pServerObj); return NULL; } @@ -93,12 +102,14 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread code = pthread_mutex_init(&(pThreadObj->mutex), NULL); if (code < 0) { tError("%s failed to init TCP process data mutex(%s)", label, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); break;; } pThreadObj->pollFd = epoll_create(10); // size does not matter if (pThreadObj->pollFd < 0) { tError("%s failed to create TCP epoll", label); + terrno = TAOS_SYSTEM_ERROR(errno); code = -1; break; } @@ -110,6 +121,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread pthread_attr_destroy(&thattr); if (code != 0) { tError("%s failed to create TCP process data thread(%s)", label, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); break; } @@ -124,6 +136,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread code = pthread_create(&(pServerObj->thread), &thattr, (void *)taosAcceptTcpConnection, (void *)(pServerObj)); pthread_attr_destroy(&thattr); if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); tError("%s failed to create TCP accept thread(%s)", label, strerror(errno)); } } @@ -147,10 +160,12 @@ static void taosStopTcpThread(SThreadObj* pThreadObj) { struct epoll_event event = { .events = EPOLLIN }; eventfd_t fd = eventfd(1, 0); if (fd == -1) { - tError("%s, failed to create eventfd, will call pthread_cancel instead, which may result in data corruption: %s", pThreadObj->label, strerror(errno)); + // failed to create eventfd, call pthread_cancel instead, which may result in data corruption: + tError("%s, failed to create eventfd(%s)", pThreadObj->label, strerror(errno)); pthread_cancel(pThreadObj->thread); } else if (epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) { - tError("%s, failed to call epoll_ctl, will call pthread_cancel instead, which may result in data corruption: %s", pThreadObj->label, strerror(errno)); + // failed to call epoll_ctl, call pthread_cancel instead, which may result in data corruption: + tError("%s, failed to call epoll_ctl(%s)", pThreadObj->label, strerror(errno)); pthread_cancel(pThreadObj->thread); } @@ -211,6 +226,7 @@ static void* taosAcceptTcpConnection(void *arg) { tTrace("%s TCP server socket was shutdown, exiting...", pServerObj->label); break; } + tError("%s TCP accept failure(%s)", pServerObj->label, strerror(errno)); continue; } @@ -254,6 +270,7 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void * if (pthread_mutex_init(&(pThreadObj->mutex), NULL) < 0) { tError("%s failed to init TCP client mutex(%s)", label, strerror(errno)); free(pThreadObj); + terrno = TAOS_SYSTEM_ERROR(errno); return NULL; } @@ -261,6 +278,7 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void * if (pThreadObj->pollFd < 0) { tError("%s failed to create TCP client epoll", label); free(pThreadObj); + terrno = TAOS_SYSTEM_ERROR(errno); return NULL; } @@ -273,6 +291,7 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void * if (code != 0) { close(pThreadObj->pollFd); free(pThreadObj); + terrno = TAOS_SYSTEM_ERROR(errno); tError("%s failed to create TCP read data thread(%s)", label, strerror(errno)); return NULL; } @@ -287,7 +306,7 @@ void taosCleanUpTcpClient(void *chandle) { if (pThreadObj == NULL) return; taosStopTcpThread(pThreadObj); - tTrace (":%s, all connections are cleaned up", pThreadObj->label); + tTrace ("%s, all connections are cleaned up", pThreadObj->label); tfree(pThreadObj); } @@ -318,7 +337,9 @@ void taosCloseTcpConnection(void *chandle) { SFdObj *pFdObj = chandle; if (pFdObj == NULL) return; - taosFreeFdObj(pFdObj); + pFdObj->thandle = NULL; + pFdObj->closedByApp = 1; + shutdown(pFdObj->fd, SHUT_WR); } int taosSendTcpData(uint32_t ip, uint16_t port, void *data, int len, void *chandle) { @@ -334,7 +355,9 @@ static void taosReportBrokenLink(SFdObj *pFdObj) { SThreadObj *pThreadObj = pFdObj->pThreadObj; // notify the upper layer, so it will clean the associated context - if (pFdObj->thandle) { + if (pFdObj->closedByApp == 0) { + shutdown(pFdObj->fd, SHUT_WR); + SRecvInfo recvInfo; recvInfo.msg = NULL; recvInfo.msgLen = 0; @@ -345,9 +368,59 @@ static void taosReportBrokenLink(SFdObj *pFdObj) { recvInfo.chandle = NULL; recvInfo.connType = RPC_CONN_TCP; (*(pThreadObj->processData))(&recvInfo); - } else { - taosFreeFdObj(pFdObj); + } + + taosFreeFdObj(pFdObj); +} + +static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { + SRpcHead rpcHead; + int32_t msgLen, leftLen, retLen, headLen; + char *buffer, *msg; + + SThreadObj *pThreadObj = pFdObj->pThreadObj; + + headLen = taosReadMsg(pFdObj->fd, &rpcHead, sizeof(SRpcHead)); + if (headLen != sizeof(SRpcHead)) { + tTrace("%s %p, read error, headLen:%d", pThreadObj->label, pFdObj->thandle, headLen); + return -1; } + + msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen); + buffer = malloc(msgLen + tsRpcOverhead); + if ( NULL == buffer) { + tError("%s %p, TCP malloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen); + return -1; + } + + msg = buffer + tsRpcOverhead; + leftLen = msgLen - headLen; + retLen = taosReadMsg(pFdObj->fd, msg + headLen, leftLen); + + if (leftLen != retLen) { + tError("%s %p, read error, leftLen:%d retLen:%d", + pThreadObj->label, pFdObj->thandle, leftLen, retLen); + free(buffer); + return -1; + } + + memcpy(msg, &rpcHead, sizeof(SRpcHead)); + + pInfo->msg = msg; + pInfo->msgLen = msgLen; + pInfo->ip = pFdObj->ip; + pInfo->port = pFdObj->port; + pInfo->shandle = pThreadObj->shandle; + pInfo->thandle = pFdObj->thandle;; + pInfo->chandle = pFdObj; + pInfo->connType = RPC_CONN_TCP; + + if (pFdObj->closedByApp) { + free(buffer); + return -1; + } + + return 0; } #define maxEvents 10 @@ -357,7 +430,6 @@ static void *taosProcessTcpData(void *param) { SFdObj *pFdObj; struct epoll_event events[maxEvents]; SRecvInfo recvInfo; - SRpcHead rpcHead; while (1) { int fdNum = epoll_wait(pThreadObj->pollFd, events, maxEvents, -1); @@ -376,51 +448,23 @@ static void *taosProcessTcpData(void *param) { continue; } + if (events[i].events & EPOLLRDHUP) { + tTrace("%s %p, FD RD hang up", pThreadObj->label, pFdObj->thandle); + taosReportBrokenLink(pFdObj); + continue; + } + if (events[i].events & EPOLLHUP) { tTrace("%s %p, FD hang up", pThreadObj->label, pFdObj->thandle); taosReportBrokenLink(pFdObj); continue; } - int32_t headLen = taosReadMsg(pFdObj->fd, &rpcHead, sizeof(SRpcHead)); - if (headLen != sizeof(SRpcHead)) { - tTrace("%s %p, read error, headLen:%d", pThreadObj->label, pFdObj->thandle, headLen); - taosReportBrokenLink(pFdObj); + if (taosReadTcpData(pFdObj, &recvInfo) < 0) { + shutdown(pFdObj->fd, SHUT_WR); continue; } - int32_t msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen); - char *buffer = malloc(msgLen + tsRpcOverhead); - if ( NULL == buffer) { - tError("%s %p, TCP malloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen); - taosReportBrokenLink(pFdObj); - continue; - } - - char *msg = buffer + tsRpcOverhead; - int32_t leftLen = msgLen - headLen; - int32_t retLen = taosReadMsg(pFdObj->fd, msg + headLen, leftLen); - - if (leftLen != retLen) { - tError("%s %p, read error, leftLen:%d retLen:%d", - pThreadObj->label, pFdObj->thandle, leftLen, retLen); - taosReportBrokenLink(pFdObj); - tfree(buffer); - continue; - } - - // tTrace("%s TCP data is received, ip:0x%x:%u len:%d", pThreadObj->label, pFdObj->ip, pFdObj->port, msgLen); - - memcpy(msg, &rpcHead, sizeof(SRpcHead)); - recvInfo.msg = msg; - recvInfo.msgLen = msgLen; - recvInfo.ip = pFdObj->ip; - recvInfo.port = pFdObj->port; - recvInfo.shandle = pThreadObj->shandle; - recvInfo.thandle = pFdObj->thandle;; - recvInfo.chandle = pFdObj; - recvInfo.connType = RPC_CONN_TCP; - pFdObj->thandle = (*(pThreadObj->processData))(&recvInfo); if (pFdObj->thandle == NULL) taosFreeFdObj(pFdObj); } @@ -433,16 +477,20 @@ static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, int fd) { struct epoll_event event; SFdObj *pFdObj = (SFdObj *)calloc(sizeof(SFdObj), 1); - if (pFdObj == NULL) return NULL; + if (pFdObj == NULL) { + return NULL; + } + pFdObj->closedByApp = 0; pFdObj->fd = fd; pFdObj->pThreadObj = pThreadObj; pFdObj->signature = pFdObj; - event.events = EPOLLIN | EPOLLPRI | EPOLLWAKEUP; + event.events = EPOLLIN | EPOLLRDHUP; event.data.ptr = pFdObj; if (epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) { tfree(pFdObj); + terrno = TAOS_SYSTEM_ERROR(errno); return NULL; } @@ -475,13 +523,10 @@ static void taosFreeFdObj(SFdObj *pFdObj) { taosCloseSocket(pFdObj->fd); pThreadObj->numOfFds--; - if (pThreadObj->numOfFds < 0) tError("%s %p, TCP thread:%d, number of FDs is negative!!!", pThreadObj->label, pFdObj->thandle, pThreadObj->threadId); - // remove from the FdObject list - if (pFdObj->prev) { (pFdObj->prev)->next = pFdObj->next; } else { diff --git a/src/util/src/tsocket.c b/src/util/src/tsocket.c index 86ae77b54c..24186cafd0 100644 --- a/src/util/src/tsocket.c +++ b/src/util/src/tsocket.c @@ -40,9 +40,27 @@ int taosGetFqdn(char *fqdn) { } uint32_t taosGetIpFromFqdn(const char *fqdn) { - struct hostent * record = gethostbyname(fqdn); - if(record == NULL) return -1; - return ((struct in_addr *)record->h_addr)->s_addr; + struct addrinfo hints, *servinfo, *p; + struct sockaddr_in *h; + uint32_t ip = -1; + + memset(&hints, 0, sizeof hints); + hints.ai_family = AF_UNSPEC; // use AF_INET6 to force IPv6 + hints.ai_socktype = SOCK_STREAM; + + if (getaddrinfo(fqdn, "http", &hints, &servinfo) != 0) { + uError("failed to get IP from %s(%s)", fqdn, strerror(errno)); + return -1; + } + + // to do: loop through all the results and connect to the first we can + for(p = servinfo; p != NULL; p = p->ai_next) { + h = (struct sockaddr_in *) p->ai_addr; + ip = h->sin_addr.s_addr; + } + + freeaddrinfo(servinfo); // all done with this structure + return ip; } // Function converting an IP address string to an unsigned int. From dad3b9f19c25c2220f6e08a59b39e842a322ee46 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 7 Jun 2020 06:07:15 +0000 Subject: [PATCH 10/22] add a new definition foe label length --- src/inc/taosdef.h | 1 + src/rpc/src/rpcMain.c | 4 +-- src/rpc/src/rpcTcp.c | 64 +++++++++++++++++++++---------------------- src/rpc/src/rpcUdp.c | 7 +++-- src/util/src/tsched.c | 3 +- 5 files changed, 41 insertions(+), 38 deletions(-) diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 29181ed78f..c75fc70d75 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -221,6 +221,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_COUNTRY_LEN 20 #define TSDB_LOCALE_LEN 64 #define TSDB_TIMEZONE_LEN 64 +#define TSDB_LABEL_LEN 8 #define TSDB_FQDN_LEN 128 #define TSDB_EP_LEN (TSDB_FQDN_LEN+6) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 9c90a90fc0..ecbc470945 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -47,7 +47,7 @@ typedef struct { uint16_t localPort; int8_t connType; int index; // for UDP server only, round robin for multiple threads - char label[12]; + char label[TSDB_LABEL_LEN]; char user[TSDB_UNI_LEN]; // meter ID char spi; // security parameter index @@ -88,7 +88,7 @@ typedef struct { } SRpcReqContext; typedef struct SRpcConn { - char info[50];// debug info: label + pConn + ahandle + char info[48];// debug info: label + pConn + ahandle int sid; // session ID uint32_t ownId; // own link ID uint32_t peerId; // peer link ID diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c index 151f5db65f..04a269502e 100644 --- a/src/rpc/src/rpcTcp.c +++ b/src/rpc/src/rpcTcp.c @@ -16,6 +16,7 @@ #include "os.h" #include "tsocket.h" #include "tutil.h" +#include "taosdef.h" #include "taoserror.h" #include "rpcLog.h" #include "rpcHead.h" @@ -46,7 +47,7 @@ typedef struct SThreadObj { int pollFd; int numOfFds; int threadId; - char label[12]; + char label[TSDB_LABEL_LEN]; void *shandle; // handle passed by upper layer during server initialization void *(*processData)(SRecvInfo *pPacket); } SThreadObj; @@ -55,7 +56,7 @@ typedef struct { int fd; uint32_t ip; uint16_t port; - char label[12]; + char label[TSDB_LABEL_LEN]; int numOfThreads; void * shandle; SThreadObj *pThreadObj; @@ -79,6 +80,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread return NULL; } + pServerObj->thread = 0; pServerObj->ip = ip; pServerObj->port = port; tstrncpy(pServerObj->label, label, sizeof(pServerObj->label)); @@ -93,8 +95,14 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread } int code = 0; + pthread_attr_t thattr; + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + pThreadObj = pServerObj->pThreadObj; for (int i = 0; i < numOfThreads; ++i) { + pThreadObj->pollFd = -1; + pThreadObj->thread = 0; pThreadObj->processData = fp; tstrncpy(pThreadObj->label, label, sizeof(pThreadObj->label)); pThreadObj->shandle = shandle; @@ -114,11 +122,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread break; } - pthread_attr_t thattr; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); code = pthread_create(&(pThreadObj->thread), &thattr, taosProcessTcpData, (void *)(pThreadObj)); - pthread_attr_destroy(&thattr); if (code != 0) { tError("%s failed to create TCP process data thread(%s)", label, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); @@ -130,11 +134,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread } if (code == 0) { - pthread_attr_t thattr; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); code = pthread_create(&(pServerObj->thread), &thattr, (void *)taosAcceptTcpConnection, (void *)(pServerObj)); - pthread_attr_destroy(&thattr); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(errno); tError("%s failed to create TCP accept thread(%s)", label, strerror(errno)); @@ -142,38 +142,39 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread } if (code != 0) { - free(pServerObj->pThreadObj); - free(pServerObj); + taosCleanUpTcpServer(pServerObj); pServerObj = NULL; } else { tTrace("%s TCP server is initialized, ip:0x%x port:%hu numOfThreads:%d", label, ip, port, numOfThreads); } + pthread_attr_destroy(&thattr); return (void *)pServerObj; } static void taosStopTcpThread(SThreadObj* pThreadObj) { pThreadObj->stop = true; + eventfd_t fd = -1; - // signal the thread to stop, try graceful method first, - // and use pthread_cancel when failed - struct epoll_event event = { .events = EPOLLIN }; - eventfd_t fd = eventfd(1, 0); - if (fd == -1) { - // failed to create eventfd, call pthread_cancel instead, which may result in data corruption: - tError("%s, failed to create eventfd(%s)", pThreadObj->label, strerror(errno)); - pthread_cancel(pThreadObj->thread); - } else if (epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) { - // failed to call epoll_ctl, call pthread_cancel instead, which may result in data corruption: - tError("%s, failed to call epoll_ctl(%s)", pThreadObj->label, strerror(errno)); - pthread_cancel(pThreadObj->thread); + if (pThreadObj->thread && pThreadObj->pollFd >=0) { + // signal the thread to stop, try graceful method first, + // and use pthread_cancel when failed + struct epoll_event event = { .events = EPOLLIN }; + fd = eventfd(1, 0); + if (fd == -1) { + // failed to create eventfd, call pthread_cancel instead, which may result in data corruption: + tError("%s, failed to create eventfd(%s)", pThreadObj->label, strerror(errno)); + pthread_cancel(pThreadObj->thread); + } else if (epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) { + // failed to call epoll_ctl, call pthread_cancel instead, which may result in data corruption: + tError("%s, failed to call epoll_ctl(%s)", pThreadObj->label, strerror(errno)); + pthread_cancel(pThreadObj->thread); + } } - pthread_join(pThreadObj->thread, NULL); - close(pThreadObj->pollFd); - if (fd != -1) { - close(fd); - } + if (pThreadObj->thread) pthread_join(pThreadObj->thread, NULL); + if (pThreadObj->pollFd >=0) close(pThreadObj->pollFd); + if (fd != -1) close(fd); while (pThreadObj->pHead) { SFdObj *pFdObj = pThreadObj->pHead; @@ -188,9 +189,8 @@ void taosCleanUpTcpServer(void *handle) { SThreadObj *pThreadObj; if (pServerObj == NULL) return; - - shutdown(pServerObj->fd, SHUT_RD); - pthread_join(pServerObj->thread, NULL); + if(pServerObj->fd >=0) shutdown(pServerObj->fd, SHUT_RD); + if(pServerObj->thread) pthread_join(pServerObj->thread, NULL); for (int i = 0; i < pServerObj->numOfThreads; ++i) { pThreadObj = pServerObj->pThreadObj + i; diff --git a/src/rpc/src/rpcUdp.c b/src/rpc/src/rpcUdp.c index a8811f4136..7e2fe0db61 100644 --- a/src/rpc/src/rpcUdp.c +++ b/src/rpc/src/rpcUdp.c @@ -18,6 +18,7 @@ #include "tsystem.h" #include "ttimer.h" #include "tutil.h" +#include "taosdef.h" #include "rpcLog.h" #include "rpcUdp.h" #include "rpcHead.h" @@ -33,7 +34,7 @@ typedef struct { int fd; uint16_t port; // peer port uint16_t localPort; // local port - char label[12]; // copy from udpConnSet; + char label[TSDB_LABEL_LEN]; // copy from udpConnSet; pthread_t thread; void *hash; void *shandle; // handle passed by upper layer during server initialization @@ -49,7 +50,7 @@ typedef struct { uint16_t port; // local Port void *shandle; // handle passed by upper layer during server initialization int threads; - char label[12]; + char label[TSDB_LABEL_LEN]; void *(*fp)(SRecvInfo *pPacket); SUdpConn udpConn[]; } SUdpConnSet; @@ -93,7 +94,7 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads } struct sockaddr_in sin; - unsigned int addrlen = sizeof(sin); + unsigned int addrlen = sizeof(sin); if (getsockname(pConn->fd, (struct sockaddr *)&sin, &addrlen) == 0 && sin.sin_family == AF_INET && addrlen == sizeof(sin)) { pConn->localPort = (uint16_t)ntohs(sin.sin_port); diff --git a/src/util/src/tsched.c b/src/util/src/tsched.c index 25893969e4..898ab70876 100644 --- a/src/util/src/tsched.c +++ b/src/util/src/tsched.c @@ -14,6 +14,7 @@ */ #include "os.h" +#include "taosdef.h" #include "tulog.h" #include "tsched.h" #include "ttimer.h" @@ -21,7 +22,7 @@ #define DUMP_SCHEDULER_TIME_WINDOW 30000 //every 30sec, take a snap shot of task queue. typedef struct { - char label[16]; + char label[TSDB_LABEL_LEN]; tsem_t emptySem; tsem_t fullSem; pthread_mutex_t queueMutex; From 85261cbfed5097a8e863a05b1a2e16650cc132cd Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 7 Jun 2020 06:52:42 +0000 Subject: [PATCH 11/22] initialize secret --- src/dnode/src/dnodePeer.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/dnode/src/dnodePeer.c b/src/dnode/src/dnodePeer.c index ea3af08d71..77a7b0dff4 100644 --- a/src/dnode/src/dnodePeer.c +++ b/src/dnode/src/dnodePeer.c @@ -113,6 +113,7 @@ static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcIpSet *pIpSet) { } int32_t dnodeInitClient() { + char secret[TSDB_KEY_LEN] = "secret"; SRpcInit rpcInit; memset(&rpcInit, 0, sizeof(rpcInit)); rpcInit.label = "DND-C"; @@ -123,7 +124,7 @@ int32_t dnodeInitClient() { rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.user = "t"; rpcInit.ckey = "key"; - rpcInit.secret = "secret"; + rpcInit.secret = secret; tsDnodeClientRpc = rpcOpen(&rpcInit); if (tsDnodeClientRpc == NULL) { From aca961cceaa39152396223ad006b0416569c819d Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 7 Jun 2020 13:53:13 +0000 Subject: [PATCH 12/22] [TD-537] the heartbeat sometimes crashes, limit size to 20 --- src/mnode/src/mnodeProfile.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mnode/src/mnodeProfile.c b/src/mnode/src/mnodeProfile.c index 541fc9cbd8..ed0e717c43 100644 --- a/src/mnode/src/mnodeProfile.c +++ b/src/mnode/src/mnodeProfile.c @@ -264,13 +264,13 @@ static int32_t mnodeRetrieveConns(SShowObj *pShow, char *data, int32_t rows, voi // not thread safe, need optimized int32_t mnodeSaveQueryStreamList(SConnObj *pConn, SCMHeartBeatMsg *pHBMsg) { pConn->numOfQueries = htonl(pHBMsg->numOfQueries); - if (pConn->numOfQueries > 0) { + if (pConn->numOfQueries > 0 && pConn->numOfQueries < 20) { pConn->pQueries = calloc(sizeof(SQueryDesc), pConn->numOfQueries); memcpy(pConn->pQueries, pHBMsg->pData, pConn->numOfQueries * sizeof(SQueryDesc)); } pConn->numOfStreams = htonl(pHBMsg->numOfStreams); - if (pConn->numOfStreams > 0) { + if (pConn->numOfStreams > 0 && pConn->numOfStreams < 20) { pConn->pStreams = calloc(sizeof(SStreamDesc), pConn->numOfStreams); memcpy(pConn->pStreams, pHBMsg->pData + pConn->numOfQueries * sizeof(SQueryDesc), pConn->numOfStreams * sizeof(SStreamDesc)); From c9bcdf418521b41e0abc55ae72a392d8beef4885 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 7 Jun 2020 13:55:50 +0000 Subject: [PATCH 13/22] [TD-527] script error while change errno --- tests/script/general/http/grafana.sim | 4 ++-- tests/script/general/http/restful_full.sim | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/script/general/http/grafana.sim b/tests/script/general/http/grafana.sim index a08c07a841..dac7552edf 100644 --- a/tests/script/general/http/grafana.sim +++ b/tests/script/general/http/grafana.sim @@ -66,7 +66,7 @@ endi system_content curl 127.0.0.1:6020/grafana/login/xx/xx/ print 3-> $system_content -if $system_content != @{"status":"error","code":1000,"desc":"invalid user"}@ then +if $system_content != @{"status":"error","code":1000,"desc":"mnode invalid user"}@ then return -1 endi @@ -78,7 +78,7 @@ endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'show databases' 127.0.0.1:6020/grafana/login/1/root/1/ print 5-> $system_content -if $system_content != @{"status":"error","code":1000,"desc":"invalid user"}@ then +if $system_content != @{"status":"error","code":1000,"desc":"mnode invalid user"}@ then return -1 endi diff --git a/tests/script/general/http/restful_full.sim b/tests/script/general/http/restful_full.sim index 0ae6873503..7194f7cbe0 100644 --- a/tests/script/general/http/restful_full.sim +++ b/tests/script/general/http/restful_full.sim @@ -93,7 +93,7 @@ endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'create database d1' 127.0.0.1:6020/rest/sql print 13-> $system_content -if $system_content != @{"status":"error","code":1000,"desc":"database aleady exist"}@ then +if $system_content != @{"status":"error","code":1000,"desc":"mnode database aleady exist"}@ then return -1 endi @@ -126,7 +126,7 @@ endi #18 system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' show tables;' 127.0.0.1:6020/rest/sql print 18-> $system_content -if $system_content != @{"status":"error","code":1000,"desc":"db not selected"}@ then +if $system_content != @{"status":"error","code":1000,"desc":"mnode db not selected"}@ then return -1 endi @@ -147,7 +147,7 @@ print =============== step3 - db system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' select * from d1.t1;' 127.0.0.1:6020/rest/sql print 21-> $system_content -if $system_content != @{"status":"error","code":1000,"desc":"invalid table id"}@ then +if $system_content != @{"status":"error","code":1000,"desc":"mnode invalid table id"}@ then return -1 endi From b907acf13a9a4d1465121d074ee6ff3dc6b4b465 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 7 Jun 2020 15:39:40 +0000 Subject: [PATCH 14/22] Bump websocket-extensions in /src/connector/grafana/tdengine Bumps [websocket-extensions](https://github.com/faye/websocket-extensions-node) from 0.1.3 to 0.1.4. - [Release notes](https://github.com/faye/websocket-extensions-node/releases) - [Changelog](https://github.com/faye/websocket-extensions-node/blob/master/CHANGELOG.md) - [Commits](https://github.com/faye/websocket-extensions-node/compare/0.1.3...0.1.4) Signed-off-by: dependabot[bot] --- src/connector/grafana/tdengine/package-lock.json | 6 +++--- src/connector/grafana/tdengine/yarn.lock | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/connector/grafana/tdengine/package-lock.json b/src/connector/grafana/tdengine/package-lock.json index f8d2df1150..7c8853b99c 100644 --- a/src/connector/grafana/tdengine/package-lock.json +++ b/src/connector/grafana/tdengine/package-lock.json @@ -3724,9 +3724,9 @@ } }, "websocket-extensions": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/websocket-extensions/-/websocket-extensions-0.1.3.tgz", - "integrity": "sha512-nqHUnMXmBzT0w570r2JpJxfiSD1IzoI+HGVdd3aZ0yNi3ngvQ4jv1dtHt5VGxfI2yj5yqImPhOK4vmIh2xMbGg==", + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/websocket-extensions/-/websocket-extensions-0.1.4.tgz", + "integrity": "sha512-OqedPIGOfsDlo31UNwYbCFMSaO9m9G/0faIHj5/dZFDMFqPTcx6UwqyOy3COEaEOg/9VsGIpdqn62W5KhoKSpg==", "dev": true }, "whatwg-encoding": { diff --git a/src/connector/grafana/tdengine/yarn.lock b/src/connector/grafana/tdengine/yarn.lock index fe7e8122ec..f785e4e478 100644 --- a/src/connector/grafana/tdengine/yarn.lock +++ b/src/connector/grafana/tdengine/yarn.lock @@ -2839,8 +2839,9 @@ websocket-driver@>=0.5.1: websocket-extensions ">=0.1.1" websocket-extensions@>=0.1.1: - version "0.1.3" - resolved "https://registry.yarnpkg.com/websocket-extensions/-/websocket-extensions-0.1.3.tgz#5d2ff22977003ec687a4b87073dfbbac146ccf29" + version "0.1.4" + resolved "https://registry.yarnpkg.com/websocket-extensions/-/websocket-extensions-0.1.4.tgz#7f8473bc839dfd87608adb95d7eb075211578a42" + integrity sha512-OqedPIGOfsDlo31UNwYbCFMSaO9m9G/0faIHj5/dZFDMFqPTcx6UwqyOy3COEaEOg/9VsGIpdqn62W5KhoKSpg== whatwg-encoding@^1.0.1: version "1.0.3" From 519cd6aee706726246c6f2cda8c560db92368aa1 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Mon, 8 Jun 2020 09:44:25 +0800 Subject: [PATCH 15/22] td-449: set parse finsish flag in select statement --- src/client/src/tscAsync.c | 6 ++++-- src/client/src/tscSQLParser.c | 3 ++- src/client/src/tscStream.c | 19 ++++++------------- tests/script/general/stream/new_stream.sim | 22 +++++++++++----------- 4 files changed, 23 insertions(+), 27 deletions(-) diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 3723c0cf65..a9c170f197 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -517,8 +517,10 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { if (pSql->pStream) { tscTrace("%p stream:%p meta is updated, start new query, command:%d", pSql, pSql->pStream, pSql->cmd.command); - tsParseSql(pSql, false); - sem_post(&pSql->rspSem); + if (!pSql->cmd.parseFinished) { + tsParseSql(pSql, false); + sem_post(&pSql->rspSem); + } return; } else { tscTrace("%p get tableMeta successfully", pSql); diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 68980bef98..1584b52e9f 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -515,8 +515,9 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { if (ret != 0) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } - } + } + pCmd->parseFinished = 1; return TSDB_CODE_SUCCESS; // do not build query message here } diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index f13e99d7b3..c4413f8541 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -78,30 +78,23 @@ static void tscProcessStreamLaunchQuery(SSchedMsg *pMsg) { int code = tscGetTableMeta(pSql, pTableMetaInfo); pSql->res.code = code; - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) return; - if (code == 0 && UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { code = tscGetSTableVgroupInfo(pSql, 0); pSql->res.code = code; - - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) return; } - tscTansformSQLFuncForSTableQuery(pQueryInfo); - // failed to get meter/metric meta, retry in 10sec. if (code != TSDB_CODE_SUCCESS) { int64_t retryDelayTime = tscGetRetryDelayTime(pStream->slidingTime, pStream->precision); tscError("%p stream:%p,get metermeta failed, retry in %" PRId64 "ms", pStream->pSql, pStream, retryDelayTime); - tscSetRetryTimer(pStream, pSql, retryDelayTime); - return; + + } else { + tscTansformSQLFuncForSTableQuery(pQueryInfo); + tscTrace("%p stream:%p start stream query on:%s", pSql, pStream, pTableMetaInfo->name); + tscDoQuery(pStream->pSql); + tscIncStreamExecutionCount(pStream); } - - tscTrace("%p stream:%p start stream query on:%s", pSql, pStream, pTableMetaInfo->name); - tscDoQuery(pStream->pSql); - - tscIncStreamExecutionCount(pStream); } static void tscProcessStreamTimer(void *handle, void *tmrId) { diff --git a/tests/script/general/stream/new_stream.sim b/tests/script/general/stream/new_stream.sim index 8aa0a89870..abc06faaef 100644 --- a/tests/script/general/stream/new_stream.sim +++ b/tests/script/general/stream/new_stream.sim @@ -1,12 +1,12 @@ -system sh/stop_dnodes.sh +#system sh/stop_dnodes.sh -system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode1 -c tableMetaKeepTimer -v 10 -system sh/exec.sh -n dnode1 -s start +#system sh/deploy.sh -n dnode1 -i 1 +#system sh/cfg.sh -n dnode1 -c walLevel -v 0 +#system sh/cfg.sh -n dnode1 -c tableMetaKeepTimer -v 10 +#system sh/exec.sh -n dnode1 -s start -sleep 3000 +#sleep 3000 sql connect print ======================== dnode1 start @@ -56,14 +56,14 @@ print $data00 $data01 $data02 $data03 sql create table $st as select count(*), count(tbcol), count(tbcol2) from $mt interval(10s) print =============== step3 -print sleep 22 seconds -sleep 22000 +print sleep 120 seconds +sleep 120000 print =============== step4 sql select * from $st print $st ==> $rows1 $data00 $data01 $data02 $data03 -if $data13 >= 51 then +if $data03 >= 51 then return -1 endi @@ -90,8 +90,8 @@ while $i < $tbNum endw print =============== step6 -print sleep 22 seconds -sleep 22000 +print sleep 120 seconds +sleep 120000 print =============== step7 From b08c2b97a16e9d299e52dcf6b02c9bd6a5e52a08 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 8 Jun 2020 11:13:08 +0800 Subject: [PATCH 16/22] [TD-545] scripts --- tests/script/general/parser/limit2.sim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/script/general/parser/limit2.sim b/tests/script/general/parser/limit2.sim index 61f817644a..5f71232585 100644 --- a/tests/script/general/parser/limit2.sim +++ b/tests/script/general/parser/limit2.sim @@ -69,6 +69,8 @@ print ====== tables created print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT + +return sleep 3000 system sh/exec.sh -n dnode1 -s start print ================== server restart completed From fb1f837d280d69fa10a0d277ed410102fffb1e32 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 8 Jun 2020 11:16:13 +0800 Subject: [PATCH 17/22] update script to support wal param --- .../perftest-scripts/perftest-csv2png.gnuplot | 2 +- tests/perftest-scripts/perftest-daily.sh | 74 +++++++++++++++---- tests/perftest-scripts/perftest-taosdemo.sh | 72 +++++++++++------- .../perftest-tsdb-compare-13d.sh | 12 +-- .../perftest-tsdb-compare-1d.sh | 12 +-- tests/perftest-scripts/runreal-13d-csv.sh | 2 +- tests/perftest-scripts/runreal-1d-csv.sh | 2 +- .../perftest-scripts/taosdemo-csv2png.gnuplot | 10 +-- 8 files changed, 124 insertions(+), 62 deletions(-) diff --git a/tests/perftest-scripts/perftest-csv2png.gnuplot b/tests/perftest-scripts/perftest-csv2png.gnuplot index 9c34ebe403..eabab268ca 100644 --- a/tests/perftest-scripts/perftest-csv2png.gnuplot +++ b/tests/perftest-scripts/perftest-csv2png.gnuplot @@ -2,7 +2,7 @@ reset set terminal png -set title "Performance Test Report" font ",20" +set title filename font ",20" set ylabel "Time in Seconds" diff --git a/tests/perftest-scripts/perftest-daily.sh b/tests/perftest-scripts/perftest-daily.sh index db9de11381..b44387047c 100755 --- a/tests/perftest-scripts/perftest-daily.sh +++ b/tests/perftest-scripts/perftest-daily.sh @@ -16,9 +16,26 @@ function echoInfo { local args="$@"; white_brackets $(green_printf "INFO") && function echoWarn { local args="$@"; echo "$(white_brackets "$(yellow_printf "WARN")" && echo " ${args}";)" 1>&2; } # function echoError { local args="$@"; echo "$(white_brackets "$(red_printf "ERROR")" && echo " ${args}";)" 1>&2; } # -function set-Wal { +function setMaxConnections { + echo "/etc/taos/taos.cfg maxConnection will be set to $1" + + hasText=`grep "maxConnections" /etc/taos/taos.cfg` + if [[ -z "$hasText" ]]; then + echo "maxConnections $1" >> /etc/taos/taos.cfg + else + sed -i 's/^maxConnections.*$/maxConnections '"$1"'/g' /etc/taos/taos.cfg + fi +} + +function setWal { echo "/etc/taos/taos.cfg walLevel will be set to $1" - sed -i 's/^walLevel.*$/walLevel '"$1"'/g' /etc/taos/taos.cfg + + hasText=`grep "walLevel" /etc/taos/taos.cfg` + if [[ -z "$hasText" ]]; then + echo "walLevel $1" >> /etc/taos/taos.cfg + else + sed -i 's/^walLevel.*$/walLevel '"$1"'/g' /etc/taos/taos.cfg + fi } function collectSysInfo { @@ -70,15 +87,25 @@ function sendReport { mimebody="MIME-Version: 1.0\nContent-Type: text/html; charset=utf-8\n" echo -e "to: ${receiver}\nsubject: Perf test report ${today}, commit ID: ${LOCAL_COMMIT}\n" | \ - (cat - && uuencode perftest-1d-$today.log perftest-1d-$today.log)| \ - (cat - && uuencode perftest-1d-report.csv perftest-1d-report-$today.csv) | \ - (cat - && uuencode perftest-1d-report.png perftest-1d-report-$today.png) | \ - (cat - && uuencode perftest-13d-$today.log perftest-13d-$today.log)| \ - (cat - && uuencode perftest-13d-report.csv perftest-13d-report-$today.csv) | \ - (cat - && uuencode perftest-13d-report.png perftest-13d-report-$today.png) | \ - (cat - && uuencode taosdemo-$today.log taosdemo-$today.log) | \ - (cat - && uuencode taosdemo-report.csv taosdemo-report-$today.csv) | \ - (cat - && uuencode taosdemo-report.png taosdemo-report-$today.png) | \ + (cat - && uuencode perftest-1d-wal1-$today.log perftest-1d-wal1-$today.log)| \ + (cat - && uuencode perftest-1d-wal1-report.csv perftest-1d-wal1-report-$today.csv) | \ + (cat - && uuencode perftest-1d-wal1-report.png perftest-1d-wal1-report-$today.png) | \ + (cat - && uuencode perftest-13d-wal1-$today.log perftest-13d-wal1-$today.log)| \ + (cat - && uuencode perftest-13d-wal1-report.csv perftest-13d-wal1-report-$today.csv) | \ + (cat - && uuencode perftest-13d-wal1-report.png perftest-13d-wal1-report-$today.png) | \ + (cat - && uuencode taosdemo-wal1-$today.log taosdemo-wal1-$today.log) | \ + (cat - && uuencode taosdemo-wal1-report.csv taosdemo-wal1-report-$today.csv) | \ + (cat - && uuencode taosdemo-rps-wal1-report.csv taosdemo-rps-wal1-report-$today.csv) | \ + (cat - && uuencode taosdemo-wal1-report.png taosdemo-wal1-report-$today.png) | \ + (cat - && uuencode perftest-1d-wal2-$today.log perftest-1d-wal2-$today.log)| \ + (cat - && uuencode perftest-1d-wal2-report.csv perftest-1d-wal2-report-$today.csv) | \ + (cat - && uuencode perftest-1d-wal2-report.png perftest-1d-wal2-report-$today.png) | \ + (cat - && uuencode perftest-13d-wal2-$today.log perftest-13d-wal2-$today.log)| \ + (cat - && uuencode perftest-13d-wal2-report.csv perftest-13d-wal2-report-$today.csv) | \ + (cat - && uuencode perftest-13d-wal2-report.png perftest-13d-wal2-report-$today.png) | \ + (cat - && uuencode taosdemo-wal2-$today.log taosdemo-wal2-$today.log) | \ + (cat - && uuencode taosdemo-wal2-report.csv taosdemo-wal2-report-$today.csv) | \ + (cat - && uuencode taosdemo-rps-wal2-report.csv taosdemo-rps-wal2-report-$today.csv) | \ (cat - && uuencode sysinfo.log sysinfo.txt) | \ (cat - && uuencode taos.cfg taos-cfg-$today.txt) | \ ssmtp "${receiver}" @@ -91,17 +118,34 @@ echo -e "cron-ran-at-${today}" >> cron.log echoInfo "Build TDengine" buildTDengine -set-Wal "2" +############################ +setMaxConnections 100 + +############################ +setWal "2" cd /root -./perftest-tsdb-compare-1d.sh +./perftest-tsdb-compare-1d.sh "wal2" cd /root -./perftest-tsdb-compare-13d.sh +./perftest-tsdb-compare-13d.sh "wal2" cd /root -./perftest-taosdemo.sh +./perftest-taosdemo.sh "wal2" +############################# +setWal "1" + +cd /root +./perftest-tsdb-compare-1d.sh "wal1" + +cd /root +./perftest-tsdb-compare-13d.sh "wal1" + +cd /root +./perftest-taosdemo.sh "wal1" + +############################# collectSysInfo echoInfo "Send Report" diff --git a/tests/perftest-scripts/perftest-taosdemo.sh b/tests/perftest-scripts/perftest-taosdemo.sh index 511ec22fec..0dd9a0572a 100755 --- a/tests/perftest-scripts/perftest-taosdemo.sh +++ b/tests/perftest-scripts/perftest-taosdemo.sh @@ -1,20 +1,20 @@ #!/bin/bash -# Coloured Echoes # -function red_echo { echo -e "\033[31m$@\033[0m"; } # -function green_echo { echo -e "\033[32m$@\033[0m"; } # -function yellow_echo { echo -e "\033[33m$@\033[0m"; } # -function white_echo { echo -e "\033[1;37m$@\033[0m"; } # -# Coloured Printfs # -function red_printf { printf "\033[31m$@\033[0m"; } # -function green_printf { printf "\033[32m$@\033[0m"; } # -function yellow_printf { printf "\033[33m$@\033[0m"; } # -function white_printf { printf "\033[1;37m$@\033[0m"; } # -# Debugging Outputs # -function white_brackets { local args="$@"; white_printf "["; printf "${args}"; white_printf "]"; } # -function echoInfo { local args="$@"; white_brackets $(green_printf "INFO") && echo " ${args}"; } # -function echoWarn { local args="$@"; echo "$(white_brackets "$(yellow_printf "WARN")" && echo " ${args}";)" 1>&2; } # -function echoError { local args="$@"; echo "$(white_brackets "$(red_printf "ERROR")" && echo " ${args}";)" 1>&2; } # +# Coloured Echoes +function red_echo { echo -e "\033[31m$@\033[0m"; } +function green_echo { echo -e "\033[32m$@\033[0m"; } +function yellow_echo { echo -e "\033[33m$@\033[0m"; } +function white_echo { echo -e "\033[1;37m$@\033[0m"; } +# Coloured Printfs +function red_printf { printf "\033[31m$@\033[0m"; } +function green_printf { printf "\033[32m$@\033[0m"; } +function yellow_printf { printf "\033[33m$@\033[0m"; } +function white_printf { printf "\033[1;37m$@\033[0m"; } +# Debugging Outputs +function white_brackets { local args="$@"; white_printf "["; printf "${args}"; white_printf "]"; } +function echoInfo { local args="$@"; white_brackets $(green_printf "INFO") && echo " ${args}"; } +function echoWarn { local args="$@"; echo "$(white_brackets "$(yellow_printf "WARN")" && echo " ${args}";)" 1>&2; } +function echoError { local args="$@"; echo "$(white_brackets "$(red_printf "ERROR")" && echo " ${args}";)" 1>&2; } function restartTaosd { systemctl stop taosd @@ -32,39 +32,57 @@ function runCreateTableOnly { echoInfo "Restart Taosd" restartTaosd - /usr/bin/time -f "Total: %e" -o totaltime.out bash -c "yes | taosdemo -n 0 2>&1 | tee taosdemo-$today.log" - demoTableOnly=`grep "Total:" totaltime.out|awk '{print $2}'` + /usr/bin/time -f "Total: %e" -o totaltime.out bash -c "yes | taosdemo -n 0 2>&1 | tee taosdemo-$1-$today.log" + demoCreateTableOnly=`grep "Total:" totaltime.out|awk '{print $2}'` +} + +function runDeleteTableOnly { + echoInfo "Restart Taosd" + restartTaosd + + /usr/bin/time -f "Total: %e" -o totaltime.out bash -c "yes | taosdemo -t 0 -D 1 2>&1 | tee taosdemo-$1-$today.log" + demoDeleteTableOnly=`grep "Total:" totaltime.out|awk '{print $2}'` } function runCreateTableThenInsert { echoInfo "Restart Taosd" restartTaosd - /usr/bin/time -f "Total: %e" -o totaltime.out bash -c "yes | taosdemo 2>&1 | tee -a taosdemo-$today.log" + /usr/bin/time -f "Total: %e" -o totaltime.out bash -c "yes | taosdemo 2>&1 | tee -a taosdemo-$1-$today.log" demoTableAndInsert=`grep "Total:" totaltime.out|awk '{print $2}'` - demoRPS=`grep "records\/second" taosdemo-$today.log | tail -n1 | awk '{print $13}'` + demoRPS=`grep "records\/second" taosdemo-$1-$today.log | tail -n1 | awk '{print $13}'` } function generateTaosdemoPlot { - echo "${today}, demoTableOnly: ${demoTableOnly}, demoTableAndInsert: ${demoTableAndInsert}" | tee -a taosdemo-$today.log - echo "${today}, ${demoTableOnly}, ${demoTableAndInsert}, ${demoRPS}" >> taosdemo-report.csv + echo "${today} $1, demoCreateTableOnly: ${demoCreateTableOnly}, demoDeleteTableOnly: ${demoDeleteTableOnly}, demoTableAndInsert: ${demoTableAndInsert}" | tee -a taosdemo-$today.log + echo "${today}, ${demoCreateTableOnly}, ${demoDeleteTableOnly}, ${demoTableAndInsert}">> taosdemo-$1-report.csv + echo "${today}, ${demoRPS}" >> taosdemo-rps-$1-report.csv - csvLines=`cat taosdemo-report.csv | wc -l` + csvLines=`cat taosdemo-$1-report.csv | wc -l` if [ "$csvLines" -gt "10" ]; then - sed -i '1d' taosdemo-report.csv + sed -i '1d' taosdemo-$1-report.csv fi - gnuplot -p taosdemo-csv2png.gnuplot + csvLines=`cat taosdemo-rps-$1-report.csv | wc -l` + + if [ "$csvLines" -gt "10" ]; then + sed -i '1d' taosdemo-rps-$1-report.csv + fi + + gnuplot -e "filename='taosdemo-$1-report'" -p taosdemo-csv2png.gnuplot + gnuplot -e "filename='taosdemo-rps-$1-report'" -p taosdemo-rps-csv2png.gnuplot } today=`date +"%Y%m%d"` cd /root echoInfo "Test Create Table Only " -runCreateTableOnly +runCreateTableOnly $1 echoInfo "Test Create Table then Insert data" -runCreateTableThenInsert +runDeleteTableOnly $1 +echoInfo "Test Create Table then Insert data" +runCreateTableThenInsert $1 echoInfo "Generate plot for taosdemo" -generateTaosdemoPlot +generateTaosdemoPlot $1 echoInfo "End of TaosDemo Test" diff --git a/tests/perftest-scripts/perftest-tsdb-compare-13d.sh b/tests/perftest-scripts/perftest-tsdb-compare-13d.sh index 4b3ed6818c..8b326d2d41 100755 --- a/tests/perftest-scripts/perftest-tsdb-compare-13d.sh +++ b/tests/perftest-scripts/perftest-tsdb-compare-13d.sh @@ -33,26 +33,26 @@ function runPerfTest13d { restartTaosd cd /home/taos/tliu/timeseriesdatabase-comparisons/build/tsdbcompare - ./runreal-13d-csv.sh 2>&1 | tee /root/perftest-13d-$today.log + ./runreal-13d-csv.sh $1 2>&1 | tee /root/perftest-13d-$1-$today.log } function generatePerfPlot13d { cd /root - csvLines=`cat perftest-13d-report.csv | wc -l` + csvLines=`cat perftest-13d-$1-report.csv | wc -l` if [ "$csvLines" -gt "10" ]; then - sed -i '1d' perftest-13d-report.csv + sed -i '1d' perftest-13d-$1-report.csv fi - gnuplot -e "filename='perftest-13d-report'" -p perftest-csv2png.gnuplot + gnuplot -e "filename='perftest-13d-$1-report'" -p perftest-csv2png.gnuplot } today=`date +"%Y%m%d"` cd /root echoInfo "run Performance Test with 13 days data" -runPerfTest13d +runPerfTest13d $1 echoInfo "Generate plot of 13 days data" -generatePerfPlot13d +generatePerfPlot13d $1 echoInfo "End of TSDB-Compare 13-days-data Test" diff --git a/tests/perftest-scripts/perftest-tsdb-compare-1d.sh b/tests/perftest-scripts/perftest-tsdb-compare-1d.sh index ebe34cde72..0931480e21 100755 --- a/tests/perftest-scripts/perftest-tsdb-compare-1d.sh +++ b/tests/perftest-scripts/perftest-tsdb-compare-1d.sh @@ -33,26 +33,26 @@ function runPerfTest1d { restartTaosd cd /home/taos/tliu/timeseriesdatabase-comparisons/build/tsdbcompare - ./runreal-1d-csv.sh 2>&1 | tee /root/perftest-1d-$today.log + ./runreal-1d-csv.sh $1 2>&1 | tee /root/perftest-1d-$1-$today.log } function generatePerfPlot1d { cd /root - csvLines=`cat perftest-1d-report.csv | wc -l` + csvLines=`cat perftest-1d-$1-report.csv | wc -l` if [ "$csvLines" -gt "10" ]; then - sed -i '2d' perftest-1d-report.csv + sed -i '2d' perftest-1d-$1-report.csv fi - gnuplot -e "filename='perftest-1d-report'" -p perftest-csv2png.gnuplot + gnuplot -e "filename='perftest-1d-$1-report'" -p perftest-csv2png.gnuplot } today=`date +"%Y%m%d"` cd /root echoInfo "run Performance Test with 1 day data" -runPerfTest1d +runPerfTest1d $1 echoInfo "Generate plot of 1 day data" -generatePerfPlot1d +generatePerfPlot1d $1 echoInfo "End of TSDB-Compare 1-day-data Test" diff --git a/tests/perftest-scripts/runreal-13d-csv.sh b/tests/perftest-scripts/runreal-13d-csv.sh index ff7ce41d4b..48ef01e6a0 100755 --- a/tests/perftest-scripts/runreal-13d-csv.sh +++ b/tests/perftest-scripts/runreal-13d-csv.sh @@ -143,7 +143,7 @@ echo "------------------------------------------------------" echo today=`date +"%Y%m%d"` -echo "${today}, ${TDWTM}, ${TDQ1}, ${TDQ2}, ${TDQ3}, ${TDQ4}" >> /root/perftest-13d-report.csv +echo "${today}, ${TDWTM}, ${TDQ1}, ${TDQ2}, ${TDQ3}, ${TDQ4}" >> /root/perftest-13d-$1-report.csv #bulk_query_gen/bulk_query_gen -format influx-http -query-type 1-host-1-hr -scale-var 10 -queries 1000 | query_benchmarker_influxdb/query_benchmarker_influxdb -urls="http://172.26.89.231:8086" #bulk_query_gen/bulk_query_gen -format tdengine -query-type 1-host-1-hr -scale-var 10 -queries 1000 | query_benchmarker_tdengine/query_benchmarker_tdengine -urls="http://172.26.89.231:6020" diff --git a/tests/perftest-scripts/runreal-1d-csv.sh b/tests/perftest-scripts/runreal-1d-csv.sh index 5cd113aadf..20c8b4138c 100755 --- a/tests/perftest-scripts/runreal-1d-csv.sh +++ b/tests/perftest-scripts/runreal-1d-csv.sh @@ -143,7 +143,7 @@ echo "------------------------------------------------------" echo today=`date +"%Y%m%d"` -echo "${today}, ${TDWTM}, ${TDQ1}, ${TDQ2}, ${TDQ3}, ${TDQ4}" >> /root/perftest-1d-report.csv +echo "${today}, ${TDWTM}, ${TDQ1}, ${TDQ2}, ${TDQ3}, ${TDQ4}" >> /root/perftest-1d-$1-report.csv #bulk_query_gen/bulk_query_gen -format influx-http -query-type 1-host-1-hr -scale-var 10 -queries 1000 | query_benchmarker_influxdb/query_benchmarker_influxdb -urls="http://172.26.89.231:8086" #bulk_query_gen/bulk_query_gen -format tdengine -query-type 1-host-1-hr -scale-var 10 -queries 1000 | query_benchmarker_tdengine/query_benchmarker_tdengine -urls="http://172.26.89.231:6020" diff --git a/tests/perftest-scripts/taosdemo-csv2png.gnuplot b/tests/perftest-scripts/taosdemo-csv2png.gnuplot index 9fcd4bb3d9..7cdcb84fce 100644 --- a/tests/perftest-scripts/taosdemo-csv2png.gnuplot +++ b/tests/perftest-scripts/taosdemo-csv2png.gnuplot @@ -2,7 +2,7 @@ reset set terminal png -set title "TaosDemo Performance Report" font ",20" +set title filename font ",20" set ylabel "Time in Seconds" @@ -14,13 +14,13 @@ set xlabel "Date" set style data linespoints set terminal pngcairo size 1024,768 enhanced font 'Segoe UI, 10' -set output 'taosdemo-report.png' +set output filename . '.png' set datafile separator ',' set key reverse Left outside set grid -plot 'taosdemo-report.csv' using 1:2 title "Create 10,000 Table", \ - "" using 1:3 title "Create 10,000 Table and Insert 100,000 data", \ - "" using 1:4 title "Request Per Second of Insert 100,000 data" +plot filename . '.csv' using 1:2 title "Create 10,000 Tables", \ + "" using 1:3 title "Delete 10,000 Tables", \ + "" using 1:4 title "Create 10,000 Tables and Insert 100,000 records" From 353dc70f4b1f6509920eda7346b6f4d5d1d1f648 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 8 Jun 2020 12:16:55 +0800 Subject: [PATCH 18/22] scripts --- tests/script/bug.sim | 2 + tests/script/general/compress/commitlog.sim | 12 ++--- tests/script/general/compress/compress.sim | 12 ++--- tests/script/general/compress/compress2.sim | 12 ++--- tests/script/general/compress/uncompress.sim | 12 ++--- tests/script/jenkins/simple.txt | 48 ++++++++++++++++++++ 6 files changed, 74 insertions(+), 24 deletions(-) create mode 100644 tests/script/bug.sim create mode 100644 tests/script/jenkins/simple.txt diff --git a/tests/script/bug.sim b/tests/script/bug.sim new file mode 100644 index 0000000000..625b9b0cad --- /dev/null +++ b/tests/script/bug.sim @@ -0,0 +1,2 @@ +run general/parser/projection_limit_offset.sim +run general/parser/limit2.sim \ No newline at end of file diff --git a/tests/script/general/compress/commitlog.sim b/tests/script/general/compress/commitlog.sim index 2c69b809d1..7c167db3f3 100644 --- a/tests/script/general/compress/commitlog.sim +++ b/tests/script/general/compress/commitlog.sim @@ -25,8 +25,8 @@ sql create table $tb (ts timestamp, b bool, t tinyint, s smallint, i int, big bi $count = 0 while $count < $N - $ms = $count . a - sql insert into $tb values( now+ $ms , 1, 0, $count , $count , $count ,'it is a string') + $ms = 1591200000000 + $count + sql insert into $tb values( $ms , 1, 0, $count , $count , $count ,'it is a string') $count = $count + 1 endw @@ -46,8 +46,8 @@ sql create table $tb (ts timestamp, f float, d double, str binary(256)) $count = 0 while $count < $N - $ms = $count . a - sql insert into $tb values( now+ $ms , $count , $count ,'it is a string') + $ms = 1591286400000 + $count + sql insert into $tb values( $ms , $count , $count ,'it is a string') $count = $count + 1 endw @@ -75,8 +75,8 @@ sql create table $tb (ts timestamp, b bool, t tinyint, s smallint, i int, big bi $count = 0 while $count < $N - $ms = $count . a - sql insert into $tb values( now+ $ms , 1 , 0 , $count , $count , $count , $count , $count ,'it is a string') + $ms = 1591372800000 + $count + sql insert into $tb values( $ms , 1 , 0 , $count , $count , $count , $count , $count ,'it is a string') $count = $count + 1 endw diff --git a/tests/script/general/compress/compress.sim b/tests/script/general/compress/compress.sim index bc56227c74..93fdcbaafa 100644 --- a/tests/script/general/compress/compress.sim +++ b/tests/script/general/compress/compress.sim @@ -26,8 +26,8 @@ sql create table $tb (ts timestamp, b bool, t tinyint, s smallint, i int, big bi $count = 0 while $count < $N - $ms = $count . a - sql insert into $tb values( now+ $ms , 1, 0, $count , $count , $count ,'it is a string') + $ms = 1591200000000 + $count + sql insert into $tb values( $ms , 1, 0, $count , $count , $count ,'it is a string') $count = $count + 1 endw @@ -48,8 +48,8 @@ sql create table $tb (ts timestamp, f float, d double, str binary(256)) $count = 0 while $count < $N - $ms = $count . a - sql insert into $tb values( now+ $ms , $count , $count ,'it is a string') + $ms = 1591286400000 + $count + sql insert into $tb values( $ms , $count , $count ,'it is a string') $count = $count + 1 endw @@ -70,8 +70,8 @@ sql create table $tb (ts timestamp, b bool, t tinyint, s smallint, i int, big bi $count = 0 while $count < $N - $ms = $count . a - sql insert into $tb values( now+ $ms , 1 , 0 , $count , $count , $count , $count , $count ,'it is a string') + $ms = 1591372800000 + $count + sql insert into $tb values( $ms , 1 , 0 , $count , $count , $count , $count , $count ,'it is a string') $count = $count + 1 endw diff --git a/tests/script/general/compress/compress2.sim b/tests/script/general/compress/compress2.sim index c3b445c04c..0eb58bd94b 100644 --- a/tests/script/general/compress/compress2.sim +++ b/tests/script/general/compress/compress2.sim @@ -26,8 +26,8 @@ sql create table $tb (ts timestamp, b bool, t tinyint, s smallint, i int, big bi $count = 0 while $count < $N - $ms = $count . a - sql insert into $tb values( now+ $ms , 1, 0, $count , $count , $count ,'it is a string') + $ms = 1591200000000 + $count + sql insert into $tb values( $ms , 1, 0, $count , $count , $count ,'it is a string') $count = $count + 1 endw @@ -48,8 +48,8 @@ sql create table $tb (ts timestamp, f float, d double, str binary(256)) $count = 0 while $count < $N - $ms = $count . a - sql insert into $tb values( now+ $ms , $count , $count ,'it is a string') + $ms = 1591286400000 + $count + sql insert into $tb values( $ms , $count , $count ,'it is a string') $count = $count + 1 endw @@ -70,8 +70,8 @@ sql create table $tb (ts timestamp, b bool, t tinyint, s smallint, i int, big bi $count = 0 while $count < $N - $ms = $count . a - sql insert into $tb values( now+ $ms , 1 , 0 , $count , $count , $count , $count , $count ,'it is a string') + $ms = 1591372800000 + $count + sql insert into $tb values( $ms , 1 , 0 , $count , $count , $count , $count , $count ,'it is a string') $count = $count + 1 endw diff --git a/tests/script/general/compress/uncompress.sim b/tests/script/general/compress/uncompress.sim index 8dde7fae34..13d288451c 100644 --- a/tests/script/general/compress/uncompress.sim +++ b/tests/script/general/compress/uncompress.sim @@ -25,8 +25,8 @@ sql create table $tb (ts timestamp, b bool, t tinyint, s smallint, i int, big bi $count = 0 while $count < $N - $ms = $count . a - sql insert into $tb values( now+ $ms , 1, 0, $count , $count , $count ,'it is a string') + $ms = 1591200000000 + $count + sql insert into $tb values( $ms , 1, 0, $count , $count , $count ,'it is a string') $count = $count + 1 endw @@ -47,8 +47,8 @@ sql create table $tb (ts timestamp, f float, d double, str binary(256)) $count = 0 while $count < $N - $ms = $count . a - sql insert into $tb values( now+ $ms , $count , $count ,'it is a string') + $ms = 1591286400000 + $count + sql insert into $tb values( $ms , $count , $count ,'it is a string') $count = $count + 1 endw @@ -69,8 +69,8 @@ sql create table $tb (ts timestamp, b bool, t tinyint, s smallint, i int, big bi $count = 0 while $count < $N - $ms = $count . a - sql insert into $tb values( now+ $ms , 1 , 0 , $count , $count , $count , $count , $count ,'it is a string') + $ms = 1591372800000 + $count + sql insert into $tb values( $ms , 1 , 0 , $count , $count , $count , $count , $count ,'it is a string') $count = $count + 1 endw diff --git a/tests/script/jenkins/simple.txt b/tests/script/jenkins/simple.txt new file mode 100644 index 0000000000..ae48994c2a --- /dev/null +++ b/tests/script/jenkins/simple.txt @@ -0,0 +1,48 @@ +cd ../../../debug; cmake .. +cd ../../../debug; make + +#./test.sh -f general/parser/lastrow.sim +#./test.sh -f general/parser/nchar.sim +#./test.sh -f general/parser/limit.sim +#./test.sh -f general/parser/limit1.sim +#./test.sh -f general/parser/limit1_tblocks100.sim +#./test.sh -f general/parser/binary_escapeCharacter.sim +#./test.sh -f general/parser/projection_limit_offset.sim +#./test.sh -f general/parser/limit2.sim +#./test.sh -f general/stable/metrics.sim +#./test.sh -f general/table/date.sim +#./test.sh -f unique/big/balance.sim +#./test.sh -f unique/column/replica3.sim +#./test.sh -f unique/db/commit.sim +./test.sh -f unique/db/delete.sim +./test.sh -f unique/db/delete.sim +./test.sh -f unique/db/delete.sim +./test.sh -f unique/db/delete.sim +./test.sh -f unique/db/delete.sim +./test.sh -f unique/db/delete.sim +./test.sh -f unique/db/delete.sim +./test.sh -f unique/db/delete.sim +./test.sh -f unique/db/delete.sim +./test.sh -f unique/db/delete.sim +./test.sh -f unique/db/delete.sim +./test.sh -f unique/db/delete.sim +./test.sh -f unique/db/delete.sim +./test.sh -f unique/db/delete.sim +./test.sh -f unique/db/delete.sim +./test.sh -f unique/db/delete.sim +./test.sh -f unique/db/delete.sim +./test.sh -f unique/db/delete.sim +./test.sh -f unique/db/delete.sim +./test.sh -f unique/db/delete.sim +./test.sh -f unique/db/delete.sim +./test.sh -f unique/db/delete.sim + +#./test.sh -f unique/db/replica_add12.sim +#./test.sh -f unique/db/replica_add13.sim +#./test.sh -f unique/vnode/replica3_basic.sim +#./test.sh -f unique/dnode/balance1.sim +#./test.sh -f unique/dnode/balance2.sim +#./test.sh -f unique/dnode/balance3.sim +#./test.sh -f unique/cluster/balance1.sim +#./test.sh -f unique/cluster/balance2.sim +#./test.sh -f unique/cluster/balance3.sim \ No newline at end of file From fa2a96e520f37976c5f81fb8320fb6741cb62910 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 8 Jun 2020 13:03:49 +0800 Subject: [PATCH 19/22] [TD-549] fix crash while vread return TSDB_CODE_NOT_READY errno --- src/dnode/src/dnodeVRead.c | 5 +++-- src/dnode/src/dnodeVWrite.c | 2 +- src/inc/vnode.h | 1 - src/vnode/src/vnodeRead.c | 26 ++++++++++++++++---------- 4 files changed, 20 insertions(+), 14 deletions(-) diff --git a/src/dnode/src/dnodeVRead.c b/src/dnode/src/dnodeVRead.c index 7f1a5b2580..2f9e9a0af9 100644 --- a/src/dnode/src/dnodeVRead.c +++ b/src/dnode/src/dnodeVRead.c @@ -192,13 +192,14 @@ void dnodeSendRpcReadRsp(void *pVnode, SReadMsg *pRead, int32_t code) { if (code == TSDB_CODE_VND_ACTION_IN_PROGRESS) return; if (code == TSDB_CODE_VND_ACTION_NEED_REPROCESSED) { dnodeContinueExecuteQuery(pVnode, pRead->rspRet.qhandle, pRead); + code = TSDB_CODE_SUCCESS; } SRpcMsg rpcRsp = { .handle = pRead->rpcMsg.handle, .pCont = pRead->rspRet.rsp, .contLen = pRead->rspRet.len, - .code = pRead->rspRet.code, + .code = code, }; rpcSendResponse(&rpcRsp); @@ -216,7 +217,7 @@ static void *dnodeProcessReadQueue(void *param) { break; } - dTrace("%p, msg:%s will be processed", pReadMsg->rpcMsg.ahandle, taosMsg[pReadMsg->rpcMsg.msgType]); + dTrace("%p, msg:%s will be processed in vread queue", pReadMsg->rpcMsg.ahandle, taosMsg[pReadMsg->rpcMsg.msgType]); int32_t code = vnodeProcessRead(pVnode, pReadMsg->rpcMsg.msgType, pReadMsg->pCont, pReadMsg->contLen, &pReadMsg->rspRet); dnodeSendRpcReadRsp(pVnode, pReadMsg, code); taosFreeQitem(pReadMsg); diff --git a/src/dnode/src/dnodeVWrite.c b/src/dnode/src/dnodeVWrite.c index 53533b8183..e61364355d 100644 --- a/src/dnode/src/dnodeVWrite.c +++ b/src/dnode/src/dnodeVWrite.c @@ -216,7 +216,7 @@ static void *dnodeProcessWriteQueue(void *param) { pHead->msgType = pWrite->rpcMsg.msgType; pHead->version = 0; pHead->len = pWrite->contLen; - dTrace("%p, msg:%s will be processed", pWrite->rpcMsg.ahandle, taosMsg[pWrite->rpcMsg.msgType]); + dTrace("%p, msg:%s will be processed in vwrite queue", pWrite->rpcMsg.ahandle, taosMsg[pWrite->rpcMsg.msgType]); } else { pHead = (SWalHead *)item; } diff --git a/src/inc/vnode.h b/src/inc/vnode.h index f4fb8060fe..4d482afa02 100644 --- a/src/inc/vnode.h +++ b/src/inc/vnode.h @@ -30,7 +30,6 @@ typedef enum _VN_STATUS { typedef struct { int len; - int code; void *rsp; void *qhandle; //used by query and retrieve msg } SRspRet; diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index 2cf72bb15d..f198c2ffe4 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -39,15 +39,21 @@ void vnodeInitReadFp(void) { int32_t vnodeProcessRead(void *param, int msgType, void *pCont, int32_t contLen, SRspRet *ret) { SVnodeObj *pVnode = (SVnodeObj *)param; - if (vnodeProcessReadMsgFp[msgType] == NULL) + if (vnodeProcessReadMsgFp[msgType] == NULL) { + vTrace("vgId:%d, msgType:%s not processed, no handle", pVnode->vgId, taosMsg[msgType]); return TSDB_CODE_VND_MSG_NOT_PROCESSED; + } - if (pVnode->status == TAOS_VN_STATUS_DELETING || pVnode->status == TAOS_VN_STATUS_CLOSING) + if (pVnode->status == TAOS_VN_STATUS_DELETING || pVnode->status == TAOS_VN_STATUS_CLOSING) { + vTrace("vgId:%d, msgType:%s not processed, vnode status is %d", pVnode->vgId, taosMsg[msgType], pVnode->status); return TSDB_CODE_VND_INVALID_VGROUP_ID; + } // TODO: Later, let slave to support query - if (pVnode->syncCfg.replica > 1 && pVnode->role != TAOS_SYNC_ROLE_MASTER) + if (pVnode->syncCfg.replica > 1 && pVnode->role != TAOS_SYNC_ROLE_MASTER) { + vTrace("vgId:%d, msgType:%s not processed, replica:%d role:%d", pVnode->vgId, taosMsg[msgType], pVnode->syncCfg.replica, pVnode->role); return TSDB_CODE_RPC_NOT_READY; + } return (*vnodeProcessReadMsgFp[msgType])(pVnode, pCont, contLen, ret); } @@ -60,11 +66,11 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, void *pCont, int32_t cont qinfo_t pQInfo = NULL; if (contLen != 0) { - pRet->code = qCreateQueryInfo(pVnode->tsdb, pVnode->vgId, pQueryTableMsg, &pQInfo); + code = qCreateQueryInfo(pVnode->tsdb, pVnode->vgId, pQueryTableMsg, &pQInfo); SQueryTableRsp *pRsp = (SQueryTableRsp *) rpcMallocCont(sizeof(SQueryTableRsp)); pRsp->qhandle = htobe64((uint64_t) (pQInfo)); - pRsp->code = pRet->code; + pRsp->code = code; pRet->len = sizeof(SQueryTableRsp); pRet->rsp = pRsp; @@ -74,9 +80,11 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, void *pCont, int32_t cont assert(pCont != NULL); pQInfo = pCont; code = TSDB_CODE_VND_ACTION_IN_PROGRESS; + vTrace("vgId:%d, QInfo:%p, dnode query msg in progress", pVnode->vgId, pQInfo); } if (pQInfo != NULL) { + vTrace("vgId:%d, QInfo:%p, do qTableQuery", pVnode->vgId, pQInfo); qTableQuery(pQInfo); // do execute query } @@ -88,18 +96,16 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, void *pCont, int32_t cont void *pQInfo = (void*) htobe64(pRetrieve->qhandle); memset(pRet, 0, sizeof(SRspRet)); - int32_t code = TSDB_CODE_SUCCESS; - vTrace("vgId:%d, QInfo:%p, retrieve msg is received", pVnode->vgId, pQInfo); - pRet->code = qRetrieveQueryResultInfo(pQInfo); - if (pRet->code != TSDB_CODE_SUCCESS) { + int32_t code = qRetrieveQueryResultInfo(pQInfo); + if (code != TSDB_CODE_SUCCESS) { //TODO pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp)); memset(pRet->rsp, 0, sizeof(SRetrieveTableRsp)); } else { // todo check code and handle error in build result set - pRet->code = qDumpRetrieveResult(pQInfo, (SRetrieveTableRsp **)&pRet->rsp, &pRet->len); + code = qDumpRetrieveResult(pQInfo, (SRetrieveTableRsp **)&pRet->rsp, &pRet->len); if (qHasMoreResultsToRetrieve(pQInfo)) { pRet->qhandle = pQInfo; From 0fc652a4b6abc07fc3effc5bf6e3a4b84bebf4b7 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Mon, 8 Jun 2020 15:06:24 +0800 Subject: [PATCH 20/22] [modify str len for version] --- packaging/release.sh | 12 ++++++------ src/util/src/version.c | 10 +++++----- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/packaging/release.sh b/packaging/release.sh index a4562d21d2..7a585431a2 100755 --- a/packaging/release.sh +++ b/packaging/release.sh @@ -162,18 +162,18 @@ done # output the version info to the buildinfo file. build_time=$(date +"%F %R") -echo "char version[64] = \"${version}\";" > ${versioninfo} -echo "char compatible_version[64] = \"${compatible_version}\";" >> ${versioninfo} -echo "char gitinfo[128] = \"$(git rev-parse --verify HEAD)\";" >> ${versioninfo} +echo "char version[12] = \"${version}\";" > ${versioninfo} +echo "char compatible_version[12] = \"${compatible_version}\";" >> ${versioninfo} +echo "char gitinfo[48] = \"$(git rev-parse --verify HEAD)\";" >> ${versioninfo} if [ "$verMode" != "cluster" ]; then - echo "char gitinfoOfInternal[128] = \"\";" >> ${versioninfo} + echo "char gitinfoOfInternal[48] = \"\";" >> ${versioninfo} else enterprise_dir="${top_dir}/../enterprise" cd ${enterprise_dir} - echo "char gitinfoOfInternal[128] = \"$(git rev-parse --verify HEAD)\";" >> ${versioninfo} + echo "char gitinfoOfInternal[48] = \"$(git rev-parse --verify HEAD)\";" >> ${versioninfo} cd ${curr_dir} fi -echo "char buildinfo[512] = \"Built by ${USER} at ${build_time}\";" >> ${versioninfo} +echo "char buildinfo[64] = \"Built by ${USER} at ${build_time}\";" >> ${versioninfo} echo "" >> ${versioninfo} tmp_version=$(echo $version | tr -s "." "_") if [ "$verMode" == "cluster" ]; then diff --git a/src/util/src/version.c b/src/util/src/version.c index b6e10d8b7e..753cf71bc8 100644 --- a/src/util/src/version.c +++ b/src/util/src/version.c @@ -1,7 +1,7 @@ -char version[64] = "2.0.0.0"; -char compatible_version[64] = "2.0.0.0"; -char gitinfo[128] = "3264067e97300c84caa61ac909d548c9ca56de6b"; -char gitinfoOfInternal[128] = "da88f4a2474737d1f9c76adcf0ff7fd0975e7342"; -char buildinfo[512] = "Built by root at 2020-04-01 14:38"; +char version[TSDB_VERSION_LEN] = "2.0.0.0"; +char compatible_version[TSDB_VERSION_LEN] = "2.0.0.0"; +char gitinfo[48] = "3264067e97300c84caa61ac909d548c9ca56de6b"; +char gitinfoOfInternal[48] = "da88f4a2474737d1f9c76adcf0ff7fd0975e7342"; +char buildinfo[64] = "Built by root at 2020-04-01 14:38"; void libtaos_1_6_5_4_Linux_x64() {}; From 0177f0df71f3f9626bf8c22e8d183ad55e147192 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 8 Jun 2020 15:09:30 +0800 Subject: [PATCH 21/22] add slguan@taosdata.com to coverity scan notification --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b0911716c5..a356dbab10 100644 --- a/.travis.yml +++ b/.travis.yml @@ -111,7 +111,7 @@ matrix: description: TDengine # Where email notification of build analysis results will be sent - notification_email: sdsang@taosdata.com + notification_email: sdsang@taosdata.com, slguan@taosdata.com # Commands to prepare for build_command # ** likely specific to your build ** From 370c67361c20d63c8913bb9afa00efc585bea3c0 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Mon, 8 Jun 2020 15:12:50 +0800 Subject: [PATCH 22/22] [modify string len for version] --- src/util/src/version.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/util/src/version.c b/src/util/src/version.c index 753cf71bc8..0d5e08ce75 100644 --- a/src/util/src/version.c +++ b/src/util/src/version.c @@ -1,5 +1,5 @@ -char version[TSDB_VERSION_LEN] = "2.0.0.0"; -char compatible_version[TSDB_VERSION_LEN] = "2.0.0.0"; +char version[12] = "2.0.0.0"; +char compatible_version[12] = "2.0.0.0"; char gitinfo[48] = "3264067e97300c84caa61ac909d548c9ca56de6b"; char gitinfoOfInternal[48] = "da88f4a2474737d1f9c76adcf0ff7fd0975e7342"; char buildinfo[64] = "Built by root at 2020-04-01 14:38";