From f982428e46fecfe2babfbe125c50dc32ff38d15a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 3 Feb 2023 14:46:45 +0800 Subject: [PATCH 01/21] mroe code --- source/dnode/vnode/src/vnd/vnodeSvr.c | 446 ++++++++++---------------- 1 file changed, 167 insertions(+), 279 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 313b9da9a5..29f6cd872e 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -30,158 +30,186 @@ static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t version, void *pReq, static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { - int32_t code = 0; +static int32_t vnodePreProcessCreateTableMsg(SVnode *pVnode, SRpcMsg *pMsg) { + int32_t code = 0; + int32_t lino = 0; + + int64_t ctime = taosGetTimestampMs(); SDecoder dc = {0}; + int32_t nReqs; + + tDecoderInit(&dc, (uint8_t *)pMsg->pCont + sizeof(SMsgHead), pMsg->contLen - sizeof(SMsgHead)); + if (tStartDecode(&dc) < 0) { + code = TSDB_CODE_INVALID_MSG; + return code; + } + + if (tDecodeI32v(&dc, &nReqs) < 0) { + code = TSDB_CODE_INVALID_MSG; + TSDB_CHECK_CODE(code, lino, _exit); + } + for (int32_t iReq = 0; iReq < nReqs; iReq++) { + tb_uid_t uid = tGenIdPI64(); + char *name = NULL; + if (tStartDecode(&dc) < 0) { + code = TSDB_CODE_INVALID_MSG; + TSDB_CHECK_CODE(code, lino, _exit); + } + + if (tDecodeI32v(&dc, NULL) < 0) { + code = TSDB_CODE_INVALID_MSG; + TSDB_CHECK_CODE(code, lino, _exit); + } + if (tDecodeCStr(&dc, &name) < 0) { + code = TSDB_CODE_INVALID_MSG; + TSDB_CHECK_CODE(code, lino, _exit); + } + *(int64_t *)(dc.data + dc.pos) = uid; + *(int64_t *)(dc.data + dc.pos + 8) = ctime; + + vTrace("vgId:%d table:%s uid:%" PRId64 " is generated", pVnode->config.vgId, name, uid); + tEndDecode(&dc); + } + + tEndDecode(&dc); + +_exit: + tDecoderClear(&dc); + return code; +} + +static int32_t vnodePreProcessSubmitMsg(SVnode *pVnode, SRpcMsg *pMsg) { + int32_t code = 0; + int32_t lino = 0; + + int64_t ctime = taosGetTimestampMs(); + SDecoder dc = {0}; + + tDecoderInit(&dc, (uint8_t *)pMsg->pCont + sizeof(SMsgHead), pMsg->contLen - sizeof(SMsgHead)); + tStartDecode(&dc); + + uint64_t nSubmitTbData; + if (tDecodeU64v(&dc, &nSubmitTbData) < 0) { + code = TSDB_CODE_INVALID_MSG; + TSDB_CHECK_CODE(code, lino, _exit); + } + + for (int32_t i = 0; i < nSubmitTbData; i++) { + if (tStartDecode(&dc) < 0) { + code = TSDB_CODE_INVALID_MSG; + TSDB_CHECK_CODE(code, lino, _exit); + } + + int32_t flags; + if (tDecodeI32v(&dc, &flags) < 0) { + code = TSDB_CODE_INVALID_MSG; + TSDB_CHECK_CODE(code, lino, _exit); + } + + if (flags & SUBMIT_REQ_AUTO_CREATE_TABLE) { + // SVCreateTbReq + if (tStartDecode(&dc) < 0) { + code = TSDB_CODE_INVALID_MSG; + TSDB_CHECK_CODE(code, lino, _exit); + } + + if (tDecodeI32v(&dc, NULL) < 0) { + code = TSDB_CODE_INVALID_MSG; + TSDB_CHECK_CODE(code, lino, _exit); + } + + char *name = NULL; + if (tDecodeCStr(&dc, &name) < 0) { + code = TSDB_CODE_INVALID_MSG; + TSDB_CHECK_CODE(code, lino, _exit); + } + + int64_t uid = metaGetTableEntryUidByName(pVnode->pMeta, name); + if (uid == 0) { + uid = tGenIdPI64(); + } + + *(int64_t *)(dc.data + dc.pos) = uid; + *(int64_t *)(dc.data + dc.pos + 8) = ctime; + + tEndDecode(&dc); + + // SSubmitTbData + int64_t suid; + if (tDecodeI64(&dc, &suid) < 0) { + code = TSDB_CODE_INVALID_MSG; + TSDB_CHECK_CODE(code, lino, _exit); + } + + *(int64_t *)(dc.data + dc.pos) = uid; + } + + tEndDecode(&dc); + } + + tEndDecode(&dc); + tDecoderClear(&dc); + +_exit: + return code; +} + +static int32_t vnodePreProcessDeleteMsg(SVnode *pVnode, SRpcMsg *pMsg) { + int32_t code = 0; + + int32_t size; + int32_t ret; + uint8_t *pCont; + SEncoder *pCoder = &(SEncoder){0}; + SDeleteRes res = {0}; + SReadHandle handle = {.meta = pVnode->pMeta, .config = &pVnode->config, .vnode = pVnode, .pMsgCb = &pVnode->msgCb}; + + code = qWorkerProcessDeleteMsg(&handle, pVnode->pQuery, pMsg, &res); + if (code) goto _exit; + + // malloc and encode + tEncodeSize(tEncodeDeleteRes, &res, size, ret); + pCont = rpcMallocCont(size + sizeof(SMsgHead)); + + ((SMsgHead *)pCont)->contLen = size + sizeof(SMsgHead); + ((SMsgHead *)pCont)->vgId = TD_VID(pVnode); + + tEncoderInit(pCoder, pCont + sizeof(SMsgHead), size); + tEncodeDeleteRes(pCoder, &res); + tEncoderClear(pCoder); + + rpcFreeCont(pMsg->pCont); + pMsg->pCont = pCont; + pMsg->contLen = size + sizeof(SMsgHead); + + taosArrayDestroy(res.uidList); + +_exit: + return code; +} + +int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { + int32_t code = 0; switch (pMsg->msgType) { case TDMT_VND_CREATE_TABLE: { - int64_t ctime = taosGetTimestampMs(); - int32_t nReqs; - - tDecoderInit(&dc, (uint8_t *)pMsg->pCont + sizeof(SMsgHead), pMsg->contLen - sizeof(SMsgHead)); - if (tStartDecode(&dc) < 0) { - code = TSDB_CODE_INVALID_MSG; - return code; - } - - if (tDecodeI32v(&dc, &nReqs) < 0) { - code = TSDB_CODE_INVALID_MSG; - goto _err; - } - for (int32_t iReq = 0; iReq < nReqs; iReq++) { - tb_uid_t uid = tGenIdPI64(); - char *name = NULL; - if (tStartDecode(&dc) < 0) { - code = TSDB_CODE_INVALID_MSG; - goto _err; - } - - if (tDecodeI32v(&dc, NULL) < 0) { - code = TSDB_CODE_INVALID_MSG; - return code; - } - if (tDecodeCStr(&dc, &name) < 0) { - code = TSDB_CODE_INVALID_MSG; - return code; - } - *(int64_t *)(dc.data + dc.pos) = uid; - *(int64_t *)(dc.data + dc.pos + 8) = ctime; - - vTrace("vgId:%d, table:%s uid:%" PRId64 " is generated", pVnode->config.vgId, name, uid); - tEndDecode(&dc); - } - - tEndDecode(&dc); - tDecoderClear(&dc); + code = vnodePreProcessCreateTableMsg(pVnode, pMsg); } break; case TDMT_VND_SUBMIT: { - int64_t ctime = taosGetTimestampMs(); - - tDecoderInit(&dc, (uint8_t *)pMsg->pCont + sizeof(SMsgHead), pMsg->contLen - sizeof(SMsgHead)); - tStartDecode(&dc); - - uint64_t nSubmitTbData; - if (tDecodeU64v(&dc, &nSubmitTbData) < 0) { - code = TSDB_CODE_INVALID_MSG; - goto _err; - } - - for (int32_t i = 0; i < nSubmitTbData; i++) { - if (tStartDecode(&dc) < 0) { - code = TSDB_CODE_INVALID_MSG; - goto _err; - } - - int32_t flags; - if (tDecodeI32v(&dc, &flags) < 0) { - code = TSDB_CODE_INVALID_MSG; - goto _err; - } - - if (flags & SUBMIT_REQ_AUTO_CREATE_TABLE) { - // SVCreateTbReq - if (tStartDecode(&dc) < 0) { - code = TSDB_CODE_INVALID_MSG; - goto _err; - } - - if (tDecodeI32v(&dc, NULL) < 0) { - code = TSDB_CODE_INVALID_MSG; - goto _err; - } - - char *name = NULL; - if (tDecodeCStr(&dc, &name) < 0) { - code = TSDB_CODE_INVALID_MSG; - goto _err; - } - - int64_t uid = metaGetTableEntryUidByName(pVnode->pMeta, name); - if (uid == 0) { - uid = tGenIdPI64(); - } - - *(int64_t *)(dc.data + dc.pos) = uid; - *(int64_t *)(dc.data + dc.pos + 8) = ctime; - - tEndDecode(&dc); - - // SSubmitTbData - int64_t suid; - if (tDecodeI64(&dc, &suid) < 0) { - code = TSDB_CODE_INVALID_MSG; - goto _err; - } - - *(int64_t *)(dc.data + dc.pos) = uid; - } - - tEndDecode(&dc); - } - - tEndDecode(&dc); - tDecoderClear(&dc); + code = vnodePreProcessSubmitMsg(pVnode, pMsg); } break; case TDMT_VND_DELETE: { - int32_t size; - int32_t ret; - uint8_t *pCont; - SEncoder *pCoder = &(SEncoder){0}; - SDeleteRes res = {0}; - SReadHandle handle = { - .meta = pVnode->pMeta, .config = &pVnode->config, .vnode = pVnode, .pMsgCb = &pVnode->msgCb}; - - code = qWorkerProcessDeleteMsg(&handle, pVnode->pQuery, pMsg, &res); - if (code) { - goto _err; - } - - // malloc and encode - tEncodeSize(tEncodeDeleteRes, &res, size, ret); - pCont = rpcMallocCont(size + sizeof(SMsgHead)); - - ((SMsgHead *)pCont)->contLen = size + sizeof(SMsgHead); - ((SMsgHead *)pCont)->vgId = TD_VID(pVnode); - - tEncoderInit(pCoder, pCont + sizeof(SMsgHead), size); - tEncodeDeleteRes(pCoder, &res); - tEncoderClear(pCoder); - - rpcFreeCont(pMsg->pCont); - pMsg->pCont = pCont; - pMsg->contLen = size + sizeof(SMsgHead); - - taosArrayDestroy(res.uidList); + code = vnodePreProcessDeleteMsg(pVnode, pMsg); } break; default: break; } - return code; - -_err: - vError("vgId%d, preprocess request failed since %s", TD_VID(pVnode), tstrerror(code)); +_exit: + if (code) { + vError("vgId%d failed to preprocess write request since %s, msg type:%d", TD_VID(pVnode), tstrerror(code), + pMsg->msgType); + } return code; } @@ -871,7 +899,6 @@ static int32_t vnodeDebugPrintSingleSubmitMsg(SMeta *pMeta, SSubmitBlk *pBlock, } static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { -#if 1 int32_t code = 0; terrno = 0; @@ -1042,145 +1069,6 @@ _exit: if (code) terrno = code; return code; - -#else - SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; - SSubmitRsp submitRsp = {0}; - int32_t nRows = 0; - int32_t tsize, ret; - SEncoder encoder = {0}; - SArray *newTbUids = NULL; - SVStatis statis = {0}; - bool tbCreated = false; - terrno = TSDB_CODE_SUCCESS; - - pRsp->code = 0; - pSubmitReq->version = version; - statis.nBatchInsert = 1; - - if (tsdbScanAndConvertSubmitMsg(pVnode->pTsdb, pSubmitReq) < 0) { - pRsp->code = terrno; - goto _exit; - } - - submitRsp.pArray = taosArrayInit(msgIter.numOfBlocks, sizeof(SSubmitBlkRsp)); - newTbUids = taosArrayInit(msgIter.numOfBlocks, sizeof(int64_t)); - if (!submitRsp.pArray || !newTbUids) { - pRsp->code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - - for (;;) { - tGetSubmitMsgNext(&msgIter, &pBlock); - if (pBlock == NULL) break; - - SSubmitBlkRsp submitBlkRsp = {0}; - tbCreated = false; - - // create table for auto create table mode - if (msgIter.schemaLen > 0) { - // tDecoderInit(&decoder, pBlock->data, msgIter.schemaLen); - // if (tDecodeSVCreateTbReq(&decoder, &createTbReq) < 0) { - // pRsp->code = TSDB_CODE_INVALID_MSG; - // tDecoderClear(&decoder); - // taosArrayDestroy(createTbReq.ctb.tagName); - // goto _exit; - // } - - // if ((terrno = grantCheck(TSDB_GRANT_TIMESERIES)) < 0) { - // pRsp->code = terrno; - // tDecoderClear(&decoder); - // taosArrayDestroy(createTbReq.ctb.tagName); - // goto _exit; - // } - - // if ((terrno = grantCheck(TSDB_GRANT_TABLE)) < 0) { - // pRsp->code = terrno; - // tDecoderClear(&decoder); - // taosArrayDestroy(createTbReq.ctb.tagName); - // goto _exit; - // } - - if (metaCreateTable(pVnode->pMeta, version, &createTbReq, &submitBlkRsp.pMeta) < 0) { - // if (terrno != TSDB_CODE_TDB_TABLE_ALREADY_EXIST) { - // submitBlkRsp.code = terrno; - // pRsp->code = terrno; - // tDecoderClear(&decoder); - // taosArrayDestroy(createTbReq.ctb.tagName); - // goto _exit; - // } - } else { - if (NULL != submitBlkRsp.pMeta) { - vnodeUpdateMetaRsp(pVnode, submitBlkRsp.pMeta); - } - - // taosArrayPush(newTbUids, &createTbReq.uid); - - submitBlkRsp.uid = createTbReq.uid; - submitBlkRsp.tblFName = taosMemoryMalloc(strlen(pVnode->config.dbname) + strlen(createTbReq.name) + 2); - sprintf(submitBlkRsp.tblFName, "%s.%s", pVnode->config.dbname, createTbReq.name); - tbCreated = true; - } - - // msgIter.uid = createTbReq.uid; - // if (createTbReq.type == TSDB_CHILD_TABLE) { - // msgIter.suid = createTbReq.ctb.suid; - // } else { - // msgIter.suid = 0; - // } - - // tDecoderClear(&decoder); - // taosArrayDestroy(createTbReq.ctb.tagName); - } - - if (tsdbInsertTableData(pVnode->pTsdb, version, &msgIter, pBlock, &submitBlkRsp) < 0) { - submitBlkRsp.code = terrno; - } - - submitRsp.numOfRows += submitBlkRsp.numOfRows; - submitRsp.affectedRows += submitBlkRsp.affectedRows; - if (tbCreated || submitBlkRsp.code) { - taosArrayPush(submitRsp.pArray, &submitBlkRsp); - } - } - - // if (taosArrayGetSize(newTbUids) > 0) { - // vDebug("vgId:%d, add %d table into query table list in handling submit", TD_VID(pVnode), - // (int32_t)taosArrayGetSize(newTbUids)); - // } - - // tqUpdateTbUidList(pVnode->pTq, newTbUids, true); - -_exit: - taosArrayDestroy(newTbUids); - // tEncodeSize(tEncodeSSubmitRsp, &submitRsp, tsize, ret); - // pRsp->pCont = rpcMallocCont(tsize); - // pRsp->contLen = tsize; - // tEncoderInit(&encoder, pRsp->pCont, tsize); - // tEncodeSSubmitRsp(&encoder, &submitRsp); - // tEncoderClear(&encoder); - - taosArrayDestroyEx(submitRsp.pArray, tFreeSSubmitBlkRsp); - - // TODO: the partial success scenario and the error case - // => If partial success, extract the success submitted rows and reconstruct a new submit msg, and push to level - // 1/level 2. - // TODO: refactor - if ((terrno == TSDB_CODE_SUCCESS) && (pRsp->code == TSDB_CODE_SUCCESS)) { - statis.nBatchInsertSuccess = 1; - tdProcessRSmaSubmit(pVnode->pSma, pReq, STREAM_INPUT__DATA_SUBMIT); - } - - // N.B. not strict as the following procedure is not atomic - atomic_add_fetch_64(&pVnode->statis.nInsert, submitRsp.numOfRows); - atomic_add_fetch_64(&pVnode->statis.nInsertSuccess, submitRsp.affectedRows); - atomic_add_fetch_64(&pVnode->statis.nBatchInsert, statis.nBatchInsert); - atomic_add_fetch_64(&pVnode->statis.nBatchInsertSuccess, statis.nBatchInsertSuccess); - - vDebug("vgId:%d, submit success, index:%" PRId64, pVnode->config.vgId, version); - return 0; -#endif - return 0; } static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { From f9e409e9910483a6a48908d2330c0710f2706f12 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 9 Feb 2023 14:28:26 +0800 Subject: [PATCH 02/21] fix:change uid in taosx to avoiding uid same in different db --- source/client/src/clientRawBlockWrite.c | 11 +++++++++-- tests/system-test/7-tmq/tmq_taosx.py | 5 +++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index f290a83df0..95e1255827 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -25,6 +25,10 @@ #include "tref.h" #include "ttimer.h" +static tb_uid_t processSuid(tb_uid_t suid, char* db){ + return suid + MurmurHash3_32(db, strlen(db)); +} + static char* buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id, int8_t t) { char* string = NULL; @@ -690,7 +694,7 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) { pReq.numOfColumns = req.schemaRow.nCols; pReq.numOfTags = req.schemaTag.nCols; pReq.commentLen = -1; - pReq.suid = req.suid; + pReq.suid = processSuid(req.suid, pRequest->pDb);; pReq.source = TD_REQ_FROM_TAOX; pReq.igExists = true; @@ -762,7 +766,7 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { // build drop stable pReq.igNotExists = true; pReq.source = TD_REQ_FROM_TAOX; - pReq.suid = req.suid; + pReq.suid = processSuid(req.suid, pRequest->pDb); STscObj* pTscObj = pRequest->pTscObj; SName tableName = {0}; @@ -880,6 +884,7 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { if (pCreateReq->type == TSDB_CHILD_TABLE) { STableMeta* pTableMeta = NULL; SName sName = {0}; + pCreateReq->ctb.suid = processSuid(pCreateReq->ctb.suid, pRequest->pDb); toName(pTscObj->acctId, pRequest->pDb, pCreateReq->ctb.stbName, &sName); code = catalogGetTableMeta(pCatalog, &conn, &sName, &pTableMeta); if (code != TSDB_CODE_SUCCESS) { @@ -1017,6 +1022,7 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { pDropReq = req.pReqs + iReq; pDropReq->igNotExists = true; + pDropReq->suid = processSuid(pDropReq->suid, pRequest->pDb); SVgroupInfo pInfo = {0}; SName pName = {0}; @@ -1638,6 +1644,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) } if (strcmp(tbName, pCreateReq.name) == 0) { cloneSVreateTbReq(&pCreateReq, &pCreateReqDst); + pCreateReqDst->ctb.suid = processSuid(pCreateReqDst->ctb.suid, pRequest->pDb); tDecoderClear(&decoderTmp); break; } diff --git a/tests/system-test/7-tmq/tmq_taosx.py b/tests/system-test/7-tmq/tmq_taosx.py index b2bf6eec0b..593c91a470 100644 --- a/tests/system-test/7-tmq/tmq_taosx.py +++ b/tests/system-test/7-tmq/tmq_taosx.py @@ -199,6 +199,11 @@ class TDTestCase: tdSql.checkData(0, 2, None) tdSql.checkData(1, 1, 1) tdSql.checkData(1, 2, '{"k1":1,"k2":"hello"}') + + tdSql.query("select * from information_schema.ins_tables where table_name = 'stt4'") + uid1 = tdSql.getData(0, 5) + uid2 = tdSql.getData(1, 5) + tdSql.checkNotEqual(uid1, uid2) return def checkWal1Vgroup(self): From 10167a90cd9c7caf0c47d0f2078c1fef0231e35e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 13 Feb 2023 09:39:21 +0800 Subject: [PATCH 03/21] refact code --- source/dnode/vnode/src/tsdb/tsdbWrite.c | 90 ------------------------- 1 file changed, 90 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c index 02d076113f..2ad971ca28 100644 --- a/source/dnode/vnode/src/tsdb/tsdbWrite.c +++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c @@ -60,23 +60,6 @@ int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq2 *pMsg, SSubmitRsp2 return 0; } -#if 0 -static FORCE_INLINE int tsdbCheckRowRange(STsdb *pTsdb, STable *pTable, STSRow *row, TSKEY minKey, TSKEY maxKey, - TSKEY now) { - TSKEY rowKey = TD_ROW_KEY(row); - if (rowKey < minKey || rowKey > maxKey) { - tsdbError("vgId:%d, table %s tid %d uid %" PRIu64 " timestamp is out of range! now %" PRId64 " minKey %" PRId64 - " maxKey %" PRId64 " row key %" PRId64, - REPO_ID(pTsdb), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), now, minKey, maxKey, - rowKey); - terrno = TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE; - return -1; - } - - return 0; -} -#endif - static FORCE_INLINE int tsdbCheckRowRange(STsdb *pTsdb, tb_uid_t uid, TSKEY rowKey, TSKEY minKey, TSKEY maxKey, TSKEY now) { if (rowKey < minKey || rowKey > maxKey) { @@ -89,79 +72,6 @@ static FORCE_INLINE int tsdbCheckRowRange(STsdb *pTsdb, tb_uid_t uid, TSKEY rowK return 0; } -#if 0 -int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) { - ASSERT(pMsg != NULL); - // STsdbMeta * pMeta = pTsdb->tsdbMeta; - SSubmitMsgIter msgIter = {0}; - SSubmitBlk *pBlock = NULL; - SSubmitBlkIter blkIter = {0}; - STSRow *row = NULL; - STsdbKeepCfg *pCfg = &pTsdb->keepCfg; - TSKEY now = taosGetTimestamp(pCfg->precision); - TSKEY minKey = now - tsTickPerMin[pCfg->precision] * pCfg->keep2; - TSKEY maxKey = tsMaxKeyByPrecision[pCfg->precision]; - - terrno = TSDB_CODE_SUCCESS; - // pMsg->length = htonl(pMsg->length); - // pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); - - if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1; - while (true) { - if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1; - if (pBlock == NULL) break; - - // pBlock->uid = htobe64(pBlock->uid); - // pBlock->suid = htobe64(pBlock->suid); - // pBlock->sversion = htonl(pBlock->sversion); - // pBlock->dataLen = htonl(pBlock->dataLen); - // pBlock->schemaLen = htonl(pBlock->schemaLen); - // pBlock->numOfRows = htonl(pBlock->numOfRows); - -#if 0 - if (pBlock->tid <= 0 || pBlock->tid >= pMeta->maxTables) { - tsdbError("vgId:%d, failed to get table to insert data, uid %" PRIu64 " tid %d", REPO_ID(pTsdb), pBlock->uid, - pBlock->tid); - terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; - return -1; - } - - STable *pTable = pMeta->tables[pBlock->tid]; - if (pTable == NULL || TABLE_UID(pTable) != pBlock->uid) { - tsdbError("vgId:%d, failed to get table to insert data, uid %" PRIu64 " tid %d", REPO_ID(pTsdb), pBlock->uid, - pBlock->tid); - terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; - return -1; - } - - if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) { - tsdbError("vgId:%d, invalid action trying to insert a super table %s", REPO_ID(pTsdb), TABLE_CHAR_NAME(pTable)); - terrno = TSDB_CODE_TDB_INVALID_ACTION; - return -1; - } - - // Check schema version and update schema if needed - if (tsdbCheckTableSchema(pTsdb, pBlock, pTable) < 0) { - if (terrno == TSDB_CODE_TDB_TABLE_RECONFIGURE) { - continue; - } else { - return -1; - } - } -#endif - tInitSubmitBlkIter(&msgIter, pBlock, &blkIter); - while ((row = tGetSubmitBlkNext(&blkIter)) != NULL) { - if (tsdbCheckRowRange(pTsdb, msgIter.uid, row, minKey, maxKey, now) < 0) { - return -1; - } - } - } - - if (terrno != TSDB_CODE_SUCCESS) return -1; - return 0; -} -#endif - int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq2 *pMsg) { int32_t code = 0; STsdbKeepCfg *pCfg = &pTsdb->keepCfg; From 0e630b8f8163ebfcd3eb44c4953ea378fe57d37a Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 13 Feb 2023 13:38:05 +0800 Subject: [PATCH 04/21] fix:remove id in execReader & enable task case --- source/dnode/vnode/src/tq/tqRead.c | 2 +- tests/parallel_test/cases.task | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 99f0ed7703..04af05cc44 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -1358,7 +1358,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { } taosArrayDestroy(qa); } else { - // TODO handle delete table from stb + tqReaderRemoveTbUidList(pExec->execHandle.pExecReader, tbUidList); } } } diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 3b4f61daee..fe6573ed4b 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -743,7 +743,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dataFromTsdbNWal.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py -#,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_taosx.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_taosx.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-19201.py ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-21561.py From 997f41c85424273b4294edb02e3db1d141697cab Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 13 Feb 2023 14:05:53 +0800 Subject: [PATCH 05/21] fix:conflict from 3.0 --- source/client/src/clientRawBlockWrite.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 191d028cec..a12f94cf1d 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -694,11 +694,7 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) { pReq.numOfColumns = req.schemaRow.nCols; pReq.numOfTags = req.schemaTag.nCols; pReq.commentLen = -1; -<<<<<<< HEAD - pReq.suid = processSuid(req.suid, pRequest->pDb);; -======= pReq.suid = processSuid(req.suid, pRequest->pDb); ->>>>>>> 6483b92f0c645b36facfb2f3415b8df9449264cf pReq.source = TD_REQ_FROM_TAOX; pReq.igExists = true; From 88970c88a95ee865e2fe4bb75ac6dc1dc13d604a Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Wed, 15 Feb 2023 09:42:25 +0800 Subject: [PATCH 06/21] fix:dispatch create sub table --- source/dnode/vnode/src/tq/tqSink.c | 2 +- source/libs/executor/src/groupoperator.c | 4 ++-- source/libs/executor/src/timewindowoperator.c | 1 + tests/script/tsim/stream/checkStreamSTable.sim | 15 +++++++++++++++ 4 files changed, 19 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 3801a25d6d..29a25e4cd0 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -498,7 +498,7 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void* taosArrayPush(tagArray, &tagVal); } } - pCreateTbReq->ctb.tagNum = taosArrayGetSize(tagArray); + pCreateTbReq->ctb.tagNum = size; STag* pTag = NULL; tTagNew(tagArray, 1, false, &pTag); diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 370e1f62de..a2c637244b 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -1013,7 +1013,7 @@ void appendCreateTableRow(SStreamState* pState, SExprSupp* pTableSup, SExprSupp* void* pGpIdCol = taosArrayGet(pDestBlock->pDataBlock, UD_GROUPID_COLUMN_INDEX); colDataAppend(pGpIdCol, pDestBlock->info.rows, (const char*)&groupId, false); - + pDestBlock->info.id.groupId = groupId; pDestBlock->info.rows++; blockDataDestroy(pTmpBlock); } @@ -1030,7 +1030,7 @@ static SSDataBlock* buildStreamCreateTableResult(SOperatorInfo* pOperator) { blockDataEnsureCapacity(pInfo->pCreateTbRes, taosHashGetSize(pInfo->pPartitions)); SSDataBlock* pSrc = pInfo->pInputDataBlock; - while (pInfo->pTbNameIte != NULL) { + if (pInfo->pTbNameIte != NULL) { SPartitionDataInfo* pParInfo = (SPartitionDataInfo*)pInfo->pTbNameIte; int32_t rowId = *(int32_t*)taosArrayGet(pParInfo->rowIds, 0); appendCreateTableRow(pOperator->pTaskInfo->streamInfo.pState, &pInfo->tbnameCalSup, &pInfo->tagCalSup, diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 1b9767f193..792225e16c 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -4778,6 +4778,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pInfo->pUpdatedMap); continue; } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) { + printDataBlock(pBlock, "single interval"); return pBlock; } else { ASSERTS(pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type"); diff --git a/tests/script/tsim/stream/checkStreamSTable.sim b/tests/script/tsim/stream/checkStreamSTable.sim index b60ab0ac05..fda78af621 100644 --- a/tests/script/tsim/stream/checkStreamSTable.sim +++ b/tests/script/tsim/stream/checkStreamSTable.sim @@ -261,6 +261,21 @@ if $data04 != NULL then goto loop2 endi +print ===== drop ... + +sql drop stream if exists streams0; +sql drop stream if exists streams1; +sql drop stream if exists streams2; +sql drop stream if exists streams3; +sql drop database if exists test; +sql drop database if exists test1; +sql drop database if exists test2; +sql drop database if exists test3; +sql drop database if exists result; +sql drop database if exists result1; +sql drop database if exists result2; +sql drop database if exists result3; + print ===== step6 sql create database result4 vgroups 1; From ff69b973ce87ed96df7a6ad4412042768f3bf2da Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 15 Feb 2023 13:52:21 +0800 Subject: [PATCH 07/21] fix _isfilled support document version --- docs/en/12-taos-sql/10-function.md | 4 ++-- docs/zh/12-taos-sql/10-function.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/en/12-taos-sql/10-function.md b/docs/en/12-taos-sql/10-function.md index 68ac0b0bb5..1a91d40902 100644 --- a/docs/en/12-taos-sql/10-function.md +++ b/docs/en/12-taos-sql/10-function.md @@ -877,8 +877,8 @@ INTERP(expr) - The number of rows in the result set of `INTERP` is determined by the parameter `EVERY(time_unit)`. Starting from timestamp1, one interpolation is performed for every time interval specified `time_unit` parameter. The parameter `time_unit` must be an integer, with no quotes, with a time unit of: a(millisecond)), s(second), m(minute), h(hour), d(day), or w(week). For example, `EVERY(500a)` will interpolate every 500 milliseconds. - Interpolation is performed based on `FILL` parameter. For more information about FILL clause, see [FILL Clause](./distinguished/#fill-clause). - `INTERP` can only be used to interpolate in single timeline. So it must be used with `partition by tbname` when it's used on a STable. -- Pseudocolumn `_irowts` can be used along with `INTERP` to return the timestamps associated with interpolation points(support after version 3.0.1.4). -- Pseudocolumn `_isfilled` can be used along with `INTERP` to indicate whether the results are original records or data points generated by interpolation algorithm(support after version 3.0.2.3). +- Pseudocolumn `_irowts` can be used along with `INTERP` to return the timestamps associated with interpolation points(support after version 3.0.2.0). +- Pseudocolumn `_isfilled` can be used along with `INTERP` to indicate whether the results are original records or data points generated by interpolation algorithm(support after version 3.0.3.0). ### LAST diff --git a/docs/zh/12-taos-sql/10-function.md b/docs/zh/12-taos-sql/10-function.md index 669d037705..98eb5a2546 100644 --- a/docs/zh/12-taos-sql/10-function.md +++ b/docs/zh/12-taos-sql/10-function.md @@ -879,8 +879,8 @@ INTERP(expr) - INTERP 根据 EVERY(time_unit) 字段来确定输出时间范围内的结果条数,即从 timestamp1 开始每隔固定长度的时间(time_unit 值)进行插值,time_unit 可取值时间单位:1a(毫秒),1s(秒),1m(分),1h(小时),1d(天),1w(周)。例如 EVERY(500a) 将对于指定数据每500毫秒间隔进行一次插值. - INTERP 根据 FILL 字段来决定在每个符合输出条件的时刻如何进行插值。关于 FILL 子句如何使用请参考 [FILL 子句](./distinguished/#fill-子句) - INTERP 只能在一个时间序列内进行插值,因此当作用于超级表时必须跟 partition by tbname 一起使用。 -- INTERP 可以与伪列 _irowts 一起使用,返回插值点所对应的时间戳(3.0.1.4版本以后支持)。 -- INTERP 可以与伪列 _isfilled 一起使用,显示返回结果是否为原始记录或插值算法产生的数据(3.0.2.3版本以后支持)。 +- INTERP 可以与伪列 _irowts 一起使用,返回插值点所对应的时间戳(3.0.2.0版本以后支持)。 +- INTERP 可以与伪列 _isfilled 一起使用,显示返回结果是否为原始记录或插值算法产生的数据(3.0.3.0版本以后支持)。 ### LAST From 49b285298df42420f4f2263a89edd491380bf96f Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 15 Feb 2023 15:10:23 +0800 Subject: [PATCH 08/21] fix: taosbenchmark print qps result (#19982) * fix: taosbenchmark print qps result * fix: update taos-tools 64a0dcb * fix: update taos-tools 7c641c5 --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 05191138e5..5f9a44084c 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 22627d7 + GIT_TAG 7c641c5 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From e3bfc9ddf343fe903edebd809f21561413cfeb65 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 15 Feb 2023 17:06:20 +0800 Subject: [PATCH 09/21] docs: add notes for countAlwaysReturnValue parameter --- docs/en/14-reference/12-config/index.md | 1 + docs/zh/14-reference/12-config/index.md | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index afa9f5a8ae..522109337f 100644 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -182,6 +182,7 @@ The parameters described in this document by the effect that they have on the sy | Meaning | count()/hyperloglog() return value or not if the result data is NULL | | Vlue Range | 0:Return empty line,1:Return 0 | | Default | 1 | +| Notes | When this parameter is setting to 1: for queries containing GROUP BY, PARTITION BY and INTERVAL clause, count/hyperloglog functions do not return value; if count/hyperloglog used together with other aggregate functions, other aggregate functions return NULL. | ### maxNumOfDistinctRes diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index 28b409de30..2506f403cf 100644 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -192,7 +192,7 @@ taos --dump-config | 取值范围 | 0 表示包含函数名,1 表示不包含函数名。 | | 缺省值 | 0 | -### countAlwaysReturnValue +### countAlwaysReturnValue | 属性 | 说明 | | -------- | -------------------------------- | @@ -200,6 +200,7 @@ taos --dump-config | 含义 | count/hyperloglog函数在数据为空或者NULL的情况下是否返回值 | | 取值范围 | 0:返回空行,1:返回 0 | | 缺省值 | 1 | +| 补充说明 | 该参数设置为 1 时: 对于查询中含有 group by,partition by 以及 interval 子句的情况下, count/hyperloglog 函数结果仍然返回空行; 查询中 count/hyperloglog 和其他聚合函数一起使用时,其他聚合函数输出结果为NULL | ## 区域相关 From bda2e544c739d5b10c7a3161cdfdf9c2f862d07c Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 15 Feb 2023 17:06:20 +0800 Subject: [PATCH 10/21] docs: add notes for countAlwaysReturnValue parameter --- docs/zh/14-reference/12-config/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index 2506f403cf..814d62343f 100644 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -200,7 +200,7 @@ taos --dump-config | 含义 | count/hyperloglog函数在数据为空或者NULL的情况下是否返回值 | | 取值范围 | 0:返回空行,1:返回 0 | | 缺省值 | 1 | -| 补充说明 | 该参数设置为 1 时: 对于查询中含有 group by,partition by 以及 interval 子句的情况下, count/hyperloglog 函数结果仍然返回空行; 查询中 count/hyperloglog 和其他聚合函数一起使用时,其他聚合函数输出结果为NULL | +| 补充说明 | 该参数设置为 1 时: 对于查询中含有 GROUP BY,PARTITION BY 以及 INTERVAL 子句的情况下, count/hyperloglog 函数结果仍然返回空行; 查询中 count/hyperloglog 和其他聚合函数一起使用时,其他聚合函数输出结果为NULL | ## 区域相关 From b1c3b705b19b6c3a0c839a7d6052d55af0906ea9 Mon Sep 17 00:00:00 2001 From: dapan1121 <72057773+dapan1121@users.noreply.github.com> Date: Wed, 15 Feb 2023 17:30:51 +0800 Subject: [PATCH 11/21] Update index.md --- docs/zh/14-reference/12-config/index.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index 814d62343f..fd37666341 100644 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -197,10 +197,10 @@ taos --dump-config | 属性 | 说明 | | -------- | -------------------------------- | | 适用范围 | 仅服务端适用 | -| 含义 | count/hyperloglog函数在数据为空或者NULL的情况下是否返回值 | +| 含义 | count/hyperloglog函数在输入数据为空或者NULL的情况下是否返回值 | | 取值范围 | 0:返回空行,1:返回 0 | | 缺省值 | 1 | -| 补充说明 | 该参数设置为 1 时: 对于查询中含有 GROUP BY,PARTITION BY 以及 INTERVAL 子句的情况下, count/hyperloglog 函数结果仍然返回空行; 查询中 count/hyperloglog 和其他聚合函数一起使用时,其他聚合函数输出结果为NULL | +| 补充说明 | 该参数设置为 1 时,如果查询中含有 GROUP BY,PARTITION BY 以及 INTERVAL 子句且相应的组或窗口内数据为空或者NULL, 对应的组或窗口将不返回查询结果 | ## 区域相关 From 7ef6bd7ad89faa40b906ada131f083b42aa731f5 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao <36554565+glzhao89@users.noreply.github.com> Date: Wed, 15 Feb 2023 17:41:22 +0800 Subject: [PATCH 12/21] Update index.md --- docs/en/14-reference/12-config/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index 522109337f..3ce0f9e097 100644 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -182,7 +182,7 @@ The parameters described in this document by the effect that they have on the sy | Meaning | count()/hyperloglog() return value or not if the result data is NULL | | Vlue Range | 0:Return empty line,1:Return 0 | | Default | 1 | -| Notes | When this parameter is setting to 1: for queries containing GROUP BY, PARTITION BY and INTERVAL clause, count/hyperloglog functions do not return value; if count/hyperloglog used together with other aggregate functions, other aggregate functions return NULL. | +| Notes | When this parameter is setting to 1, for queries containing GROUP BY, PARTITION BY and INTERVAL clause, and input data in certain groups or windows is empty or NULL, the corresponding groups or windows have no return values | ### maxNumOfDistinctRes From 7f1186c876750b27b9307320a6859a0e62051a50 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao <36554565+glzhao89@users.noreply.github.com> Date: Wed, 15 Feb 2023 17:42:42 +0800 Subject: [PATCH 13/21] Update index.md --- docs/en/14-reference/12-config/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index 3ce0f9e097..2b129533eb 100644 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -179,7 +179,7 @@ The parameters described in this document by the effect that they have on the sy | Attribute | Description | | -------- | -------------------------------- | | Applicable | Server only | -| Meaning | count()/hyperloglog() return value or not if the result data is NULL | +| Meaning | count()/hyperloglog() return value or not if the input data is NULL | | Vlue Range | 0:Return empty line,1:Return 0 | | Default | 1 | | Notes | When this parameter is setting to 1, for queries containing GROUP BY, PARTITION BY and INTERVAL clause, and input data in certain groups or windows is empty or NULL, the corresponding groups or windows have no return values | From d303ad359e2d3e846246f8d547fdf692b8da94cc Mon Sep 17 00:00:00 2001 From: Ganlin Zhao <36554565+glzhao89@users.noreply.github.com> Date: Wed, 15 Feb 2023 17:43:38 +0800 Subject: [PATCH 14/21] Update index.md --- docs/en/14-reference/12-config/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index 2b129533eb..5938b95912 100644 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -179,7 +179,7 @@ The parameters described in this document by the effect that they have on the sy | Attribute | Description | | -------- | -------------------------------- | | Applicable | Server only | -| Meaning | count()/hyperloglog() return value or not if the input data is NULL | +| Meaning | count()/hyperloglog() return value or not if the input data is empty or NULL | | Vlue Range | 0:Return empty line,1:Return 0 | | Default | 1 | | Notes | When this parameter is setting to 1, for queries containing GROUP BY, PARTITION BY and INTERVAL clause, and input data in certain groups or windows is empty or NULL, the corresponding groups or windows have no return values | From c1a3a0855b157133ddd6dfd43858f19b42b1426d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 15 Feb 2023 17:59:54 +0800 Subject: [PATCH 15/21] finish code --- include/common/tdataformat.h | 2 +- source/dnode/vnode/src/tsdb/tsdbWrite.c | 2 +- source/dnode/vnode/src/vnd/vnodeSvr.c | 243 +++++++++++++++--------- 3 files changed, 157 insertions(+), 90 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index d7a62f5402..a36a7513f3 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -205,7 +205,7 @@ struct SColData { int32_t numOfNull; // # of null int32_t numOfValue; // # of vale int32_t nVal; - uint8_t flag; + int8_t flag; uint8_t *pBitMap; int32_t *aOffset; int32_t nData; diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c index 2ad971ca28..301b504346 100644 --- a/source/dnode/vnode/src/tsdb/tsdbWrite.c +++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c @@ -22,7 +22,7 @@ * us: 3600*1000000*8765*1000 // 1970 + 1000 years * ns: 3600*1000000000*8765*292 // 1970 + 292 years */ -static int64_t tsMaxKeyByPrecision[] = {31556995200000L, 31556995200000000L, 9214646400000000000L}; +int64_t tsMaxKeyByPrecision[] = {31556995200000L, 31556995200000000L, 9214646400000000000L}; // static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg); diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 005c95ab38..983bea3706 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -13,7 +13,12 @@ * along with this program. If not, see . */ +#include +#include "tencode.h" +#include "tmsg.h" #include "vnd.h" +#include "vnode.h" +#include "vnodeInt.h" static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); @@ -31,6 +36,48 @@ static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodePreprocessCreateTableReq(SVnode *pVnode, SDecoder *pCoder, int64_t ctime) { + int32_t code = 0; + int32_t lino = 0; + + if (tStartDecode(pCoder) < 0) { + code = TSDB_CODE_INVALID_MSG; + TSDB_CHECK_CODE(code, lino, _exit); + } + + // flags + if (tDecodeI32v(pCoder, NULL) < 0) { + code = TSDB_CODE_INVALID_MSG; + TSDB_CHECK_CODE(code, lino, _exit); + } + + // name + char *name = NULL; + if (tDecodeCStr(pCoder, &name) < 0) { + code = TSDB_CODE_INVALID_MSG; + TSDB_CHECK_CODE(code, lino, _exit); + } + + // uid + int64_t uid = metaGetTableEntryUidByName(pVnode->pMeta, name); + if (uid == 0) { + uid = tGenIdPI64(); + } + *(int64_t *)(pCoder->data + pCoder->pos) = uid; + + // ctime + *(int64_t *)(pCoder->data + pCoder->pos + 8) = ctime; + + tEndDecode(pCoder); + +_exit: + if (code) { + vError("vgId:%d %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); + } else { + vTrace("vgId:%d %s done, table:%s uid generated:%" PRId64, TD_VID(pVnode), __func__, name, uid); + } + return code; +} static int32_t vnodePreProcessCreateTableMsg(SVnode *pVnode, SRpcMsg *pMsg) { int32_t code = 0; int32_t lino = 0; @@ -50,26 +97,8 @@ static int32_t vnodePreProcessCreateTableMsg(SVnode *pVnode, SRpcMsg *pMsg) { TSDB_CHECK_CODE(code, lino, _exit); } for (int32_t iReq = 0; iReq < nReqs; iReq++) { - tb_uid_t uid = tGenIdPI64(); - char *name = NULL; - if (tStartDecode(&dc) < 0) { - code = TSDB_CODE_INVALID_MSG; - TSDB_CHECK_CODE(code, lino, _exit); - } - - if (tDecodeI32v(&dc, NULL) < 0) { - code = TSDB_CODE_INVALID_MSG; - TSDB_CHECK_CODE(code, lino, _exit); - } - if (tDecodeCStr(&dc, &name) < 0) { - code = TSDB_CODE_INVALID_MSG; - TSDB_CHECK_CODE(code, lino, _exit); - } - *(int64_t *)(dc.data + dc.pos) = uid; - *(int64_t *)(dc.data + dc.pos + 8) = ctime; - - vTrace("vgId:%d table:%s uid:%" PRId64 " is generated", pVnode->config.vgId, name, uid); - tEndDecode(&dc); + code = vnodePreprocessCreateTableReq(pVnode, &dc, ctime); + TSDB_CHECK_CODE(code, lino, _exit); } tEndDecode(&dc); @@ -78,80 +107,118 @@ _exit: tDecoderClear(&dc); return code; } - -static int32_t vnodePreProcessSubmitMsg(SVnode *pVnode, SRpcMsg *pMsg) { +extern int64_t tsMaxKeyByPrecision[]; +static int32_t vnodePreProcessSubmitTbData(SVnode *pVnode, SDecoder *pCoder, int64_t ctime) { int32_t code = 0; int32_t lino = 0; - int64_t ctime = taosGetTimestampMs(); - SDecoder dc = {0}; - - tDecoderInit(&dc, (uint8_t *)pMsg->pCont + sizeof(SMsgHead), pMsg->contLen - sizeof(SMsgHead)); - tStartDecode(&dc); - - uint64_t nSubmitTbData; - if (tDecodeU64v(&dc, &nSubmitTbData) < 0) { + if (tStartDecode(pCoder) < 0) { code = TSDB_CODE_INVALID_MSG; TSDB_CHECK_CODE(code, lino, _exit); } - for (int32_t i = 0; i < nSubmitTbData; i++) { - if (tStartDecode(&dc) < 0) { - code = TSDB_CODE_INVALID_MSG; - TSDB_CHECK_CODE(code, lino, _exit); - } - - int32_t flags; - if (tDecodeI32v(&dc, &flags) < 0) { - code = TSDB_CODE_INVALID_MSG; - TSDB_CHECK_CODE(code, lino, _exit); - } - - if (flags & SUBMIT_REQ_AUTO_CREATE_TABLE) { - // SVCreateTbReq - if (tStartDecode(&dc) < 0) { - code = TSDB_CODE_INVALID_MSG; - TSDB_CHECK_CODE(code, lino, _exit); - } - - if (tDecodeI32v(&dc, NULL) < 0) { - code = TSDB_CODE_INVALID_MSG; - TSDB_CHECK_CODE(code, lino, _exit); - } - - char *name = NULL; - if (tDecodeCStr(&dc, &name) < 0) { - code = TSDB_CODE_INVALID_MSG; - TSDB_CHECK_CODE(code, lino, _exit); - } - - int64_t uid = metaGetTableEntryUidByName(pVnode->pMeta, name); - if (uid == 0) { - uid = tGenIdPI64(); - } - - *(int64_t *)(dc.data + dc.pos) = uid; - *(int64_t *)(dc.data + dc.pos + 8) = ctime; - - tEndDecode(&dc); - - // SSubmitTbData - int64_t suid; - if (tDecodeI64(&dc, &suid) < 0) { - code = TSDB_CODE_INVALID_MSG; - TSDB_CHECK_CODE(code, lino, _exit); - } - - *(int64_t *)(dc.data + dc.pos) = uid; - } - - tEndDecode(&dc); + SSubmitTbData submitTbData; + if (tDecodeI32v(pCoder, &submitTbData.flags) < 0) { + code = TSDB_CODE_INVALID_MSG; + TSDB_CHECK_CODE(code, lino, _exit); } - tEndDecode(&dc); - tDecoderClear(&dc); + if (submitTbData.flags & SUBMIT_REQ_AUTO_CREATE_TABLE) { + code = vnodePreprocessCreateTableReq(pVnode, pCoder, ctime); + TSDB_CHECK_CODE(code, lino, _exit); + } + + // submit data + if (tDecodeI64(pCoder, &submitTbData.suid) < 0) { + code = TSDB_CODE_INVALID_MSG; + TSDB_CHECK_CODE(code, lino, _exit); + } + if (tDecodeI64(pCoder, &submitTbData.uid) < 0) { + code = TSDB_CODE_INVALID_MSG; + TSDB_CHECK_CODE(code, lino, _exit); + } + if (tDecodeI32v(pCoder, &submitTbData.sver) < 0) { + code = TSDB_CODE_INVALID_MSG; + TSDB_CHECK_CODE(code, lino, _exit); + } + + // scan and check + TSKEY now = ctime; + if (pVnode->config.tsdbCfg.precision == TSDB_TIME_PRECISION_MICRO) { + now *= 1000; + } else if (pVnode->config.tsdbCfg.precision == TSDB_TIME_PRECISION_NANO) { + now *= 1000000; + } + TSKEY minKey = now - tsTickPerMin[pVnode->config.tsdbCfg.precision] * pVnode->config.tsdbCfg.keep2; + TSKEY maxKey = tsMaxKeyByPrecision[pVnode->config.tsdbCfg.precision]; + if (submitTbData.flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + uint64_t nColData; + if (tDecodeU64v(pCoder, &nColData) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + SColData colData = {0}; + pCoder->pos += tGetColData(pCoder->data + pCoder->pos, &colData); + + for (int32_t iRow = 0; iRow < colData.nVal; iRow++) { + if (((TSKEY *)colData.pData)[iRow] < minKey || ((TSKEY *)colData.pData)[iRow] > maxKey) { + code = TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE; + goto _exit; + } + } + } else { + uint64_t nRow; + if (tDecodeU64v(pCoder, &nRow) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + for (int32_t iRow = 0; iRow < nRow; ++iRow) { + SRow *pRow = (SRow *)(pCoder->data + pCoder->pos); + pCoder->pos += pRow->len; + + if (pRow->ts < minKey || pRow->ts > maxKey) { + code = TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE; + goto _exit; + } + } + } + + tEndDecode(pCoder); _exit: + return 0; +} +static int32_t vnodePreProcessSubmitMsg(SVnode *pVnode, SRpcMsg *pMsg) { + int32_t code = 0; + int32_t lino = 0; + + SDecoder *pCoder = &(SDecoder){0}; + + tDecoderInit(pCoder, (uint8_t *)pMsg->pCont + sizeof(SMsgHead), pMsg->contLen - sizeof(SMsgHead)); + + if (tStartDecode(pCoder) < 0) { + code = TSDB_CODE_INVALID_MSG; + TSDB_CHECK_CODE(code, lino, _exit); + } + + uint64_t nSubmitTbData; + if (tDecodeU64v(pCoder, &nSubmitTbData) < 0) { + code = TSDB_CODE_INVALID_MSG; + TSDB_CHECK_CODE(code, lino, _exit); + } + + int64_t ctime = taosGetTimestampMs(); + for (int32_t i = 0; i < nSubmitTbData; i++) { + code = vnodePreProcessSubmitTbData(pVnode, pCoder, ctime); + TSDB_CHECK_CODE(code, lino, _exit); + } + + tEndDecode(pCoder); + +_exit: + tDecoderClear(pCoder); return code; } @@ -923,11 +990,11 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq } tDecoderClear(&dc); - // check - code = tsdbScanAndConvertSubmitMsg(pVnode->pTsdb, pSubmitReq); - if (code) { - goto _exit; - } + // // check + // code = tsdbScanAndConvertSubmitMsg(pVnode->pTsdb, pSubmitReq); + // if (code) { + // goto _exit; + // } for (int32_t i = 0; i < TARRAY_SIZE(pSubmitReq->aSubmitTbData); ++i) { SSubmitTbData *pSubmitTbData = taosArrayGet(pSubmitReq->aSubmitTbData, i); From 05c9d46facb07c5b59e70d1859bb52023d224ef5 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Thu, 16 Feb 2023 10:22:31 +0800 Subject: [PATCH 16/21] fix:wrong tag value --- source/libs/executor/inc/executorimpl.h | 2 +- source/libs/executor/src/filloperator.c | 6 ++++-- source/libs/executor/src/groupoperator.c | 6 ++++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 549ce6ae79..1712cba0f5 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -873,7 +873,7 @@ void getNextIntervalWindow(SInterval* pInterval, STimeWindow* tw, int32_t ord int32_t qAppendTaskStopInfo(SExecTaskInfo* pTaskInfo, SExchangeOpStopInfo* pInfo); int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, int32_t pos, int32_t order, int64_t* pData); -void appendCreateTableRow(SStreamState* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, int64_t groupId, +void appendCreateTableRow(SStreamState* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, uint64_t groupId, SSDataBlock* pSrcBlock, int32_t rowId, SSDataBlock* pDestBlock); SSDataBlock* buildCreateTableBlock(SExprSupp* tbName, SExprSupp* tag); diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index 41e4c990f8..f30fe30e35 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -1259,9 +1259,11 @@ static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) { memcpy(pInfo->pSrcBlock->info.parTbName, pBlock->info.parTbName, TSDB_TABLE_NAME_LEN); pInfo->srcRowIndex = 0; } break; + case STREAM_CREATE_CHILD_TABLE: { + return pBlock; + } break; default: - ASSERT(0); - break; + ASSERTS(pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type"); } } diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index a2c637244b..9fd8f7d3a2 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -985,11 +985,13 @@ static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) { return pDest; } -void appendCreateTableRow(SStreamState* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, int64_t groupId, +void appendCreateTableRow(SStreamState* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, uint64_t groupId, SSDataBlock* pSrcBlock, int32_t rowId, SSDataBlock* pDestBlock) { void* pValue = NULL; if (streamStateGetParName(pState, groupId, &pValue) != 0) { SSDataBlock* pTmpBlock = blockCopyOneRow(pSrcBlock, rowId); + memset(pTmpBlock->info.parTbName, 0, TSDB_TABLE_NAME_LEN); + pTmpBlock->info.id.groupId = groupId; if (pTableSup->numOfExprs > 0) { projectApplyFunctions(pTableSup->pExprInfo, pDestBlock, pTmpBlock, pTableSup->pCtx, pTableSup->numOfExprs, NULL); SColumnInfoData* pTbCol = taosArrayGet(pDestBlock->pDataBlock, UD_TABLE_NAME_COLUMN_INDEX); @@ -999,6 +1001,7 @@ void appendCreateTableRow(SStreamState* pState, SExprSupp* pTableSup, SExprSupp* int32_t len = TMIN(varDataLen(pData), TSDB_TABLE_NAME_LEN - 1); memcpy(tbName, varDataVal(pData), len); streamStatePutParName(pState, groupId, tbName); + memcpy(pTmpBlock->info.parTbName, tbName, len); pDestBlock->info.rows--; } else { void* pTbNameCol = taosArrayGet(pDestBlock->pDataBlock, UD_TABLE_NAME_COLUMN_INDEX); @@ -1013,7 +1016,6 @@ void appendCreateTableRow(SStreamState* pState, SExprSupp* pTableSup, SExprSupp* void* pGpIdCol = taosArrayGet(pDestBlock->pDataBlock, UD_GROUPID_COLUMN_INDEX); colDataAppend(pGpIdCol, pDestBlock->info.rows, (const char*)&groupId, false); - pDestBlock->info.id.groupId = groupId; pDestBlock->info.rows++; blockDataDestroy(pTmpBlock); } From e541528ab6704ea1ae866c476bc0ac2832332230 Mon Sep 17 00:00:00 2001 From: xiaolei li <85657333+xleili@users.noreply.github.com> Date: Thu, 16 Feb 2023 12:43:45 +0800 Subject: [PATCH 17/21] docs: update node-rest doc add insert sample (#20008) * docs: update node-rest doc, add insert sample * remove cn character in examples --- docs/en/14-reference/03-connector/08-node.mdx | 72 ++++++++++++++++- .../node/restexample/insert_example.js | 19 +++++ docs/examples/node/restexample/restChecker.js | 78 ++++++++++++++++++ docs/zh/08-connector/35-node.mdx | 80 ++++++++++++++++++- 4 files changed, 244 insertions(+), 5 deletions(-) create mode 100644 docs/examples/node/restexample/insert_example.js create mode 100644 docs/examples/node/restexample/restChecker.js diff --git a/docs/en/14-reference/03-connector/08-node.mdx b/docs/en/14-reference/03-connector/08-node.mdx index 83479f91da..a6bd772bc9 100644 --- a/docs/en/14-reference/03-connector/08-node.mdx +++ b/docs/en/14-reference/03-connector/08-node.mdx @@ -32,7 +32,9 @@ Please refer to [version support list](/reference/connector#version-support) ## Supported features -### Native connectors + + + 1. Connection Management 2. General Query @@ -41,12 +43,16 @@ Please refer to [version support list](/reference/connector#version-support) 5. Subscription 6. Schemaless -### REST Connector + + 1. Connection Management 2. General Query 3. Continuous Query + + + ## Installation Steps ### Pre-installation preparation @@ -115,6 +121,9 @@ npm install @tdengine/rest ### Verify + + + After installing the TDengine client, use the `nodejsChecker.js` program to verify that the current environment supports Node.js access to TDengine. Verification in details: @@ -131,6 +140,28 @@ node nodejsChecker.js host=localhost - After executing the above steps, the command-line will output the result of `nodejsChecker.js` connecting to the TDengine instance and performing a simple insert and query. + + + +After installing the TDengine client, use the `restChecker.js` program to verify that the current environment supports Node.js access to TDengine. + +Verification in details: + +- Create an installation test folder such as `~/tdengine-test`. Download the [restChecker.js source code](https://github.com/taosdata/TDengine/tree/3.0/docs/examples/node/restexample/restChecker.js) to your local. + +- Execute the following command from the command-line. + +```bash +npm init -y +npm install @tdengine/rest +node restChecker.js +``` + +- After executing the above steps, the command-line will output the result of `restChecker.js` connecting to the TDengine instance and performing a simple insert and query. + + + + ## Establishing a connection Please choose to use one of the connectors. @@ -182,24 +213,61 @@ let cursor = conn.cursor(); #### SQL Write + + + + + + +```js +{{#include docs/examples/node/restexample/insert_example.js}} +``` + + + + #### InfluxDB line protocol write + + + + + + #### OpenTSDB Telnet line protocol write + + + + + + #### OpenTSDB JSON line protocol write + + + + + + ### Querying data + + + + + + ## More sample programs diff --git a/docs/examples/node/restexample/insert_example.js b/docs/examples/node/restexample/insert_example.js new file mode 100644 index 0000000000..d9fddf9f28 --- /dev/null +++ b/docs/examples/node/restexample/insert_example.js @@ -0,0 +1,19 @@ +const { options, connect } = require("@tdengine/rest"); + +async function sqlInsert() { + options.path = "/rest/sql"; + options.host = "localhost"; + options.port = 6041; + let conn = connect(options); + let cursor = conn.cursor(); + try { + let res = await cursor.query('CREATE DATABASE power'); + res = await cursor.query('CREATE STABLE power.meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupId int)'); + res = await cursor.query('INSERT INTO power.d1001 USING power.meters TAGS ("California.SanFrancisco", 2) VALUES (NOW, 10.2, 219, 0.32)'); + console.log("res.getResult()", res.getResult()); + } catch (err) { + console.log(err); + } +} +sqlInsert(); + diff --git a/docs/examples/node/restexample/restChecker.js b/docs/examples/node/restexample/restChecker.js new file mode 100644 index 0000000000..a999684e57 --- /dev/null +++ b/docs/examples/node/restexample/restChecker.js @@ -0,0 +1,78 @@ +const { options, connect } = require("@tdengine/rest"); +options.path = '/rest/sql/' +options.host = 'localhost'; +options.port = 6041; +options.user = "root"; +options.passwd = "taosdata"; + +//optional +// options.url = "http://127.0.0.1:6041"; + +const db = 'rest_ts_db'; +const table = 'rest' +const createDB = `create database if not exists ${db} keep 3650`; +const dropDB = `drop database if exists ${db}`; +const createTB = `create table if not exists ${db}.${table}(ts timestamp,i8 tinyint,i16 smallint,i32 int,i64 bigint,bnr binary(40),nchr nchar(40))`; +const addColumn = `alter table ${db}.${table} add column new_column nchar(40) `; +const dropColumn = `alter table ${db}.${table} drop column new_column`; +const insertSql = `insert into ${db}.${table} values('2022-03-30 18:30:51.567',1,2,3,4,'binary1','nchar1')` + + `('2022-03-30 18:30:51.568',5,6,7,8,'binary2','nchar2')` + + `('2022-03-30 18:30:51.569',9,0,1,2,'binary3','nchar3')`; +const querySql = `select * from ${db}.${table}`; +const errorSql = 'show database'; + +let conn = connect(options); +let cursor = conn.cursor(); + +async function execute(sql, pure = true) { + let result = await cursor.query(sql, pure); + // print query result as taos shell + // Get Result object, return Result object. + console.log("result.getResult()",result.getResult()); + // Get Meta data, return Meta[]|undefined(when execute failed this is undefined). + console.log("result.getMeta()",result.getMeta()); + // Get data,return Array>|undefined(when execute failed this is undefined). + console.log("result.getData()",result.getData()); + // Get affect rows,return number|undefined(when execute failed this is undefined). + console.log("result.getAffectRows()",result.getAffectRows()); + // Get command,return SQL send to server(need to `query(sql,false)`,set 'pure=false',default true). + console.log("result.getCommand()",result.getCommand()); + // Get error code ,return number|undefined(when execute failed this is undefined). + console.log("result.getErrCode()",result.getErrCode()); + // Get error string,return string|undefined(when execute failed this is undefined). + console.log("result.getErrStr()",result.getErrStr()); +} + +(async () => { + // start execute time + let start = new Date().getTime(); + await execute(createDB); + console.log("-----------------------------------") + + await execute(createTB); + console.log("-----------------------------------") + + await execute(addColumn); + console.log("----------------------------------") + + await execute(dropColumn); + console.log("-----------------------------------") + + await execute(insertSql); + console.log("-----------------------------------") + + await execute(querySql); + console.log("-----------------------------------") + + await execute(errorSql); + console.log("-----------------------------------") + + await execute(dropDB); + // finish time + let end = new Date().getTime(); + console.log("total spend time:%d ms",end - start); +})() + + + + diff --git a/docs/zh/08-connector/35-node.mdx b/docs/zh/08-connector/35-node.mdx index f2aff41da2..1cdacf3c1b 100644 --- a/docs/zh/08-connector/35-node.mdx +++ b/docs/zh/08-connector/35-node.mdx @@ -13,6 +13,8 @@ import NodeInfluxLine from "../07-develop/03-insert-data/_js_line.mdx"; import NodeOpenTSDBTelnet from "../07-develop/03-insert-data/_js_opts_telnet.mdx"; import NodeOpenTSDBJson from "../07-develop/03-insert-data/_js_opts_json.mdx"; import NodeQuery from "../07-develop/04-query-data/_js.mdx"; +import RESTQuery from "../07-develop/04-query-data/_js.mdx"; +import RESTSQLInsert from "../07-develop/03-insert-data/_js_sql.mdx"; `@tdengine/client` 和 `@tdengine/rest` 是 TDengine 的官方 Node.js 语言连接器。 Node.js 开发人员可以通过它开发可以存取 TDengine 集群数据的应用软件。注意:从 TDengine 3.0 开始 Node.js 原生连接器的包名由 `td2.0-connector` 改名为 `@tdengine/client` 而 rest 连接器的包名由 `td2.0-rest-connector` 改为 `@tdengine/rest`。并且不与 TDengine 2.x 兼容。 @@ -31,7 +33,8 @@ REST 连接器支持所有能运行 Node.js 的平台。 ## 支持的功能特性 -### 原生连接器 + + 1. 连接管理 2. 普通查询 @@ -40,12 +43,17 @@ REST 连接器支持所有能运行 Node.js 的平台。 5. 订阅功能 6. Schemaless -### REST 连接器 + + 1. 连接管理 2. 普通查询 3. 连续查询 + + + + ## 安装步骤 ### 安装前准备 @@ -114,6 +122,9 @@ npm install @tdengine/rest ### 安装验证 + + + 在安装好 TDengine 客户端后,使用 nodejsChecker.js 程序能够验证当前环境是否支持 Node.js 方式访问 TDengine。 验证方法: @@ -130,11 +141,35 @@ node nodejsChecker.js host=localhost - 执行以上步骤后,在命令行会输出 nodejsChecker.js 连接 TDengine 实例,并执行简单插入和查询的结果。 + + + +在安装好 TDengine 客户端后,使用 nodejsChecker.js 程序能够验证当前环境是否支持 Node.js 方式访问 TDengine。 + +验证方法: + +- 新建安装验证目录,例如:`~/tdengine-test`,下载 GitHub 上 [restChecker.js 源代码](https://github.com/taosdata/TDengine/tree/3.0/docs/examples/node/restexample/restChecker.js)到本地。 + +- 在命令行中执行以下命令。 + +```bash +npm init -y +npm install @tdengine/rest +node restChecker.js +``` + +- 执行以上步骤后,在命令行会输出 restChecker.js 连接 TDengine 实例,并执行简单插入和查询的结果。 + + + + + + ## 建立连接 请选择使用一种连接器。 - + 安装并引用 `@tdengine/client` 包。 @@ -181,24 +216,63 @@ let cursor = conn.cursor(); #### SQL 写入 + + + + + + +```js +{{#include docs/examples/node/restexample/insert_example.js}} +``` + + + + + + #### InfluxDB 行协议写入 + + + + + + #### OpenTSDB Telnet 行协议写入 + + + + + + #### OpenTSDB JSON 行协议写入 + + + + + + ### 查询数据 + + + + + + ## 更多示例程序 From 13bdd5afae166ea4307ffbb6992385e1131e0383 Mon Sep 17 00:00:00 2001 From: xiaolei li <85657333+xleili@users.noreply.github.com> Date: Thu, 16 Feb 2023 14:00:20 +0800 Subject: [PATCH 18/21] docs: update node-rest examples and reference (#20011) --- docs/en/14-reference/03-connector/08-node.mdx | 8 ++++++++ docs/examples/node/restexample/query_example.js | 16 ++++++++++++++++ docs/zh/08-connector/35-node.mdx | 10 ++++++++-- 3 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 docs/examples/node/restexample/query_example.js diff --git a/docs/en/14-reference/03-connector/08-node.mdx b/docs/en/14-reference/03-connector/08-node.mdx index a6bd772bc9..b7c2e6b765 100644 --- a/docs/en/14-reference/03-connector/08-node.mdx +++ b/docs/en/14-reference/03-connector/08-node.mdx @@ -265,6 +265,14 @@ let cursor = conn.cursor(); + + + + +```js +{{#include docs/examples/node/restexample/query_example.js}} +``` + diff --git a/docs/examples/node/restexample/query_example.js b/docs/examples/node/restexample/query_example.js new file mode 100644 index 0000000000..0edce64a24 --- /dev/null +++ b/docs/examples/node/restexample/query_example.js @@ -0,0 +1,16 @@ +const { options, connect } = require("@tdengine/rest"); + +async function query() { + options.path = "/rest/sql"; + options.host = "localhost"; + options.port = 6041; + let conn = connect(options); + let cursor = conn.cursor(); + try { + let res = await cursor.query('select * from power.meters'); + console.log("res.getResult()", res.getResult()); + } catch (err) { + console.log(err); + } +} +query(); diff --git a/docs/zh/08-connector/35-node.mdx b/docs/zh/08-connector/35-node.mdx index 1cdacf3c1b..25f8bdf177 100644 --- a/docs/zh/08-connector/35-node.mdx +++ b/docs/zh/08-connector/35-node.mdx @@ -13,8 +13,6 @@ import NodeInfluxLine from "../07-develop/03-insert-data/_js_line.mdx"; import NodeOpenTSDBTelnet from "../07-develop/03-insert-data/_js_opts_telnet.mdx"; import NodeOpenTSDBJson from "../07-develop/03-insert-data/_js_opts_json.mdx"; import NodeQuery from "../07-develop/04-query-data/_js.mdx"; -import RESTQuery from "../07-develop/04-query-data/_js.mdx"; -import RESTSQLInsert from "../07-develop/03-insert-data/_js_sql.mdx"; `@tdengine/client` 和 `@tdengine/rest` 是 TDengine 的官方 Node.js 语言连接器。 Node.js 开发人员可以通过它开发可以存取 TDengine 集群数据的应用软件。注意:从 TDengine 3.0 开始 Node.js 原生连接器的包名由 `td2.0-connector` 改名为 `@tdengine/client` 而 rest 连接器的包名由 `td2.0-rest-connector` 改为 `@tdengine/rest`。并且不与 TDengine 2.x 兼容。 @@ -270,6 +268,14 @@ let cursor = conn.cursor(); + + + + +```js +{{#include docs/examples/node/restexample/query_example.js}} +``` + From 3c37ad11e67e37b78f359d9cb409b1ab0bc821ef Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 16 Feb 2023 15:55:59 +0800 Subject: [PATCH 19/21] fix: other problems --- source/dnode/vnode/src/vnd/vnodeSvr.c | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 983bea3706..5eeb11b2f5 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -13,7 +13,6 @@ * along with this program. If not, see . */ -#include #include "tencode.h" #include "tmsg.h" #include "vnd.h" @@ -36,7 +35,7 @@ static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodePreprocessCreateTableReq(SVnode *pVnode, SDecoder *pCoder, int64_t ctime) { +static int32_t vnodePreprocessCreateTableReq(SVnode *pVnode, SDecoder *pCoder, int64_t ctime, int64_t *pUid) { int32_t code = 0; int32_t lino = 0; @@ -75,6 +74,7 @@ _exit: vError("vgId:%d %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); } else { vTrace("vgId:%d %s done, table:%s uid generated:%" PRId64, TD_VID(pVnode), __func__, name, uid); + if (pUid) *pUid = uid; } return code; } @@ -97,7 +97,7 @@ static int32_t vnodePreProcessCreateTableMsg(SVnode *pVnode, SRpcMsg *pMsg) { TSDB_CHECK_CODE(code, lino, _exit); } for (int32_t iReq = 0; iReq < nReqs; iReq++) { - code = vnodePreprocessCreateTableReq(pVnode, &dc, ctime); + code = vnodePreprocessCreateTableReq(pVnode, &dc, ctime, NULL); TSDB_CHECK_CODE(code, lino, _exit); } @@ -123,8 +123,9 @@ static int32_t vnodePreProcessSubmitTbData(SVnode *pVnode, SDecoder *pCoder, int TSDB_CHECK_CODE(code, lino, _exit); } + int64_t uid; if (submitTbData.flags & SUBMIT_REQ_AUTO_CREATE_TABLE) { - code = vnodePreprocessCreateTableReq(pVnode, pCoder, ctime); + code = vnodePreprocessCreateTableReq(pVnode, pCoder, ctime, &uid); TSDB_CHECK_CODE(code, lino, _exit); } @@ -133,10 +134,13 @@ static int32_t vnodePreProcessSubmitTbData(SVnode *pVnode, SDecoder *pCoder, int code = TSDB_CODE_INVALID_MSG; TSDB_CHECK_CODE(code, lino, _exit); } - if (tDecodeI64(pCoder, &submitTbData.uid) < 0) { - code = TSDB_CODE_INVALID_MSG; - TSDB_CHECK_CODE(code, lino, _exit); + + if (submitTbData.flags & SUBMIT_REQ_AUTO_CREATE_TABLE) { + *(int64_t *)(pCoder->data + pCoder->pos) = uid; + } else { + tDecodeI64(pCoder, &submitTbData.uid); } + if (tDecodeI32v(pCoder, &submitTbData.sver) < 0) { code = TSDB_CODE_INVALID_MSG; TSDB_CHECK_CODE(code, lino, _exit); @@ -188,7 +192,7 @@ static int32_t vnodePreProcessSubmitTbData(SVnode *pVnode, SDecoder *pCoder, int tEndDecode(pCoder); _exit: - return 0; + return code; } static int32_t vnodePreProcessSubmitMsg(SVnode *pVnode, SRpcMsg *pMsg) { int32_t code = 0; @@ -990,12 +994,6 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq } tDecoderClear(&dc); - // // check - // code = tsdbScanAndConvertSubmitMsg(pVnode->pTsdb, pSubmitReq); - // if (code) { - // goto _exit; - // } - for (int32_t i = 0; i < TARRAY_SIZE(pSubmitReq->aSubmitTbData); ++i) { SSubmitTbData *pSubmitTbData = taosArrayGet(pSubmitReq->aSubmitTbData, i); From 1640ccc4b28347a84c53bcc209746406c714d388 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 16 Feb 2023 17:50:45 +0800 Subject: [PATCH 20/21] fix: another error --- source/dnode/vnode/src/vnd/vnodeSvr.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 5eeb11b2f5..a34836959c 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -137,6 +137,7 @@ static int32_t vnodePreProcessSubmitTbData(SVnode *pVnode, SDecoder *pCoder, int if (submitTbData.flags & SUBMIT_REQ_AUTO_CREATE_TABLE) { *(int64_t *)(pCoder->data + pCoder->pos) = uid; + pCoder->pos += sizeof(int64_t); } else { tDecodeI64(pCoder, &submitTbData.uid); } From 48fe2c0e06f88a843c96050a1cb81bac9c2413c0 Mon Sep 17 00:00:00 2001 From: xinsheng Ren <285808407@qq.com> Date: Thu, 16 Feb 2023 18:12:02 +0800 Subject: [PATCH 21/21] install description on mac (#20016) Co-authored-by: facetosea <25808407@qq.com> --- packaging/tools/mac_before_install.txt | 2 +- packaging/tools/mac_before_install_client.txt | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 packaging/tools/mac_before_install_client.txt diff --git a/packaging/tools/mac_before_install.txt b/packaging/tools/mac_before_install.txt index 3b6d610e88..a58bab7b65 100644 --- a/packaging/tools/mac_before_install.txt +++ b/packaging/tools/mac_before_install.txt @@ -1,4 +1,4 @@ -TDengine is a high-efficient, scalable, high-available distributed time-series database, which makes a lot of optimizations on inserting and querying data, which is far more efficient than normal regular databases. So TDengine can meet the high requirements of IOT and other areas on storing and querying a large amount of data. +TDengine is an open-source, cloud-native time-series database optimized for Internet of Things (IoT), Connected Cars, and Industrial IoT. With its built-in caching, stream processing, and data subscription capabilities, TDengine offers a simplified solution for time-series data processing. To configure TDengine : edit /etc/taos/taos.cfg To start service : launchctl start com.tdengine.taosd diff --git a/packaging/tools/mac_before_install_client.txt b/packaging/tools/mac_before_install_client.txt new file mode 100644 index 0000000000..d66fa67358 --- /dev/null +++ b/packaging/tools/mac_before_install_client.txt @@ -0,0 +1,7 @@ +TDengine is an open-source, cloud-native time-series database optimized for Internet of Things (IoT), Connected Cars, and Industrial IoT. With its built-in caching, stream processing, and data subscription capabilities, TDengine offers a simplified solution for time-series data processing. + +Once it's installed, please take the steps below: +1: open a terminal/shell in Mac +2: if connecting to Cloud Service, follow the instructions on your cloud service account and configure the environment variable +3: if connecting to another TDengine Service, you can also view help information via "taos --help" +4: execute command taos \ No newline at end of file