From 47ff0e34b056c7d788624ba74c1fca9287ddeefe Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 1 Sep 2022 20:16:02 +0800 Subject: [PATCH 1/7] refactor idx code --- source/libs/index/inc/indexInt.h | 29 +++++----- source/libs/index/src/index.c | 8 +-- source/libs/index/src/indexCache.c | 5 +- source/libs/index/src/indexComm.c | 4 +- source/libs/index/src/indexFilter.c | 83 +++++++++++++--------------- source/libs/index/src/indexFstFile.c | 6 +- source/libs/index/src/indexTfile.c | 10 ++-- source/libs/transport/src/transCli.c | 6 +- 8 files changed, 66 insertions(+), 85 deletions(-) diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index 065f4acb57..9605528ad6 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -40,26 +40,31 @@ extern "C" { #define indexTrace(...) do { if (idxDebugFlag & DEBUG_TRACE) { taosPrintLog("IDX", DEBUG_TRACE, idxDebugFlag, __VA_ARGS__);} } while (0) // clang-format on +extern void* indexQhandle; + typedef enum { LT, LE, GT, GE, CONTAINS, EQ } RangeType; typedef enum { kTypeValue, kTypeDeletion } STermValueType; typedef enum { kRebuild, kFinished } SIdxStatus; typedef struct SIndexStat { - int32_t totalAdded; // - int32_t totalDeled; // - int32_t totalUpdated; // - int32_t totalTerms; // - int32_t distinctCol; // distinct column + int32_t total; + int32_t add; // + int32_t del; // + int32_t update; // + int32_t terms; // + int32_t distCol; // distinct column } SIndexStat; struct SIndex { + SIndexOpts opts; + int64_t refId; void* cache; void* tindex; SHashObj* colObj; // < field name, field id> - int64_t suid; // current super table id, -1 is normal table - int32_t cVersion; // current version allocated to cache + int64_t suid; // current super table id, -1 is normal table + int32_t version; // current version allocated to cache SLRUCache* lru; char* path; @@ -68,7 +73,6 @@ struct SIndex { TdThreadMutex mtx; tsem_t sem; bool quit; - SIndexOpts opts; }; struct SIndexMultiTermQuery { @@ -111,14 +115,15 @@ typedef struct Iterate { void iterateValueDestroy(IterateValue* iv, bool destroy); -extern void* indexQhandle; - typedef struct TFileCacheKey { uint64_t suid; uint8_t colType; char* colName; int32_t nColName; } ICacheKey; + +int32_t idxSerialCacheKey(ICacheKey* key, char* buf); + int idxFlushCacheToTFile(SIndex* sIdx, void*, bool quit); int64_t idxAddRef(void* p); @@ -126,10 +131,6 @@ int32_t idxRemoveRef(int64_t ref); void idxAcquireRef(int64_t ref); void idxReleaseRef(int64_t ref); -int32_t idxSerialCacheKey(ICacheKey* key, char* buf); -// int32_t indexSerialKey(ICacheKey* key, char* buf); -// int32_t indexSerialTermKey(SIndexTerm* itm, char* buf); - #define IDX_TYPE_CONTAIN_EXTERN_TYPE(ty, exTy) (((ty >> 4) & (exTy)) != 0) #define IDX_TYPE_GET_TYPE(ty) (ty & 0x0F) diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index be64a8b44d..d9a6b80f3d 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -25,10 +25,6 @@ #include "tref.h" #include "tsched.h" -#ifdef USE_LUCENE -#include "lucene++/Lucene_c.h" -#endif - #define INDEX_NUM_OF_THREADS 5 #define INDEX_QUEUE_SIZE 200 @@ -74,7 +70,7 @@ void indexCleanup() { typedef struct SIdxColInfo { int colId; // generated by index internal - int cVersion; + int version; } SIdxColInfo; static TdThreadOnce isInit = PTHREAD_ONCE_INIT; @@ -123,7 +119,7 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { } idx->colObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); - idx->cVersion = 1; + idx->version = 1; idx->path = tstrdup(path); taosThreadMutexInit(&idx->mtx, NULL); tsem_init(&idx->sem, 0, 0); diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index 794b85d244..7e867db755 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -566,7 +566,6 @@ int idxCachePut(void* cache, SIndexTerm* term, uint64_t uid) { taosThreadMutexUnlock(&pCache->mtx); idxCacheUnRef(pCache); return 0; - // encode end } void idxCacheForceToMerge(void* cache) { IndexCache* pCache = cache; @@ -602,10 +601,10 @@ static int32_t idxQueryMem(MemTable* mem, SIndexTermQuery* query, SIdxTRslt* tr, } } int idxCacheSearch(void* cache, SIndexTermQuery* query, SIdxTRslt* result, STermValueType* s) { - int64_t st = taosGetTimestampUs(); if (cache == NULL) { return 0; } + IndexCache* pCache = cache; MemTable *mem = NULL, *imm = NULL; @@ -616,6 +615,8 @@ int idxCacheSearch(void* cache, SIndexTermQuery* query, SIdxTRslt* result, STerm idxMemRef(imm); taosThreadMutexUnlock(&pCache->mtx); + int64_t st = taosGetTimestampUs(); + int ret = (mem && mem->mem) ? idxQueryMem(mem, query, result, s) : 0; if (ret == 0 && *s != kTypeDeletion) { // continue search in imm diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index cd52d122f7..691eb6771c 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -178,9 +178,9 @@ TExeCond tDoCompare(__compar_fn_t func, int8_t comparType, void* a, void* b) { // optime later int32_t ret = func(a, b); switch (comparType) { - case QUERY_LESS_THAN: { + case QUERY_LESS_THAN: if (ret < 0) return MATCH; - } break; + break; case QUERY_LESS_EQUAL: { if (ret <= 0) return MATCH; break; diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 75844ce76f..94a857c1d7 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -26,6 +26,44 @@ #define SIF_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) #define SIF_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) #define SIF_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) + +typedef union { + uint8_t u8; + uint16_t u16; + uint32_t u32; + uint64_t u64; + + int8_t i8; + int16_t i16; + int32_t i32; + int64_t i64; + + double d; + float f; +} SDataTypeBuf; + +#define SIF_DATA_CONVERT(type, val, dst) \ + do { \ + if (type == TSDB_DATA_TYPE_DOUBLE) \ + dst = GET_DOUBLE_VAL(val); \ + else if (type == TSDB_DATA_TYPE_BIGINT) \ + dst = *(int64_t *)val; \ + else if (type == TSDB_DATA_TYPE_INT) \ + dst = *(int32_t *)val; \ + else if (type == TSDB_DATA_TYPE_SMALLINT) \ + dst = *(int16_t *)val; \ + else if (type == TSDB_DATA_TYPE_TINYINT) \ + dst = *(int8_t *)val; \ + else if (type == TSDB_DATA_TYPE_UTINYINT) \ + dst = *(uint8_t *)val; \ + else if (type == TSDB_DATA_TYPE_USMALLINT) \ + dst = *(uint16_t *)val; \ + else if (type == TSDB_DATA_TYPE_UINT) \ + dst = *(uint32_t *)val; \ + else if (type == TSDB_DATA_TYPE_UBIGINT) \ + dst = *(uint64_t *)val; \ + } while (0); + // clang-format on typedef struct SIFParam { SHashObj *pFilter; @@ -48,7 +86,6 @@ typedef struct SIFCtx { SHashObj *pRes; /* element is SIFParam */ bool noExec; // true: just iterate condition tree, and add hint to executor plan SIndexMetaArg arg; - // SIdxFltStatus st; } SIFCtx; static int32_t sifGetFuncFromSql(EOperatorType src, EIndexQueryType *dst) { @@ -75,11 +112,6 @@ static int32_t sifGetFuncFromSql(EOperatorType src, EIndexQueryType *dst) { typedef int32_t (*sif_func_t)(SIFParam *left, SIFParam *rigth, SIFParam *output); static sif_func_t sifNullFunc = NULL; -// typedef struct SIFWalkParm -// construct tag filter operator later -// static void destroyTagFilterOperatorInfo(void *param) { -// STagFilterOperatorInfo *pInfo = (STagFilterOperatorInfo *)param; -//} static void sifFreeParam(SIFParam *param) { if (param == NULL) return; @@ -365,42 +397,6 @@ static Filter sifGetFilterFunc(EIndexQueryType type, bool *reverse) { } return NULL; } -typedef union { - uint8_t u8; - uint16_t u16; - uint32_t u32; - uint64_t u64; - - int8_t i8; - int16_t i16; - int32_t i32; - int64_t i64; - - double d; - float f; -} SDataTypeBuf; - -#define SIF_DATA_CONVERT(type, val, dst) \ - do { \ - if (type == TSDB_DATA_TYPE_DOUBLE) \ - dst = GET_DOUBLE_VAL(val); \ - else if (type == TSDB_DATA_TYPE_BIGINT) \ - dst = *(int64_t *)val; \ - else if (type == TSDB_DATA_TYPE_INT) \ - dst = *(int32_t *)val; \ - else if (type == TSDB_DATA_TYPE_SMALLINT) \ - dst = *(int16_t *)val; \ - else if (type == TSDB_DATA_TYPE_TINYINT) \ - dst = *(int8_t *)val; \ - else if (type == TSDB_DATA_TYPE_UTINYINT) \ - dst = *(uint8_t *)val; \ - else if (type == TSDB_DATA_TYPE_USMALLINT) \ - dst = *(uint16_t *)val; \ - else if (type == TSDB_DATA_TYPE_UINT) \ - dst = *(uint32_t *)val; \ - else if (type == TSDB_DATA_TYPE_UBIGINT) \ - dst = *(uint64_t *)val; \ - } while (0); static void sifSetFltParam(SIFParam *left, SIFParam *right, SDataTypeBuf *typedata, SMetaFltParam *param) { int8_t ltype = left->colValType, rtype = right->colValType; @@ -693,11 +689,8 @@ static int32_t sifExecLogic(SLogicConditionNode *node, SIFCtx *ctx, SIFParam *ou for (int32_t m = 0; m < node->pParameterList->length; m++) { if (node->condType == LOGIC_COND_TYPE_AND) { taosArrayAddAll(output->result, params[m].result); - // taosArrayDestroy(params[m].result); - // params[m].result = NULL; } else if (node->condType == LOGIC_COND_TYPE_OR) { taosArrayAddAll(output->result, params[m].result); - // params[m].result = NULL; } else if (node->condType == LOGIC_COND_TYPE_NOT) { // taosArrayAddAll(output->result, params[m].result); } diff --git a/source/libs/index/src/indexFstFile.c b/source/libs/index/src/indexFstFile.c index 1900e50973..e6d1edfeda 100644 --- a/source/libs/index/src/indexFstFile.c +++ b/source/libs/index/src/indexFstFile.c @@ -211,9 +211,7 @@ IdxFstFile* idxFileCreate(void* wrt) { return cw; } void idxFileDestroy(IdxFstFile* cw) { - // free wrt object: close fd or free mem idxFileFlush(cw); - // idxFileCtxDestroy((IFileCtx *)(cw->wrt)); taosMemoryFree(cw); } @@ -222,10 +220,8 @@ int idxFileWrite(IdxFstFile* write, uint8_t* buf, uint32_t len) { return 0; } // update checksum - // write data to file/socket or mem IFileCtx* ctx = write->wrt; - - int nWrite = ctx->write(ctx, buf, len); + int nWrite = ctx->write(ctx, buf, len); assert(nWrite == len); write->count += len; diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index 0a47fc0f16..48a0c631cf 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -183,13 +183,14 @@ TFileReader* tfileReaderCreate(IFileCtx* ctx) { return NULL; } reader->ctx = ctx; + reader->remove = false; if (0 != tfileReaderVerify(reader)) { indexError("invalid tfile, suid:%" PRIu64 ", colName:%s", reader->header.suid, reader->header.colName); tfileReaderDestroy(reader); return NULL; } - // T_REF_INC(reader); + if (0 != tfileReaderLoadHeader(reader)) { indexError("failed to load index header, suid:%" PRIu64 ", colName:%s", reader->header.suid, reader->header.colName); @@ -203,7 +204,6 @@ TFileReader* tfileReaderCreate(IFileCtx* ctx) { tfileReaderDestroy(reader); return NULL; } - reader->remove = false; return reader; } @@ -211,7 +211,6 @@ void tfileReaderDestroy(TFileReader* reader) { if (reader == NULL) { return; } - // T_REF_INC(reader); fstDestroy(reader->fst); if (reader->remove) { indexInfo("%s is removed", reader->ctx->file.buf); @@ -222,6 +221,7 @@ void tfileReaderDestroy(TFileReader* reader) { taosMemoryFree(reader); } + static int32_t tfSearchTerm(void* reader, SIndexTerm* tem, SIdxTRslt* tr) { int ret = 0; char* p = tem->colVal; @@ -494,7 +494,6 @@ int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SIdxTRslt* tr TFileWriter* tfileWriterOpen(char* path, uint64_t suid, int64_t version, const char* colName, uint8_t colType) { char fullname[256] = {0}; tfileGenFileFullName(fullname, path, suid, colName, version); - // indexInfo("open write file name %s", fullname); IFileCtx* wcx = idxFileCtxCreate(TFILE, fullname, false, 1024 * 1024 * 64); if (wcx == NULL) { return NULL; @@ -503,8 +502,8 @@ TFileWriter* tfileWriterOpen(char* path, uint64_t suid, int64_t version, const c TFileHeader tfh = {0}; tfh.suid = suid; tfh.version = version; - memcpy(tfh.colName, colName, strlen(colName)); tfh.colType = colType; + memcpy(tfh.colName, colName, strlen(colName)); return tfileWriterCreate(wcx, &tfh); } @@ -706,7 +705,6 @@ static bool tfileIteratorNext(Iterate* iiter) { iv->type = ADD_VALUE; // value in tfile always ADD_VALUE iv->colVal = colVal; return true; - // std::string key(ch, sz); } static IterateValue* tifileIterateGetValue(Iterate* iter) { return &iter->val; } diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 41688c7330..4b5441f738 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1304,11 +1304,7 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { pTransInst->cfp(pTransInst->parent, pResp, NULL); return 0; } - /* - * no retry - * 1. query conn - * 2. rpc thread already receive quit msg - */ + STransConnCtx* pCtx = pMsg->ctx; int32_t code = pResp->code; From 9dcfba1970801506bc80d08aba4e3f5795180608 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 1 Sep 2022 20:34:36 +0800 Subject: [PATCH 2/7] refactor index code --- source/libs/index/src/indexFilter.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 94a857c1d7..e7f221de3d 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -26,6 +26,7 @@ #define SIF_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) #define SIF_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) #define SIF_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) +// clang-format on typedef union { uint8_t u8; @@ -64,7 +65,6 @@ typedef union { dst = *(uint64_t *)val; \ } while (0); -// clang-format on typedef struct SIFParam { SHashObj *pFilter; From 502d75302ec3115d5e8e968147b71c3717682cdf Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 2 Sep 2022 10:00:15 +0800 Subject: [PATCH 3/7] refactor(taosx): filter meta --- source/dnode/vnode/src/tq/tqExec.c | 51 ++++++++++++++++------------- source/libs/executor/src/executor.c | 2 +- 2 files changed, 29 insertions(+), 24 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index 8c3fa25446..d00907f677 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -241,6 +241,20 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp continue; } } + if (pHandle->fetchMeta) { + SSubmitBlk* pBlk = pReader->pBlock; + if (pBlk->schemaLen > 0) { + if (pRsp->createTableNum == 0) { + pRsp->createTableLen = taosArrayInit(0, sizeof(int32_t)); + pRsp->createTableReq = taosArrayInit(0, sizeof(void*)); + } + void* createReq = taosMemoryCalloc(1, pBlk->schemaLen); + memcpy(createReq, pBlk->data, pBlk->schemaLen); + taosArrayPush(pRsp->createTableLen, &pBlk->schemaLen); + taosArrayPush(pRsp->createTableReq, &createReq); + pRsp->createTableNum++; + } + } tqAddBlockDataToRsp(&block, (SMqDataRsp*)pRsp, taosArrayGetSize(block.pDataBlock)); blockDataFreeRes(&block); tqAddBlockSchemaToRsp(pExec, (SMqDataRsp*)pRsp); @@ -261,34 +275,25 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp continue; } } + if (pHandle->fetchMeta) { + SSubmitBlk* pBlk = pReader->pBlock; + if (pBlk->schemaLen > 0) { + if (pRsp->createTableNum == 0) { + pRsp->createTableLen = taosArrayInit(0, sizeof(int32_t)); + pRsp->createTableReq = taosArrayInit(0, sizeof(void*)); + } + void* createReq = taosMemoryCalloc(1, pBlk->schemaLen); + memcpy(createReq, pBlk->data, pBlk->schemaLen); + taosArrayPush(pRsp->createTableLen, &pBlk->schemaLen); + taosArrayPush(pRsp->createTableReq, &createReq); + pRsp->createTableNum++; + } + } tqAddBlockDataToRsp(&block, (SMqDataRsp*)pRsp, taosArrayGetSize(block.pDataBlock)); blockDataFreeRes(&block); tqAddBlockSchemaToRsp(pExec, (SMqDataRsp*)pRsp); pRsp->blockNum++; } -#if 1 - if (pHandle->fetchMeta && pRsp->blockNum) { - SSubmitMsgIter iter = {0}; - tInitSubmitMsgIter(pReq, &iter); - STaosxRsp* pXrsp = (STaosxRsp*)pRsp; - while (1) { - SSubmitBlk* pBlk = NULL; - if (tGetSubmitMsgNext(&iter, &pBlk) < 0) break; - if (pBlk == NULL) break; - if (pBlk->schemaLen > 0) { - if (pXrsp->createTableNum == 0) { - pXrsp->createTableLen = taosArrayInit(0, sizeof(int32_t)); - pXrsp->createTableReq = taosArrayInit(0, sizeof(void*)); - } - void* createReq = taosMemoryCalloc(1, pBlk->schemaLen); - memcpy(createReq, pBlk->data, pBlk->schemaLen); - taosArrayPush(pXrsp->createTableLen, &pBlk->schemaLen); - taosArrayPush(pXrsp->createTableReq, &createReq); - pXrsp->createTableNum++; - } - } - } -#endif } if (pRsp->blockNum == 0) { diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 278f02b228..a3e0b2b733 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -811,7 +811,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT } } - // TODO after dropping table, table may be not found + // TODO after dropping table, table may not found ASSERT(found); if (pTableScanInfo->dataReader == NULL) { From 18b342456752867d5a548328e9aef20b6c4b196a Mon Sep 17 00:00:00 2001 From: Linhe Huo Date: Fri, 2 Sep 2022 11:52:35 +0800 Subject: [PATCH 4/7] feat: support taosx packaging and install (#16597) --- packaging/tools/install.sh | 3 +++ packaging/tools/make_install.sh | 4 ++++ packaging/tools/makepkg.sh | 2 ++ 3 files changed, 9 insertions(+) diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 39606ead30..f2f72acafa 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -33,6 +33,7 @@ adapterName="taosadapter" benchmarkName="taosBenchmark" dumpName="taosdump" demoName="taosdemo" +xname="taosx" data_dir=${dataDir} log_dir=${logDir} @@ -199,6 +200,7 @@ function install_bin() { ${csudo}rm -f ${bin_link_dir}/${demoName} || : ${csudo}rm -f ${bin_link_dir}/${benchmarkName} || : ${csudo}rm -f ${bin_link_dir}/${dumpName} || : + ${csudo}rm -f ${bin_link_dir}/${xname} || : ${csudo}rm -f ${bin_link_dir}/set_core || : ${csudo}rm -f ${bin_link_dir}/TDinsight.sh || : @@ -212,6 +214,7 @@ function install_bin() { [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -s ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${demoName} || : [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -s ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${benchmarkName} || : [ -x ${install_main_dir}/bin/${dumpName} ] && ${csudo}ln -s ${install_main_dir}/bin/${dumpName} ${bin_link_dir}/${dumpName} || : + [ -x ${install_main_dir}/bin/${xname} ] && ${csudo}ln -s ${install_main_dir}/bin/${dumpName} ${bin_link_dir}/${xname} || : [ -x ${install_main_dir}/bin/TDinsight.sh ] && ${csudo}ln -s ${install_main_dir}/bin/TDinsight.sh ${bin_link_dir}/TDinsight.sh || : [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || : [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index f554942ce3..a6dceeeaad 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -172,6 +172,7 @@ function install_bin() { ${csudo}rm -f ${bin_link_dir}/udfd || : ${csudo}rm -f ${bin_link_dir}/taosdemo || : ${csudo}rm -f ${bin_link_dir}/taosdump || : + ${csudo}rm -f ${bin_link_dir}/taosx || : if [ "$osType" != "Darwin" ]; then ${csudo}rm -f ${bin_link_dir}/perfMonitor || : @@ -184,6 +185,7 @@ function install_bin() { [ -f ${binary_dir}/build/bin/taosdump ] && ${csudo}cp -r ${binary_dir}/build/bin/taosdump ${install_main_dir}/bin || : [ -f ${binary_dir}/build/bin/taosadapter ] && ${csudo}cp -r ${binary_dir}/build/bin/taosadapter ${install_main_dir}/bin || : [ -f ${binary_dir}/build/bin/udfd ] && ${csudo}cp -r ${binary_dir}/build/bin/udfd ${install_main_dir}/bin || : + [ -f ${binary_dir}/build/bin/taosx ] && ${csudo}cp -r ${binary_dir}/build/bin/taosx ${install_main_dir}/bin || : ${csudo}cp -r ${binary_dir}/build/bin/${serverName} ${install_main_dir}/bin || : ${csudo}cp -r ${script_dir}/taosd-dump-cfg.gdb ${install_main_dir}/bin || : @@ -199,6 +201,7 @@ function install_bin() { [ -x ${install_main_dir}/bin/udfd ] && ${csudo}ln -s ${install_main_dir}/bin/udfd ${bin_link_dir}/udfd || : [ -x ${install_main_dir}/bin/taosdump ] && ${csudo}ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || : [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo}ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || : + [ -x ${install_main_dir}/bin/taosx ] && ${csudo}ln -s ${install_main_dir}/bin/taosx ${bin_link_dir}/taosx || : [ -x ${install_main_dir}/bin/perfMonitor ] && ${csudo}ln -s ${install_main_dir}/bin/perfMonitor ${bin_link_dir}/perfMonitor || : [ -x ${install_main_dir}/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || : @@ -215,6 +218,7 @@ function install_bin() { [ -x ${install_main_dir}/bin/udfd ] || [ -x ${install_main_2_dir}/bin/udfd ] && ${csudo}ln -s ${install_main_dir}/bin/udfd ${bin_link_dir}/udfd || ${csudo}ln -s ${install_main_2_dir}/bin/udfd || : [ -x ${install_main_dir}/bin/taosdump ] || [ -x ${install_main_2_dir}/bin/taosdump ] && ${csudo}ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || ln -s ${install_main_2_dir}/bin/taosdump ${bin_link_dir}/taosdump || : [ -x ${install_main_dir}/bin/taosdemo ] || [ -x ${install_main_2_dir}/bin/taosdemo ] && ${csudo}ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || ln -s ${install_main_2_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || : + [ -x ${install_main_dir}/bin/taosx ] || [ -x ${install_main_2_dir}/bin/taosx ] && ${csudo}ln -s ${install_main_dir}/bin/taosx ${bin_link_dir}/taosx || ln -s ${install_main_2_dir}/bin/taosx ${bin_link_dir}/taosx || : fi } diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index f5e3bf1882..2305b96b36 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -80,10 +80,12 @@ else ${build_dir}/bin/taosBenchmark \ ${build_dir}/bin/TDinsight.sh \ $tdinsight_caches" + [ -f ${build_dir}/bin/taosx ] && taosx_bin="${build_dir}/bin/taosx" bin_files="${build_dir}/bin/${serverName} \ ${build_dir}/bin/${clientName} \ ${taostools_bin_files} \ + ${taosx_bin} \ ${build_dir}/bin/taosadapter \ ${build_dir}/bin/udfd \ ${script_dir}/remove.sh \ From bee64e1a1557c4411aebd5ad6c8415593e916731 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 2 Sep 2022 13:21:10 +0800 Subject: [PATCH 5/7] feat: update taos-tools f169c0f for 3.0 (#16603) --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 74c2dbca30..3c7067a9f8 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 212c34d + GIT_TAG f169c0f SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From ee24902c3083bb49e36302f25510f15e4c1cb80f Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 2 Sep 2022 13:51:04 +0800 Subject: [PATCH 6/7] fix: fix doc select clause issue --- docs/en/12-taos-sql/06-select.md | 6 ------ docs/zh/12-taos-sql/06-select.md | 6 ------ 2 files changed, 12 deletions(-) diff --git a/docs/en/12-taos-sql/06-select.md b/docs/en/12-taos-sql/06-select.md index 1dd0caed38..1f86111925 100644 --- a/docs/en/12-taos-sql/06-select.md +++ b/docs/en/12-taos-sql/06-select.md @@ -52,11 +52,6 @@ window_clause: { | STATE_WINDOW(col) | INTERVAL(interval_val [, interval_offset]) [SLIDING (sliding_val)] [WATERMARK(watermark_val)] [FILL(fill_mod_and_val)] -changes_option: { - DURATION duration_val - | ROWS rows_val -} - group_by_clause: GROUP BY expr [, expr] ... HAVING condition @@ -126,7 +121,6 @@ SELECT DISTINCT col_name [, col_name ...] FROM tb_name; 1. Configuration parameter `maxNumOfDistinctRes` in `taos.cfg` is used to control the number of rows to output. The minimum configurable value is 100,000, the maximum configurable value is 100,000,000, the default value is 1,000,000. If the actual number of rows exceeds the value of this parameter, only the number of rows specified by this parameter will be output. 2. It can't be guaranteed that the results selected by using `DISTINCT` on columns of `FLOAT` or `DOUBLE` are exactly unique because of the precision errors in floating point numbers. -3. `DISTINCT` can't be used in the sub-query of a nested query statement, and can't be used together with aggregate functions, `GROUP BY` or `JOIN` in the same SQL statement. ::: diff --git a/docs/zh/12-taos-sql/06-select.md b/docs/zh/12-taos-sql/06-select.md index d8ff3f04ed..b0a7d88efe 100644 --- a/docs/zh/12-taos-sql/06-select.md +++ b/docs/zh/12-taos-sql/06-select.md @@ -53,11 +53,6 @@ window_clause: { | STATE_WINDOW(col) | INTERVAL(interval_val [, interval_offset]) [SLIDING (sliding_val)] [WATERMARK(watermark_val)] [FILL(fill_mod_and_val)] -changes_option: { - DURATION duration_val - | ROWS rows_val -} - group_by_clause: GROUP BY expr [, expr] ... HAVING condition @@ -127,7 +122,6 @@ SELECT DISTINCT col_name [, col_name ...] FROM tb_name; 1. cfg 文件中的配置参数 maxNumOfDistinctRes 将对 DISTINCT 能够输出的数据行数进行限制。其最小值是 100000，最大值是 100000000，默认值是 10000000。如果实际计算结果超出了这个限制，那么会仅输出这个数量范围内的部分。 2. 由于浮点数天然的精度机制原因，在特定情况下，对 FLOAT 和 DOUBLE 列使用 DISTINCT 并不能保证输出值的完全唯一性。 -3. 在当前版本下，DISTINCT 不能在嵌套查询的子查询中使用，也不能与聚合函数、GROUP BY、或 JOIN 在同一条语句中混用。 ::: From ae54c4a50bf5c0eeb84c2f31795cea5638019c7a Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 2 Sep 2022 14:24:45 +0800 Subject: [PATCH 7/7] enh: let show vnodes works --- source/common/src/systable.c | 8 +++-- source/dnode/mnode/impl/src/mndVgroup.c | 39 ++++++++++++++++--------- tests/script/tsim/db/basic1.sim | 31 ++++++++++++++++++++ 3 files changed, 61 insertions(+), 17 deletions(-) diff --git a/source/common/src/systable.c b/source/common/src/systable.c index f107e78478..3edf26cca7 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -257,10 +257,12 @@ static const SSysDbTableSchema subscriptionSchema[] = { }; static const SSysDbTableSchema vnodesSchema[] = { - {.name = "dnode_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, - {.name = "dnode_endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, - {.name = "status", .bytes = 20 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY, .sysInfo = true}, + {.name = "replica", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true}, + {.name = "status", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "dnode_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, + {.name = "dnode_ep", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, }; static const SSysTableMeta infosMeta[] = { diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 09eed7fb32..6ab979213d 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -791,32 +791,43 @@ static int32_t mndRetrieveVnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB if (pShow->pIter == NULL) break; for (int32_t i = 0; i < pVgroup->replica && numOfRows < rows; ++i) { - SVnodeGid *pVgid = &pVgroup->vnodeGid[i]; + SVnodeGid *pVgid = &pVgroup->vnodeGid[i]; + SColumnInfoData *pColInfo = NULL; cols = 0; - SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->vgId, false); - SName name = {0}; - char db[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - tNameFromString(&name, pVgroup->dbName, T_NAME_ACCT | T_NAME_DB); - tNameGetDbName(&name, varDataVal(db)); - varDataSetLen(db, strlen(varDataVal(db))); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)db, false); - - uint32_t val = 0; - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&val, false); + colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->replica, false); char buf[20] = {0}; STR_TO_VARSTR(buf, syncStr(pVgid->role)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)buf, false); + const char *dbname = mndGetDbStr(pVgroup->dbName); + char b1[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + if (dbname != NULL) { + STR_WITH_MAXSIZE_TO_VARSTR(b1, dbname, TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE); + } else { + STR_WITH_MAXSIZE_TO_VARSTR(b1, "NULL", TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE); + } pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->replica, false); // onlines + colDataAppend(pColInfo, numOfRows, (const char *)b1, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)&pVgid->dnodeId, false); + + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + char b2[TSDB_EP_LEN + VARSTR_HEADER_SIZE] = {0}; + if (pDnode != NULL) { + STR_WITH_MAXSIZE_TO_VARSTR(b2, pDnode->ep, TSDB_EP_LEN + VARSTR_HEADER_SIZE); + } else { + STR_WITH_MAXSIZE_TO_VARSTR(b2, "NULL", TSDB_EP_LEN + VARSTR_HEADER_SIZE); + } + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)b2, false); numOfRows++; } diff --git a/tests/script/tsim/db/basic1.sim b/tests/script/tsim/db/basic1.sim index 69eeb9347b..3f6bf05398 100644 --- a/tests/script/tsim/db/basic1.sim +++ b/tests/script/tsim/db/basic1.sim @@ -107,6 +107,37 @@ if $data30 != 12 then return -1 endi +print =============== show vnodes +sql show vnodes 1 +if $rows != 9 then + return -1 +endi + +if $data(4)[1] != 1 then + return -1 +endi + +if $data(4)[2] != leader then + return -1 +endi + +if $data(4)[3] != d2 then + return -1 +endi + +if $data(4)[4] != 1 then + return -1 +endi + +if $data(4)[5] != localhost:7100 then + return -1 +endi + +#sql show vnodes 'localhost:7100' +#if $rows != 9 then +# return -1 +#endi + print =============== drop database sql drop database d2 sql drop database d3